import datetime as dt

from rest_framework import status
from rest_framework.exceptions import NotAuthenticated, PermissionDenied
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView

from bumblebee.core.exceptions import (
    MissingFieldsError,
    NoneExistenceError,
    UrlParameterError,
)
from bumblebee.core.helpers import create_400, create_500
from bumblebee.feeds.api.serializers.feed_serializers import (
    FeedBuzzSerializer,
    FeedRebuzzSerializer,
)
from bumblebee.feeds.api.serializers.user_serializers import FeedUserSerializer
from bumblebee.feeds.utils import (
    get_follow_suggestions_for_user,
    get_folowing_buzzes_for_user,
)
from bumblebee.users.utils import DbExistenceChecker

class FeedBuzzListView(APIView):
    """ """

    permission_classes = [IsAuthenticated]

    def get_posts(self, *args, **kwargs):
        """ """

        return get_folowing_buzzes_for_user(self.request.user)

    def get(self, request, *args, **kwargs):
        """ """

        try:

            post_instances = self.get_posts()
            user_serializer = FeedUserSerializer(self.request.user, many=False)
            buzz_serializer = FeedBuzzSerializer(
                post_instances.get("buzzes"), many=True
            )
            rebuzz_serializer = FeedRebuzzSerializer(
                post_instances.get("rebuzzes"), many=True
            )

            return Response(
                data=dict(
                    updated_time=dt.datetime.now(),
                    user=user_serializer.data,
                    post=buzz_serializer.data + rebuzz_serializer.data,
                ),
                status=status.HTTP_200_OK,
            )

        except (MissingFieldsError, UrlParameterError, NoneExistenceError) as error:
            return Response(error.message, status=error.message.get("status"))

        except (PermissionDenied, NotAuthenticated) as error:
            return Response(
                create_400(
                    error.status_code,
                    error.get_codes(),
                    error.get_full_details().get("message"),
                ),
                status=error.status_code,
            )

        except Exception as error:
            return Response(
                create_500(
                    cause=error.args[0] or None,
                    verbose=f"Could not get feed due to an unknown error",
                ),
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )

class FeedFollowSuggestionsListView(APIView):
    """ """

    permission_classes = [IsAuthenticated]

    def get_suggestions(self, *args, **kwargs):
        """ """

        return get_follow_suggestions_for_user(self.request.user)

    def get(self, request, *args, **kwargs):
        """ """

        try:
            suggestion_instances = self.get_suggestions()
            user_serializer = FeedUserSerializer(self.request.user, many=False)
            suggestion_serializer = FeedUserSerializer(suggestion_instances, many=True)

            return Response(
                data=dict(
                    updated_time=dt.datetime.now(),
                    user=user_serializer.data,
                    suggestions=suggestion_serializer.data,
                ),
                status=status.HTTP_200_OK,
            )

        except (MissingFieldsError, UrlParameterError, NoneExistenceError) as error:
            return Response(error.message, status=error.message.get("status"))

        except (PermissionDenied, NotAuthenticated) as error:
            return Response(
                create_400(
                    error.status_code,
                    error.get_codes(),
                    error.get_full_details().get("message"),
                ),
                status=error.status_code,
            )

        except Exception as error:
            return Response(
                create_500(
                    cause=error.args[0] or None,
                    verbose=f"Could not get suggestions due to an unknown error",
                ),
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )


from django.shortcuts import get_object_or_404, render, redirect, HttpResponseRedirect
from django.views.generic import TemplateView, UpdateView
from django.contrib.auth import get_user_model
from .models import Message
from django.urls import reverse
from django.contrib import messages
from review.models import Review
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from .forms import EditProfileForm
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
# Create your views here.

from django.template import RequestContext

def handler404(request, *args, **argv):
    response = render('404.html', {}, context_instance=RequestContext(request))
    response.status_code = 404
    return response

class HomePageView(TemplateView):
    template_name = 'home.html'

def MessageView(request, username):
    User = get_user_model()
    message_user = get_object_or_404(User, username=username)
    all_messages = message_user.messages.all()
    if request.method == 'POST':
        message = request.POST['message']
        user_new_message = Message.objects.create(
            customuser=message_user, text=message)
        user_new_message.save()
        messages.success(
            request, 'Compliments dropped successfully, Create your own Account below to receive and give Anonymous compliments')
        return redirect('account_signup')

    return render(request, 'message.html', {'all_messages': all_messages,
                                            'message_user': message_user,
                                            })

class UserProfile(LoginRequiredMixin, TemplateView):
    template_name = 'user_profile.html'

CustomUser = get_user_model()

@login_required(login_url='account_login')

def EditProfile(request, username):
    user = get_object_or_404(get_user_model(), username=username)
    if request.method == 'POST':
        form = EditProfileForm(request.POST, instance=user)
        if form.is_valid():
            form.save()
            # user_profile = user.get_absolute_url()
            user_profile = reverse('user_profile')
            return HttpResponseRedirect(user_profile)

    form = EditProfileForm(instance=user)
    return render(request, 'edit_profile.html', {'form': form})

@login_required(login_url='account_login')
def delete_message(request, m_id):
    message_instance = get_object_or_404(Message, id=m_id)
    message_instance_user = message_instance.customuser
    if message_instance_user == request.user:
        message_instance.delete()
        return redirect(message_instance_user.get_absolute_url())
    else:
        return redirect('home')

@login_required(login_url='account_login')
def spam_message(request, m_id):
    # get the Message filtering by the id provided in the url as *args
    message_instance = get_object_or_404(Message, id=m_id)
    # the person who created the message
    message_instance_user = message_instance.customuser
    if message_instance_user == request.user:
        message_instance.text = 'The owner has marked this message as a spam.'
        message_instance.save()
        return redirect(message_instance_user.get_absolute_url())

    else:
        if request.user.is_authenticated:
            url = 'home'
        else:
            url = 'account_login'
        return redirect(url)

class AbouUs(TemplateView):
    template_name = 'about.html'

class ContactUs(TemplateView):
    template_name = 'contact.html'

def ReviewView(request):
    reviews = Review.objects.all()
    template_name = "review.html"
    paginator = Paginator(reviews, 5)
    page_number = request.GET.get('page')
    try:
        page_obj = paginator.get_page(page_number)
    except PageNotAnInteger:
        page_obj = paginator.page(page_number)

    except EmptyPage:
        page_obj = paginator.page(paginator.num_pages)

    return render(request, template_name, {'page_obj': page_obj})

@login_required(login_url='account_login')
def SettingsView(request):

    return render(request, 'settings.html', {})

def AddReview(request):
    if request.method == 'POST':
        name = request.POST['name']
        review = request.POST['review']
        occupation = request.POST['occupation']

        new_review = Review.objects.create(
            name=name, review=review, occupation=occupation)
        new_review.save()
        messages.success(request, 'Review submitted successfully')
        return redirect('review')
    return render(request, 'add_review.html', {})


"""
Created on Wed Feb 24 12:34:17 2021

@author: Narmin Ghaffari Laleh
"""

##############################################################################

from dataGenerator.dataSetGenerator_ClamMil import Generic_MIL_Dataset
import utils.utils as utils
from extractFeatures import ExtractFeatures
from utils.core_utils import Train_MIL_CLAM
from utils.data_utils import ConcatCohorts_Classic
from eval.eval import CalculatePatientWiseAUC, CalculateTotalROC, MergeResultCSV
from sklearn.model_selection import StratifiedKFold
import numpy as np
import os
import pandas as pd
import random
from sklearn import preprocessing
import torch

##############################################################################

def CLAM_MIL_Training(args):

    targetLabels = args.target_labels
    args.feat_dir = args.feat_dir[0]

    for targetLabel in targetLabels:
        for repeat in range(args.repeatExperiment):

            args.target_label = targetLabel
            random.seed(args.seed)
            args.projectFolder = utils.CreateProjectFolder(args.project_name, args.adressExp, targetLabel, args.model_name, repeat+1)
            print(args.projectFolder)
            if os.path.exists(args.projectFolder):
                continue
            else:
                os.mkdir(args.projectFolder)

            args.result_dir = os.path.join(args.projectFolder, 'RESULTS')
            os.makedirs(args.result_dir, exist_ok = True)
            args.split_dir = os.path.join(args.projectFolder, 'SPLITS')
            os.makedirs(args.split_dir, exist_ok = True)

            reportFile  = open(os.path.join(args.projectFolder,'Report.txt'), 'a', encoding="utf-8")
            reportFile.write('-' * 30 + '\n')
            reportFile.write(str(args))
            reportFile.write('-' * 30 + '\n')
            if args.extractFeature:
                imgs = os.listdir(args.datadir_train[0])
                imgs = [os.path.join(args.datadir_train[0], i) for i in imgs]
                ExtractFeatures(data_dir = imgs, feat_dir = args.feat_dir, batch_size = args.batch_size, target_patch_size = -1, filterData = True)
            print('\nLOAD THE DATASET FOR TRAINING...\n')
            patientsList, labelsList, args.csvFile = ConcatCohorts_Classic(imagesPath = args.datadir_train,
                                                                          cliniTablePath = args.clini_dir, slideTablePath = args.slide_dir,
                                                                          label = targetLabel, minNumberOfTiles = args.minNumBlocks,
                                                                          outputPath = args.projectFolder, reportFile = reportFile, csvName = args.csv_name,
                                                                          patientNumber = args.numPatientToUse)

            yTrueLabel = utils.CheckForTargetType(labelsList)
            le = preprocessing.LabelEncoder()
            yTrue = le.fit_transform(yTrueLabel)
            args.num_classes = len(set(yTrue))
            args.target_labelDict = dict(zip(le.classes_, range(len(le.classes_))))
            utils.Summarize(args, list(yTrue), reportFile)
            print('\nLoad the DataSet...')
            dataset = Generic_MIL_Dataset(csv_path = args.csvFile,
                                          data_dir = args.feat_dir,
                                          shuffle = False,
                                          seed = args.seed,
                                          print_info = True,
                                          label_dict  = args.target_labelDict,
                                          patient_strat = True,
                                          label_col = args.target_label,
                                          ignore = [],
                                          reportFile = reportFile)
            if len(patientsList) < 20:
                continue

            if args.train_full:
                print('-' * 30)
                print('IT IS A FULL TRAINING FOR ' + targetLabel + '!')
                train_data = pd.DataFrame(list(zip(patientsList, yTrue, yTrueLabel)), columns = ['PATIENT', 'yTrue', 'yTrueLabel'])
                if args.early_stopping:
                    val_data = train_data.groupby('yTrue', group_keys = False).apply(lambda x: x.sample(frac = 0.1))
                    train_data = train_data[~train_data['PATIENT'].isin(list(val_data['PATIENT']))]
                    train_data.reset_index(inplace = True, drop = True)
                    val_data.reset_index(inplace = True, drop = True)
                    df = pd.DataFrame({'train': pd.Series(train_data['PATIENT']), 'test': pd.Series([]), 'val' : pd.Series(val_data['PATIENT'])})
                    df.to_csv(os.path.join(args.split_dir, 'TrainSplit.csv'), index = False)
                    train_dataset, val_dataset, test_dataset = dataset.Return_splits(from_id = False, csv_path = os.path.join(args.split_dir, 'TrainSplit.csv'))
                else:
                    df = pd.DataFrame({'train': pd.Series(train_data['PATIENT']), 'test': pd.Series([]), 'val' : pd.Series([])})
                    df.to_csv(os.path.join(args.split_dir, 'TrainValSplit.csv'), index = False)
                    train_dataset, val_dataset, test_dataset = dataset.Return_splits(from_id = False, csv_path = os.path.join(args.split_dir, 'TrainValSplit.csv'))

                datasets = (train_dataset, val_dataset, test_dataset)
                model, _, _  = Train_MIL_CLAM(datasets = datasets, fold = 'FULL', args = args, trainFull = True)
                torch.save(model.state_dict(), os.path.join(args.projectFolder, 'RESULTS', 'finalModel'))
                print()
                print('-' * 30)
                reportFile.close()
            else:
                print('IT IS A ' + str(args.k) + 'FOLD CROSS VALIDATION TRAINING FOR ' + targetLabel + '!')
                patientID = np.array(patientsList)
                yTrue = np.array(yTrue)
                yTrueLabel = np.array(yTrueLabel)

                folds = args.k
                kf = StratifiedKFold(n_splits = folds, random_state = args.seed, shuffle = True)
                kf.get_n_splits(patientID, yTrue)

                foldcounter = 1

                for train_index, test_index in kf.split(patientID, yTrue):

                    testPatients = patientID[test_index]
                    trainPatients = patientID[train_index]
                    testyTrue = yTrue[test_index]
                    trainyTrue = yTrue[train_index]
                    testyTrueLabel = yTrueLabel[test_index]
                    trainyTrueLabel = yTrueLabel[train_index]

                    print('GENERATE NEW TILES...\n')
                    print('FOR TRAIN SET...\n')
                    train_data = pd.DataFrame(list(zip(trainPatients, trainyTrue, trainyTrueLabel)), columns = ['PATIENT', 'yTrue', 'yTrueLabel'])
                    print('FOR VALIDATION SET...\n')
                    val_data = train_data.groupby('yTrue', group_keys = False).apply(lambda x: x.sample(frac = 0.1))
                    train_data = train_data[~train_data['PATIENT'].isin(list(val_data['PATIENT']))]
                    print('FOR TEST SET...\n')
                    test_data = pd.DataFrame(list(zip(testPatients, testyTrue, testyTrueLabel)), columns = ['PATIENT', 'yTrue', 'yTrueLabel'])
                    train_data.reset_index(inplace = True, drop = True)
                    test_data.reset_index(inplace = True, drop = True)
                    val_data.reset_index(inplace = True, drop = True)

                    print('-' * 30)
                    print("K FOLD VALIDATION STEP => {}".format(foldcounter))
                    print('-' * 30)

                    df = pd.DataFrame({'train': pd.Series(train_data['PATIENT']), 'test': pd.Series(test_data['PATIENT']), 'val' : pd.Series(val_data['PATIENT'])})
                    df.to_csv(os.path.join(args.split_dir, 'TrainTestValSplit_{}.csv'.format(foldcounter)), index = False)
                    train_dataset, val_dataset, test_dataset = dataset.Return_splits(from_id = False, csv_path = os.path.join(args.split_dir, 'TrainTestValSplit_{}.csv'.format(foldcounter)))
                    datasets = (train_dataset, val_dataset, test_dataset)

                    model, results, test_auc  = Train_MIL_CLAM(datasets = datasets, fold = foldcounter, args = args, trainFull = False)
                    reportFile.write('AUC calculated by CLAM' + '\n')
                    reportFile.write(str(test_auc) + '\n')
                    reportFile.write('-' * 30 + '\n')

                    patients = []
                    filaNames = []
                    yTrue_test = []
                    yTrueLabe_test = []
                    probs = {}

                    for i_temp in range(args.num_classes):
                        key = utils.get_key_from_value(args.target_labelDict, i_temp)
                        probs[key] = []

                    for item in list(results.keys()):
                        temp = results[item]
                        patients.append(temp['PATIENT'])
                        filaNames.append(temp['FILENAME'])
                        yTrue_test.append(temp['label'])
                        yTrueLabe_test.append(utils.get_key_from_value(args.target_labelDict, temp['label']))

                        for key in list(args.target_labelDict.keys()):
                            probs[key].append(temp['prob'][0][utils.get_value_from_key(args.target_labelDict, key)])

                    probs = pd.DataFrame.from_dict(probs)
                    df = pd.DataFrame(list(zip(patients, filaNames, yTrue_test, yTrueLabe_test)), columns =['PATIENT', 'FILENAME', 'yTrue', 'yTrueLabel'])
                    df = pd.concat([df, probs], axis = 1)
                    testResultsPath = os.path.join(args.result_dir, 'TEST_RESULT_SLIDE_BASED_FOLD_' + str(foldcounter) + '.csv')
                    df.to_csv(testResultsPath, index = False)
                    CalculatePatientWiseAUC(resultCSVPath = testResultsPath, args = args, foldcounter = foldcounter , clamMil = True, reportFile = reportFile)
                    reportFile.write('-' * 30 + '\n')
                    foldcounter +=  1

                patientScoreFiles = []
                slideScoreFiles = []
                for i in range(args.k):
                    patientScoreFiles.append('TEST_RESULT_PATIENT_BASED_FOLD_' + str(i + 1) + '.csv')
                    slideScoreFiles.append('TEST_RESULT_SLIDE_BASED_FOLD_' + str(i + 1) + '.csv')

                CalculateTotalROC(resultsPath = args.result_dir, results = patientScoreFiles, target_labelDict =  args.target_labelDict, reportFile = reportFile)
                reportFile.write('-' * 30 + '\n')
                MergeResultCSV(args.result_dir, slideScoreFiles, milClam = True)
                reportFile.close()

##############################################################################


from .common import *  # NOQA
import pytest

project_detail = {"project": None, "namespace": None, "cluster": None,
                  "project2": None, "namespace2": None, "cluster2": None}
user_token = {"user_c1_p1_owner": {"user": None, "token": None},
              "user_c1_p1_member": {"user": None, "token": None},
              "user_c1_p2_owner": {"user": None, "token": None},
              "user_standard": {"user": None, "token": None}}

CATALOG_URL = "https://git.rancher.io/charts"
MYSQL_EXTERNALID_037 = "catalog://?catalog=library&template=mysql" \
                       "&version=0.3.7"
MYSQL_EXTERNALID_038 = "catalog://?catalog=library&template=mysql" \
                       "&version=0.3.8"
WORDPRESS_EXTID = "catalog://?catalog=library&template=wordpress" \
                  "&version=1.0.5"

def cluster_and_client(cluster_id, mgmt_client):
    cluster = mgmt_client.by_id_cluster(cluster_id)
    url = cluster.links.self + '/schemas'
    client = rancher.Client(url=url,
                            verify=False,
                            token=mgmt_client.token)
    return cluster, client

def wait_for_template_to_be_created(client, name, timeout=45):
    found = False
    start = time.time()
    interval = 0.5
    while not found:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for templates")
        templates = client.list_template(catalogId=name)
        if len(templates) > 0:
            found = True
        time.sleep(interval)
        interval *= 2

def check_condition(condition_type, status):
    def _find_condition(resource):
        if not hasattr(resource, "conditions"):
            return False

        if resource.conditions is None:
            return False

        for condition in resource.conditions:
            if condition.type == condition_type and condition.status == status:
                return True
        return False

    return _find_condition

def test_tiller():
    name = random_test_name()
    admin_client = get_user_client()

    clusters = admin_client.list_cluster(name=CLUSTER_NAME).data
    assert len(clusters) > 0
    cluster_id = clusters[0].id

    p = admin_client. \
        create_project(name="test-" + random_str(),
                       clusterId=cluster_id,
                       resourceQuota={
                           "limit": {
                               "secrets": "1"}},
                       namespaceDefaultResourceQuota={
                           "limit": {
                               "secrets": "1"}}
                       )

    p = admin_client.reload(p)
    proj_client = rancher.Client(url=p.links.self +
                                 '/schemas', verify=False,
                                 token=USER_TOKEN)
    # need a cluster scoped client to create a namespace
    _cluster, cluster_client = cluster_and_client(cluster_id, admin_client)
    ns = cluster_client.create_namespace(name=random_str(),
                                         projectId=p.id,
                                         resourceQuota={
                                             "limit": {
                                                 "secrets": "1"
                                             }}
                                         )
    wait_for_template_to_be_created(admin_client, "library")
    app = proj_client.create_app(
        name=name,
        externalId=WORDPRESS_EXTID,
        targetNamespace=ns.name,
        projectId=p.id,
        answers=get_defaut_question_answers(admin_client, WORDPRESS_EXTID)
    )

    app = proj_client.reload(app)
    # test for tiller to be stuck on bad installs
    wait_for_condition(proj_client, app, check_condition('Installed', 'False'))
    # cleanup by deleting project
    admin_client.delete(p)

def test_app_deploy():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    wait_for_template_to_be_created(admin_client, "library")
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    proj_client.delete(app)

def test_app_delete():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    app = proj_client.delete(app)
    validate_app_deletion(proj_client, app.id)

def test_app_upgrade_version():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    new_answer = get_defaut_question_answers(
            admin_client,
            MYSQL_EXTERNALID_038)
    app = proj_client.update(
        obj=app,
        externalId=MYSQL_EXTERNALID_038,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=new_answer)
    app = proj_client.reload(app)
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_038)
    assert app.externalId == MYSQL_EXTERNALID_038, "incorrect template version"
    proj_client.delete(app)

def test_app_rollback():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    rev_id = app.appRevisionId
    new_answer = get_defaut_question_answers(
            admin_client,
            MYSQL_EXTERNALID_038)
    app = proj_client.update(
        obj=app,
        externalId=MYSQL_EXTERNALID_038,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=new_answer)
    app = proj_client.reload(app)
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_038)
    assert app.externalId == MYSQL_EXTERNALID_038, "incorrect template version"
    proj_client.action(obj=app,
                       action_name='rollback',
                       revisionId=rev_id)
    app = proj_client.reload(app)
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    assert app.externalId == MYSQL_EXTERNALID_037, "incorrect template version"
    proj_client.delete(app)

def test_app_answer_override():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answers = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answers)
    print("App is active")
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    answers["mysqlUser"] = "admin1234"
    app = proj_client.update(
        obj=app,
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answers)
    app = proj_client.reload(app)
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037, answers)
    assert app["answers"].mysqlUser == "admin1234", \
        "incorrect answer upgrade"
    proj_client.delete(app)

def test_rbac_app_project_scope_deploy():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    catalog = admin_client.create_projectCatalog(
        name="projectcatalog",
        baseType="projectCatalog",
        branch="master",
        url=CATALOG_URL,
        projectId=project_detail["project"].id)
    time.sleep(5)
    pId = project_detail["project"].id.split(":")[1]
    catalog_proj_scoped_ext_id = "catalog://?catalog=" + pId + \
                                 "/projectcatalog&type=" \
                                 "projectCatalog&template=" \
                                 "mysql&version=0.3.8"
    answers = get_defaut_question_answers(
        admin_client,
        catalog_proj_scoped_ext_id)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=catalog_proj_scoped_ext_id,
        answers=answers,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id)
    validate_catalog_app(proj_client, app, catalog_proj_scoped_ext_id)
    p2, ns2 = create_project_and_ns(
        USER_TOKEN,
        project_detail["cluster"],
        random_test_name("testapp"))
    #Assign role
    assign_members_to_project(admin_client,
                              user_token["user_c1_p2_owner"]["user"],
                              p2,
                              "project-owner")
    #Verify "project-owner" of p1 can list the added catalog
    user1_client = get_client_for_token(
        user_token["user_c1_p1_owner"]["token"])
    catalogs_list = user1_client.list_projectCatalog()
    assert len(catalogs_list) == 1, \
        "Project catalog not found for the user"
    assert catalogs_list["data"][0]["name"] == \
           "projectcatalog", "Incorrect project catalog found"

    # Verify "project-member" of p1 can list the added catalog
    user2_client = get_client_for_token(
        user_token["user_c1_p1_member"]["token"])
    catalogs_list_2 = user2_client.list_projectCatalog()
    assert len(catalogs_list_2) == 1, \
        "Project catalog not found for the user"

    # Verify "project-owner" of p2 CANNOT list the added catalog
    user3_client = get_client_for_token(
        user_token["user_c1_p2_owner"]["token"])
    catalogs_list_3 = user3_client.list_projectCatalog()
    assert len(catalogs_list_3) == 0, \
        "Project catalog found for the user"

    # Verify A standard user CANNOT list the added catalog
    user4_client = get_client_for_token(
        user_token["user_standard"]["token"])
    catalogs_list_4 = user4_client.list_projectCatalog()
    assert len(catalogs_list_4) == 0, \
        "Project catalog found for the user"
    admin_client.delete(p2)

@pytest.fixture(scope='module', autouse="True")
def create_project_client(request):
    client = get_admin_client()
    clusters = client.list_cluster(name=CLUSTER_NAME).data
    assert len(clusters) > 0
    project_detail["project"], project_detail["namespace"] = \
        create_project_and_ns(USER_TOKEN, clusters[0],
                              random_test_name("testapp"))
    project_detail["cluster"] = clusters[0]
    #create users
    user_token["user_c1_p1_owner"]["user"], \
        user_token["user_c1_p1_owner"]["token"] = create_user(client)
    user_token["user_c1_p1_member"]["user"], \
        user_token["user_c1_p1_member"]["token"] = create_user(client)
    user_token["user_c1_p2_owner"]["user"], \
        user_token["user_c1_p2_owner"]["token"] = create_user(client)
    user_token["user_standard"]["user"], \
        user_token["user_standard"]["token"] = create_user(client)
    #Assign roles to the users
    assign_members_to_project(client,
                              user_token["user_c1_p1_owner"]["user"],
                              project_detail["project"],
                              "project-owner")
    assign_members_to_project(client,
                              user_token["user_c1_p1_member"]["user"],
                              project_detail["project"],
                              "project-member")

    def fin():
        client = get_user_client()
        client.delete(project_detail["project"])
    request.addfinalizer(fin)


""" this is a mixture of the best #free twitter sentimentanalysis modules on github.
    i took the most usable codes and mixed them into one because all of them
    where for a linguistical search not usable and did not show a retweet or a full tweet
    no output as csv, only few informations of a tweet, switching language
    or even to compare linguistic features in tweets of two different langauges and etc. etc ...
    special and many many thanks to https://github.com/vprusso/youtube_tutorials who showed on his
    page a tutorial on how to do a sentimentanalysis with python
    i did this for users with not much skills and linguistical background to help them to get a corpus of twitterdata
    and to show them how to do a comparison between sentence based vs document based sentimentanalysis
    credits to all AVAILABLE FREE AND SIMPLE sentimentanalysis programms (dec. 2019) on github.
    many thanks to everybody and of course to github for making this exchange and usage possible!
    cemre koc (Goethe University, Frankfurt) Python3.7
"""

from textblob import TextBlob   #Sentimentlexikon FOR GERMAN (TEXTBLOB_DE import textblob_de
import re                       #modul for regular expressions

from tweepy import API                      #Twitter API modul for more info: look tweepy doc please!
from tweepy import Cursor
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream

import tweepy                           #usage of diffrent feautres of my programm
import sys                              #only if wanted
import csv                              ##only if wanted (see rest of programm)
import pandas as pd                     #pandas for illustration
import authentification                 #access to twitter
import numpy as np                      #collection of tweets via numpy
import matplotlib.pyplot as plt         #if needed (see below for ploting)
import numpy

#output screen (if you use pycharm for full screen view)
#only if needed
pd.set_option('display.max_rows', 1000000000000)
pd.set_option('display.max_columns', 1000000000)
pd.set_option('display.width', 100000000000)
pd.set_option('display.float_format', '{:20,.2f}'.format)

#for maximal OUTPUT!
#pd.set_option('display.max_colwidth', -1)

#TWITTER AUTHENTIFICTION (Twitter development)
#please fill for that the identification.py with your credentials!
#you need a twitter developer account for getting these informations
class TwitterAuthenticator():

    def authenticate_twitter_app(self):
        auth = OAuthHandler(authentification.CONSUMER_KEY, authentification.CONSUMER_SECRET)
        auth.set_access_token(authentification.ACCESS_TOKEN, authentification.ACCESS_TOKEN_SECRET)
        return auth

#TWITTER CLIENT SERVER
class TwitterClient():
    def __init__(self, twitter_user=None):
        self.auth = TwitterAuthenticator().authenticate_twitter_app()
        self.twitter_client = API(self.auth)
        self.twitter_user = twitter_user

    def get_twitter_client_api(self):
        return self.twitter_client

    def get_user_timeline_tweets(self, num_tweets):
        tweets = []
        for tweet in Cursor(self.twitter_client.user_timeline, id=self.twitter_user).items(num_tweets):
            tweets.append(tweet)
        return tweets

    def get_friend_list(self, num_friends):
        friend_list = []
        for friend in Cursor(self.twitter_client.friends, id=self.twitter_user).items(num_friends):
            friend_list.append(friend)
        return friend_list

    def get_home_timeline_tweets(self, num_tweets):
        home_timeline_tweets = []
        for tweet in Cursor(self.twitter_client.home_timeline, id=self.twitter_user).items(num_tweets):
            home_timeline_tweets.append(tweet)
        return home_timeline_tweets

#TWITTER STREAMER FOR STREAMING AND LIVE TWEETS
class TwitterStreamer():

    def __init__(self):
        self.twitter_autenticator = TwitterAuthenticator()

    def stream_tweets(self, fetched_tweets_filename, hash_tag_list):
        # AUTHENTIFICATION AND CONNECTION TO API
        listener = TwitterListener(fetched_tweets_filename)
        auth = self.twitter_autenticator.authenticate_twitter_app()
        stream = Stream(auth, listener)

        #you can use the stream.filter for defining the search for words/hasthags!!!!!!
        #same sentimentanalysis works for words or hashtags!!!
        stream.filter(track=hash_tag_list)

#TWITTER STREAM LISTENER FOR PRINTING TWEETS
class TwitterListener(StreamListener):

    def __init__(self, fetched_tweets_filename):
        self.fetched_tweets_filename = fetched_tweets_filename

    def on_data(self, data):
        try:
            print(data)
            with open(self.fetched_tweets_filename, 'a') as tf:
                tf.write(data)
            return True
        except BaseException as e:
            print("Error on_data %s" % str(e))
        return True

    def on_error(self, status):
        if status == 420:
            #OCCURS IF RATE LIMIT IS PASSED
            return False
        print(status)

#FOR ANALYZING CLEANING TWEETS (TO CONTENT)
class TweetAnalyzer():
#DELTETE ALL UNNECESSARY CHARACTERS
    def clean_tweet(self, tweet):
        return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", tweet).split())

#SIMPLE SENTIMENTANALYSIS VIA TEXTBLOB (englisch)
    def analyze_sentiment(self, tweet):
        analysis = TextBlob(self.clean_tweet(tweet))

        if analysis.sentiment.polarity > 0:
            return 1
        elif analysis.sentiment.polarity == 0:
            return 0
        else:
            return -1
#You can use the following classification of polarity for sentence based analysis
#since i am using this programm for document level classification I left it with 1 -1  and 0
#       if (polarity == 0):
#            print("Neutral")
#     elif (polarity > 0 and polarity <= 0.3):
#            print("Schwach positiv")
#        elif (polarity > 0.3 and polarity <= 0.6):
#            print("positiv")
#        elif (polarity > 0.6 and polarity <= 1.0):
#            print("Stark positiv")
#        elif (polarity > -0.3 and polarity <= 0):
#            print("schwach negativ")
#        elif (polarity > -0.6 and polarity <= -0.3):
#            print("Negativ")
#        elif (polarity >= -1.0 and polarity <= -0.6):
#            print("Stark negativ")

    def tweets_to_data_frame(self, tweets):
        df = pd.DataFrame(data=[tweet.full_text for tweet in tweets], columns=['tweets'])
#THIS IS FOR RETWEETS OF A CERTAIN TWEET! BUT BE CARFUL ONLY A CERTAIN NUMBER OF TWEETS PER DAY!
#TWITTER RESTRICTION
#remove the """ for usage!

        """replies = []
        non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
        for full_tweets in tweepy.Cursor(api.user_timeline, screen_name='GretaThunberg', timeout=999999).items(20):
            for tweet in tweepy.Cursor(api.search, q='to:GretaThunberg', since_id=1203618558267273225,
                                       result_type='recent',
                                       timeout=999999).items(100):
                if hasattr(tweet, 'in_reply_to_status_id_str'):
                    if (tweet.in_reply_to_status_id_str == full_tweets.id_str):
                        replies.append(tweet.text)
            print("Tweet :", full_tweets.text.translate(non_bmp_map))
            for elements in replies:
                print("Replies :", elements)
            # replies.clear()"""

#DATA SET VIA DATAFRAME TO SHOW WITH NUMPY
#YOU CAN PRINT GIVEN DATA LIKE LENGTH RETWEET NUMBER LANGUAGE etc. CHOSSE:

#['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__',
# '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '_
# _lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '_
# _sizeof__', '__str__', '__subclasshook__', '__weakref__', '_api', '_json', 'author', 'contributors', 'coordinates',
# 'created_at', 'destroy', 'display_text_range', 'entities', 'favorite', 'favorite_count', 'favorited',
# 'full_text', 'geo', 'id', 'id_str', 'in_reply_to_screen_name', 'in_reply_to_status_id', 'in_reply_to_status_id_str',
# 'in_reply_to_user_id', 'in_reply_to_user_id_str', 'is_quote_status', 'lang', 'parse', 'parse_list', 'place',
# 'possibly_sensitive', 'quoted_status', 'quoted_status_id', 'quoted_status_id_str', 'quoted_status_permalink',
# 'retweet', 'retweet_count', 'retweeted', 'retweets', 'source', 'source_url', 'truncated', 'user']

        df['id'] = np.array([tweet.id for tweet in tweets])
        df['len'] = np.array([len(tweet.full_text) for tweet in tweets])
        df['date'] = np.array([tweet.created_at for tweet in tweets])
        #df['source'] = np.array([tweet.source for tweet in tweets])
        df['likes'] = np.array([tweet.favorite_count for tweet in tweets])
        df['retweets'] = np.array([tweet.retweet_count for tweet in tweets])
        df["lang"] = ([tweet.lang for tweet in tweets])
        #df["in_reply_to_status_id_str"] = ([tweet.replies for tweet in tweets])

        return df

#Programm begins here!!

if __name__ == '__main__':

    twitter_client = TwitterClient()
    tweet_analyzer = TweetAnalyzer()

    api = twitter_client.get_twitter_client_api()
    #TWEET MODE EXTENDED FOR FULL TWEET OUTPUT!! RETWEETS STAY THE SAME!
    #COUNT= LAST TWEET NUMBER OF USER (SCREEN NAME)
    #HERE FOR GRETA THUNBERG, JUST DELETE AND TYPE ACCOUNT NAME TO CHANGE
    #FOR EXAMPLE rtErdogan (for president of turkey), realDonaldTrump (for Trump) etc...
    tweets = api.user_timeline(screen_name="GretaThunberg", count=200, tweet_mode="extended")

    #print DATA
    print(dir(tweets[0]))
    #print(tweets[0].retweet_count)  #retweet count print
    #sentimentanalysis for printing it in a dataframe with the other informations!
    df = tweet_analyzer.tweets_to_data_frame(tweets)
    df['sentiment'] = np.array([tweet_analyzer.analyze_sentiment(tweet) for tweet in df['tweets']])

    #AVARAGE LENGTH OF ALL TWEETS
    #print(np.mean(df['len']))

    # GET NUMBER OF LIKES
    #print(np.max(df['likes']))

    # GET NUMBER OF RETWEETS
    #print(np.max(df['retweets']))

    #EXAMPLE RETWEET STATUS OF A CERTAIN TWEET ID
    #To get ID you need to look on your broswers URL of this CERTAIN TWEET
    #print(np.max(df["lang"]))
    ##print(df.in_reply_to_status_id[1075801005504258061])

    #ANYWAY THERE IS A RESTRICTION SINCE 2019 ON ONLY 200 TWEETS
    #THANK YOU CAMBRIDGE ANALYTICA
    print(df.head(200))

    #    DO CSV FILE (DELETE OR NAME IT NEW TO MAKE IT SEPRATE)
    #df.to_csv('KocSentiment.csv')

    #TIME SERIES FOR CHART VIEW!!! DONT FORGET TO TURN ON MATPLOT LIBRARY
    #time_likes = pd.Series(data=df['len'].values, index=df['date'])
    #time_likes.plot(figsize=(16, 4), color='r')
    #plt.show()

    #time_favs = pd.Series(data=df['likes'].values, index=df['date'])
    #time_favs.plot(figsize=(16, 4), color='r')
    #plt.show()

    #time_retweets = pd.Series(data=df['retweets'].values, index=df['date'])
    #time_retweets.plot(figsize=(16, 4), color='r')
    #plt.show()

    #LAYERED VIEW! FOR COMPARISON !!
    #time_likes = pd.Series(data=df['likes'].values, index=df['date'])
    #time_likes.plot(figsize=(16, 4), label="likes", legend=True)

    #time_retweets = pd.Series(data=df['retweets'].values, index=df['date'])
    #time_retweets.plot(figsize=(16, 4), label="retweets", legend=True)
    #plt.show()


import numpy as np
from astropy import table
from glob import glob
import pandas as pd
from scipy.stats import binned_statistic

def get_outlier_fraction(tbl, suffix='', bins=20):
    diff = np.array(np.abs(tbl['z_est'] - tbl['z']) > 0.15 * (1 + tbl['z']),
                    dtype=float)
    stat = binned_statistic(tbl['z%s' % suffix], diff, statistic='mean',
                            bins=bins)
    return stat.statistic

def get_diagnostics(z1, z2):
    diff = np.array(z1 - z2) / (1 + np.array((z1)))
    outlier_mask = np.abs(diff) < 0.15  # * (1 + z1)
    med = np.median(diff)
    mad = np.median(np.abs(diff - med))
    return 100*np.array((np.mean(diff[outlier_mask]),
                         np.std(diff[outlier_mask]),
                         med, mad, 1-outlier_mask.mean()))

def run_for_table_old(name, min=None):
    t = table.Table.from_pandas(pd.read_csv(name))
    tmax = t['mags'].max()
    t = t[t['z_est'] > 0]
    if min is None:
        max_mag = 2
        while max_mag <= max(max_mag, tmax):
            t_ = t[t['mags'] <= max_mag]
            if len(t_) > 0.9 * len(t):
                break
            max_mag += 1
        diag_old = get_diagnostics(t_['z'], t_['z_est'])
        max_outlier_rate = diag_old[-1]
        used_fraction = len(t_)*100 / len(t)
        i = 2
        for i in range(max_mag, tmax + 1):
            t_ = t[t['mags'] <= i]
            x = t_['z']
            y = t_['z_est']
            if len(t_) == 0:
                break
            diag = get_diagnostics(x, y)
            print(name, i, '%.3f' % diag[-1], len(t_), i,
                  '%.3f' % max_outlier_rate)
            if diag[-1] > max_outlier_rate:
                break
            diag_old = diag
            used_fraction = len(t_)*100 / len(t)
    else:
        i = min + 1
        t_ = t[t['mags'] <= int(min)]
        diag_old = get_diagnostics(t_['z'], t_['z_est'])
        used_fraction = len(t_)*100 / len(t)
    return len(t_['z']), diag_old, i - 1, used_fraction

def run_for_table(name):
    if name.endswith('csv'):
        df = pd.read_csv(name)
    elif name.endswith('parquet'):
        df = pd.read_parquet(name, columns=['mags', 'z', 'z_est'])
    else:
        return [0, [0]*5]
    x = df['z']
    y = df['z_est']
    diag = get_diagnostics(x, y)
    return len(df['z']), diag

def name_to_caption(name):
    output = name.split('/')[-1].replace('.csv', '').replace('.parquet', '')
    if '-' in output:
        output_parts = output.split('-')[1:-1]
        output = ' '.join([s.replace('_', ' ').replace('+', '')
                           for s in output_parts])
    output = output.replace('  ', ' ').replace(' ', ', ')
    return output

def get_stats_for_file(name, **kwargs):
    output = table.Table(names=['Name', 'Mean', 'Std', 'Median',
                                'MAD', 'Outliers',
                                'Count'],
                         dtype=[str, float, float, float, float, float,
                                int])
    row = run_for_table(name, **kwargs)
    output.add_row([name_to_caption(name), *row[1], row[0]])
    return output

def get_stats_for_folder(folder, **kwargs):
    output = table.Table(names=['Name', 'Mean', 'Std', 'Median',
                                'MAD', 'Outliers',
                                'Count'],
                         dtype=[str, float, float, float, float, float,
                                int])

    names = glob('%s/*.csv' % folder) + glob('%s/*.parquet' % folder)
    names.sort()
    for f in names:
        row = run_for_table(f, **kwargs)
        output.add_row([name_to_caption(f), *row[1], row[0]])
    return output


import netCDF4
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.colors import Normalize
import cartopy.crs as ccrs
import matplotlib.colors as colors

# Land cover
lc = netCDF4.Dataset("../data/LandCover_half.nc")
lc.set_auto_mask(True)
lc.variables
land_cover = lc["Land Cover"][:]

# fix landcover
halfway = np.shape(land_cover)[1]//2
first_half = land_cover[:,:halfway]
second_half = land_cover[:,halfway:]
land_cover = np.concatenate((second_half,first_half), axis=1)
#land_cover[land_cover == 0] = np.nan
#land_cover[land_cover > 10] = np.nan

land_cover_int = land_cover.astype(int)  # Convert land_cover to int

unique_land_cover = np.unique(land_cover_int)

# plot
fig = plt.figure(figsize=(10, 6))
ax = plt.axes(projection=ccrs.PlateCarree())
image = ax.imshow(land_cover_int , origin="lower", extent=[-180, 180, -90, 90], transform=ccrs.PlateCarree())
ax.coastlines()
ax.set_global()
plt.show()

#ds = netCDF4.Dataset("../results/iso_result_mon_mean.nc")
ds = netCDF4.Dataset("../results/iso_result_OMI_filterd_mon_mean.nc")
ds.set_auto_mask(True)

dar = ds["J_times_IspS"][:]
dar[dar > 1000] = np.nan

dar_mean = np.nanmean(dar, axis=0)
dar_mean.shape

# Subset everything above 30 degrees North
def calculate_mean_subset(latitude_threshold,latitude, data):
    lat_indices = np.where(latitude > latitude_threshold)[0]
    data_subset = data[:, lat_indices, :]
    data_mean_subset = np.nanmean(data_subset, axis=0)
    anomaly_subset = np.subtract(data_subset, data_mean_subset)
    norm_subset = np.divide(anomaly_subset, data_mean_subset)
    mean_data_subset = np.column_stack((np.arange(12), [np.nanmean(norm_subset[i, :, :]) for i in range(12)]))
    return mean_data_subset

mean_iso_subset_30 = calculate_mean_subset(30, latitude_iso, dar)

print(mean_iso_subset_30)

calculate_mean_subset(30, latitude_iso, dar)

# Normalize the data between the maximum and 0
anomaly = np.subtract(dar, dar_mean)
norm = np.divide(anomaly, dar_mean)

np.nanmean(norm[5,:,:])

np.nanmin(norm[5,:,:])
np.nanmax(norm[5,:,:])

# Create a figure with multiple subplots
fig, axs = plt.subplots(3, 4, figsize=(16, 12), subplot_kw={'projection': ccrs.PlateCarree()})

# Iterate over the range of plots you want to display (0 to 11)
for i in range(12):
    # Calculate the subplot indices based on the iteration index
    row = i // 4  # Row index
    col = i % 4   # Column index

    # Plot the normalized data in the current subplot
    im = axs[row, col].imshow(norm[i, :, :], origin="lower", extent=[-180, 180, -90, 90], transform=ccrs.PlateCarree(), cmap='RdYlGn', vmin=-4, vmax=4, )

    axs[row, col].set_title("Iso_{:02d}".format(i))  # Format the title with leading zeros

    # Add coastlines
    axs[row, col].coastlines()

# Create a colorbar for the figure
fig.colorbar(im, ax=axs, fraction=0.022, pad=0.03, location='bottom')

# Adjust the spacing between subplots
plt.tight_layout()

# Move the colorbar below the subplots
fig.subplots_adjust(bottom=0.15)

# Display the plot
plt.show()
ds.close()

np.where(land_cover_int == 1)
norm[5, np.where(land_cover_int == 1)]
mean_norm_values = []
for lc in unique_land_cover:
    indices = np.where(land_cover_int == lc)  # Find indices of matching land_cover values
    mean_norm = np.mean(norm[indices])    # Calculate mean norm for the current land_cover value
    mean_norm_values.append(mean_norm)

print("Mean norm values for each land_cover group:")
for lc, mean_norm in zip(unique_land_cover, mean_norm_values):
    print(f"Land Cover {lc}: {mean_norm}")

#### Compare to the HCHO
HCHO = netCDF4.Dataset("../data/OMI_iso_estimate/mon_average_OMI.nc")
HCHO.set_auto_mask(False)

## Conveert isoprene values from the units kg/gridcell/month to kg/m2/month
def gridcell_to_m2(latitude):
    "Find the number of m2 in each grid cell in a 0.5 by 0.5 degree grid"
    half_degree_lat = 111111.1/2 # latitude lengths stay at about 111.1km per degree
    half_degree_lon = np.cos(np.deg2rad(latitude)) * (111111.1/2) # equation to get the length of each half degree of longitude (changes with latitude)
    meter_square_gridcell = half_degree_lat * half_degree_lon
    return meter_square_gridcell

latitudes = [float(lat) for lat in HCHO['lat'][:]] #get each latitude degree as a float
no_m2_in_grid = [gridcell_to_m2(lat) for lat in latitudes] #gets the number of m2 blocks in each grid cell
tiled = np.tile(no_m2_in_grid, (len(HCHO['lon'][:]), 1)).T #repeat across each longitude as distance remains the same across each latitude degree

# Get the isoprene emmited per m2
isoprene_per_m2 = (HCHO["EMworldC5H8"][:])/(tiled)

# Calculate the threshold value for the top 5%
threshold = np.percentile(isoprene_per_m2, 99.98)

isoprene_per_m2[isoprene_per_m2 == 0] = np.nan

# Set values above the threshold to NaN
isoprene_per_m2[isoprene_per_m2 > threshold] = np.nan

HCHO_mean = np.nanmean(isoprene_per_m2, axis=0)
HCHO_anomaly = np.subtract(isoprene_per_m2, HCHO_mean)
HCHO_norm = np.divide(HCHO_anomaly, HCHO_mean)

i = 6
np.nanmean(HCHO_norm[i,:,:])
np.nanmin(HCHO_norm[i,:,:])

# Subset everything above 30 degrees North
latitude_HCHO = HCHO.variables["lat"][:]

def HCHO_std_anomoly_subset(latitude_threshold, latitude, data):
    lat_indices = np.where(latitude > latitude_threshold)[0]
    data_subset = np.flipud(data)[:, lat_indices, :]
    data_mean_subset = np.nanmean(data_subset, axis=0)
    data_anomaly_subset = np.subtract(data_subset, data_mean_subset)
    data_norm_subset = np.divide(data_anomaly_subset, data_mean_subset)
    std_anomoly_subset = np.column_stack((np.arange(12), [np.nanmean(data_norm_subset[i, :, :]) for i in range(12)]))
    return std_anomoly_subset

mean_HCHO_subset_30 = HCHO_std_anomoly_subset(30, latitude_HCHO, isoprene_per_m2)

print(mean_HCHO_subset_30)
HCHO_std_anomoly_subset(0, latitude_HCHO, isoprene_per_m2)

# Create a figure and a single subplot
fig, ax = plt.subplots(figsize=(10, 4))

# Plot the first line with a red color
ax.plot(mean_HCHO_subset_30[:, 0], mean_HCHO_subset_30[:, 1], 'r-', label='HCHO')

# Plot the second line with a blue color
ax.plot(mean_iso_subset_30[:, 0], mean_iso_subset_30[:, 1], 'b-', label='Model')

# Set the title and legend
ax.set_title('Average anomaly in the Northern temperate zone (>30 degrees)')
ax.legend()

# Display the plot
plt.show()

np.nanmin(HCHO_norm)

# Create a figure with multiple subplots
fig, axs = plt.subplots(3, 4, figsize=(16, 12), subplot_kw={'projection': ccrs.PlateCarree()})

# Iterate over the range of plots you want to display (0 to 11)
for i in range(12):
    # Calculate the subplot indices based on the iteration index
    row = i // 4  # Row index
    col = i % 4   # Column index

    # Plot the normalized data in the current subplot
    im = axs[row, col].imshow(np.flipud(HCHO_norm[i, :, :]), origin="lower", extent=[-180, 180, -90, 90], transform=ccrs.PlateCarree(), cmap='RdYlGn')#, vmin=-4, vmax=4, )

    axs[row, col].set_title("OMI_{:02d}".format(i))  # Format the title with leading zeros

    # Add coastlines
    axs[row, col].coastlines()

# Create a colorbar for the figure
fig.colorbar(im, ax=axs, fraction=0.022, pad=0.03, location='bottom')

# Adjust the spacing between subplots
plt.tight_layout()

# Move the colorbar below the subplots
fig.subplots_adjust(bottom=0.15)

# Display the plot
plt.show()

HCHO.close()


# -*- coding: utf-8 -*-
"""

Created on Tue Apr 28 23:15:17 2020

@author: malrawi

"""

import dicttoxml
import ruamel.yaml # https://pypi.org/project/ruamel.yaml/
import json

def read_yaml_as_dict(fname):
    """
    A function used to read the ddoif dictionary in yaml format and return it as a python dictionary.
    This functiona makes use of ruamel.yaml https://pypi.org/project/ruamel.yaml/

    ...

    Input arguments
    ----------
    - in_f: the name of yaml file

    Output arguments:
        - ddoif_dict: All attributes  the ddoif dictionary

    Methods
    -------
        ddoif_dict = ddoif_read(in_f='ATest.ddof', check_CRC=True)

    """
    with open(fname) as fp:
        # The FullLoader parameter handles the conversion from YAML
        # scalar values to Python the dictionary format
        # ddoif_dict = yaml.load(fp, Loader=yaml.FullLoader)
        # ddoif_dict = yaml.safe_load(fp)

        yaml = ruamel.yaml.YAML(typ='safe') # this is claimed to be the safest way to load yaml  https://stackoverflow.com/questions/50846431/converting-a-yaml-file-to-python-json-object
        ddoif_dict = yaml.load(fp)
    return ddoif_dict

def yaml_to_xml(yaml_fname, xml_fname, ids=False):
    """
    A function used to read the ddoif dictionary in yaml format and save it into an xml file.
    This function makes use of dicttoxml package.

    ...

    Input arguments
    ----------
    - yaml_fname: the name of yaml file as input
    - xml_fname: the name of output xml file

    Output arguments:

    Methods
    -------
        yaml_to_xml('ddoif_dictionary.yaml', 'ddoif_dictionary.xml')

    """

    yaml_dict = read_yaml_as_dict(yaml_fname)
    xml_obj = dicttoxml.dicttoxml(yaml_dict, custom_root='ddoif', attr_type=False, ids=ids)
    print('Converting to xml using dicttoxml version -- ', dicttoxml.__version__)
    # print_xml(xml_obj)

    with open(xml_fname, "wb") as fp:
        fp.write(xml_obj)

def yaml_to_json(yaml_fname, json_fname):
    """
    A function used to read the ddoif dictionary in yaml format and save it into a json file.
    This functiona makes use of json package.

    ...

    Input arguments:
    ----------
    - yaml_fname: the name of yaml file as input
    - json_fname: the name of ouput json file

    Output arguments:

    Methods:
    -------
        yaml_to_xml('ddoif_dictionary.yaml', 'ddoif_dictionary.xml')

    """

    yaml_dict = read_yaml_as_dict(yaml_fname)
    with open(json_fname, 'w') as fp:
        json.dump(yaml_dict, fp, indent=True, )

def print_xml(xml_obj): # xml_obj = dicttoxml.dicttoxml(yaml_dict, custom_root='ddoif', attr_type=False)
    from xml.dom.minidom import parseString
    dom = parseString(xml_obj)
    print(dom.toprettyxml())


import datetime
import re
from random import shuffle
from collections import defaultdict

from django.utils.translation import (
    activate,
    get_language_info,
    get_language,
)
from django import http
from django.shortcuts import render
from django.core.cache import cache
from django.conf import settings
from django.utils.translation import get_language
from django.utils.translation import ugettext as _
from django.utils import timezone

try:
    from nltk.corpus import wordnet
except ImportError:
    wordnet = None

from kl.search.models import Word, Search

SEARCH_SUMMARY_SKIPS = (
    'crossword', 'korsord', 'fuck', 'peter', 'motherfucker',
)

def niceboolean(value):
    if isinstance(value, bool):
        return value
    falseness = ('', 'no', 'off', 'false', 'none', '0', 'f')
    return str(value).lower().strip() not in falseness

def uniqify(seq, idfun=None): # Alex Martelli ******* order preserving
    if idfun is None:
        def idfun(x): return x
    seen = set()
    result = []
    for item in seq:
        marker = idfun(item)
        # in old Python versions:
        # if seen.has_key(marker)
        # but in new ones:
        ##if marker in seen: continue
        if marker in seen:
            continue
        seen.add(marker)
        result.append(item)
    return result

class SearchResult(object):
    def __init__(self, word, definition='', by_clue=None):
        self.word = word
        self.definition = definition
        self.by_clue = by_clue

def home(request, json=False, record_search=True):
    # By default we are set to record the search in our stats
    # This can be overwritten by a CGI variable called 'r'
    # E.g. r=0 or r=no
    if request.GET.get('r'):
        record_search = niceboolean(request.GET.get('r'))

    language = request.GET.get('lang', get_language()).lower()

    slots = None

    if request.GET.get('l'):
        try:
            length = int(request.GET.get('l'))
        except ValueError:
            return http.HttpResponseRedirect('/?error=length')
        slots = request.GET.getlist('s')
        # if not type(slots) is list:
        if not isinstance(slots, list):
            return http.HttpResponseRedirect('/?error=slots')

        notletters = request.GET.get('notletters', '').upper()
        notletters = [x.strip() for x in notletters.split(',')
                      if len(x.strip()) == 1 and not x.strip().isdigit()]

        if not len(slots) >= length:
            return http.HttpResponseRedirect('/?error=slots&error=length')

        if not [x for x in slots if x.strip()]:
            # all blank
            return http.HttpResponseRedirect('/?error=slots')

        clues = request.GET.get('clues', '')
        if clues and ' ' in clues and ',' not in clues:
            clues = clues.replace(' ',', ')
        clues = [
            x.strip() for x in clues.split(',')
            if (
                x.strip() and
                x.strip().lower() not in STOPWORDS and
                not x.count(' ')
            )
        ]

        search_results = [] # the final simple list that is sent back

        for clue in clues:
            alternatives = _find_alternative_synonyms(
                clue,
                slots[:length],
                language,
                notletters=notletters,
                request=request
            )
            search_results.extend([
                SearchResult(x, by_clue=clue)
                for x in alternatives
            ])

        # find some alternatives
        search = ''.join([x and x.lower() or ' ' for x in slots[:length]])
        cache_key = '_find_alternatives_%s_%s' % (search, language)
        if notletters:
            cache_key += '__not' + ''.join(notletters)
        cache_key = cache_key.replace(' ','_')
        if re.findall('\s', cache_key):
            raise ValueError(
                'invalid cache_key search=%r, language=%r' % (search, language)
            )

        alternatives = cache.get(cache_key)
        if alternatives is None:
            alternatives = _find_alternatives(
                slots[:length],
                language,
                notletters=notletters
            )
            cache.set(cache_key, alternatives, 60 * 60 * 24)

        alternatives_count = len(alternatives)
        alternatives_truncated = False
        if alternatives_count > 100:
            alternatives = alternatives[:100]
            alternatives_truncated = True

        result = dict(
            length=length,
            search=search,
            word_count=alternatives_count,
            alternatives_truncated=alternatives_truncated,
        )
        already_found = [x.word for x in search_results]
        search_results.extend([
            SearchResult(each.word, definition=each.definition)
            for each in alternatives
            if each.word not in already_found
        ])

        match_points = None
        match_points = []
        if search_results:
            first_word = search_results[0].word
            for i, letter in enumerate(first_word):
                if letter.lower() == search[i]:
                    match_points.append(1)
                else:
                    match_points.append(0)
            result['match_points'] = match_points

        result['words'] = []
        for search_result in search_results:
            v = dict(word=search_result.word)
            if search_result.definition:
                v['definition'] = search_result.definition
            if search_result.by_clue:
                v['by_clue'] = search_result.by_clue
            result['words'].append(v)

        if alternatives_count == 1:
            result['match_text'] = _("1 match found")
        elif alternatives_count:
            if alternatives_truncated:
                result['match_text'] = _(
                    "%(count)s matches found but only showing first 100"
                ) % dict(count=alternatives_count)
            else:
                result['match_text'] = _("%(count)s matches found") % dict(
                    count=alternatives_count
                )
        else:
            result['match_text'] = _("No matches found unfortunately :(")

        found_word = None
        if len(search_results) == 1:
            try:
                found_word = Word.objects.get(
                    word=search_results[0].word,
                    language=language
                )
            except Word.DoesNotExist:
                # this it was probably not from the database but
                # from the wordnet stuff
                found_word = None

        if record_search:

            _record_search(
                search,
                user_agent=request.META.get('HTTP_USER_AGENT',''),
                ip_address=request.META.get('REMOTE_ADDR',''),
                found_word=found_word,
                language=language
            )

        request.session['has_searched'] = True

        if json:
            return http.JsonResponse(result)
            # return _render_json(result)

    else:
        length = ''

    show_example_search = not bool(request.session.get('has_searched'))
    most_recent_search_word = None
    if not show_example_search:
        most_recent_search_word = _get_recent_search_word(request)

    lang = get_language()
    accept_clues = (
        wordnet is not None and lang.lower() in ('en', 'en-gb', 'en-us')
    )

    context = {
        'length': length,
        'slots': slots,
        'accept_clues': accept_clues,
        'show_example_search': show_example_search,
        'most_recent_search_word': most_recent_search_word,
    }
    return render(request, 'search/home.html', context)

def _find_alternatives(slots, language, notletters=[]):
    length = len(slots)

    if length == 1:
        return Word.objects.filter(length=1, word=slots[0], language=language)

    filter_ = dict(length=length, language=language)
    slots = [x and x.lower() or ' ' for x in slots]
    search = ''.join(slots)
    start = ''
    end = ''
    try:
        start = re.findall('^\w+', search)[0]
        if len(start) > 1:
            filter_['first2'] = start[:2].lower()
            if len(start) > 2:
                filter_['word__istartswith'] = start
        else:
            filter_['first1'] = start.lower()
    except IndexError:
        pass

    try:
        end = re.findall('\w+$', search)[0]
        if len(end) > 1:
            filter_['last2'] = end[-2:].lower()
            if len(end) > 2:
                filter_['word__iendswith'] = end
        else:
            filter_['last1'] = end.lower()

    except IndexError:
        pass

    def filter_match(match):
        if end:
            matchable_string = search[len(start):-len(end)]
            found_string = match.word[len(start):-len(end)]
        else:
            matchable_string = search[len(start):]
            found_string = match.word[len(start):]

        assert len(matchable_string) == len(found_string), \
        "matchable_string=%r, found_string=%r" % (matchable_string, found_string)

        for i, each in enumerate(matchable_string):
            if each != ' ' and each != found_string[i]:
                # can't be match
                return False
        return True

    search_base = Word.objects
    limit = 10000
    # if the filter is really vague and the length is high we're going to get
    # too many objects and we need to cut our losses.
    if filter_['length'] > 5:
        if filter_.get('word__istartswith') and filter_.get('word__iendswith'):
            # It's long but has a startswith and an endswith, increase the limit
            limit = 5000
        elif filter_.get('word__istartswith') or filter_.get('word__iendswith'):
            # we're going to get less than above but still many
            limit = 2500
        else:
            limit = 1000

    # if there's neither a start or a end (e.g. '_E_E_A_') it will get all words
    # that are of that length then end truncate the result set then filter them
    # as a string operation. Then there's a chance it might not ever test word we
    # are looking for.
    if not start and not end:
        # must come up with some other crazy icontains filter
        # Look for the longest lump of letter. For example in '_E_ERA_' 'era' is
        # the longest lump
        #lumps = re.findall('\w+', search)
        lumps = search.split()

        longest = sorted(lumps, lambda x,y: cmp(len(y), len(x)))[0]
        if len(longest) > 1:
            filter_['word__icontains'] = longest
        else:
            for each in uniqify(lumps):
                search_base = search_base.filter(word__icontains=each)

            limit = search_base.filter(**filter_).order_by('word').count()
    elif (start and len(start) <= 2) or (end and len(end) <= 2):
        # If you search for somethin like "___TAM__T"
        # We so far only know it's 9 characters long (french as 21k 9 characters long
        # words).
        # We also have one tiny little 't' at the end but there's still
        # 4086 options
        for lump in re.findall(r'\s(\w+)\s', search):
            filter_['word__icontains'] = lump

    search_qs = search_base.filter(**filter_)
    for notletter in notletters:
        search_qs = search_qs.exclude(word__icontains=notletter)

    all_matches = [x for x
                   in search_qs.order_by('word')[:limit]
                   if filter_match(x)]
    return uniqify(all_matches, lambda x: x.word.lower())

def _find_alternative_synonyms(
    word,
    slots,
    language,
    notletters=None,
    request=None
):
    length = len(slots)
    if notletters is None:
        notletters = []

    slots = [x and x.lower() or ' ' for x in slots]
    search = ''.join(slots)
    start = ''
    end = ''
    try:
        start = re.findall('^\w+', search)[0]
    except IndexError:
        pass

    try:
        end = re.findall('\w+$', search)[0]
    except IndexError:
        pass

    def filter_match(word):

        if end and not word.endswith(end):
            # Don't even bother
            return False
        elif start and not word.startswith(start):
            # Don't even bother
            return False

        if end:
            matchable_string = search[len(start):-len(end)]
            found_string = word[len(start):-len(end)]
        else:
            matchable_string = search[len(start):]
            found_string = word[len(start):]
        assert len(matchable_string) == len(found_string)
        for i, each in enumerate(matchable_string):
            if each != ' ' and each != found_string[i]:
                # can't be match
                return False
        return True

    def test(word):
        if len(word) == length:
            if not notletters:
                for letter in word:
                    if letter.upper() in notletters:
                        return False
            return filter_match(word)

    for variation in _get_variations(word, greedy=True, request=request):
        if test(variation):
            yield variation

def _get_variations(word, greedy=False,
                    store_definitions=True,
                    request=None):
    a = _get_variations_wordnet(
        word,
        greedy=greedy,
        store_definitions=store_definitions
    )
    return a
    # b = _get_variations_synonym_dot_com(
    #     word,
    #     greedy=greedy,
    #     store_definitions=store_definitions,
    #     request=request
    # )
    # return a + b

def _record_search(
    search_word,
    user_agent='',
    ip_address='',
    found_word=None,
    language=None,
    search_type='',
):
    if len(user_agent) > 200:
        user_agent = user_agent[:200]
    if len(ip_address) > 15:
        import warnings
        warnings.warn("ip_address too long (%r)" % ip_address)
        ip_address = ''
    elif ip_address == '127.0.0.1' and settings.DEBUG:
        # because 127.0.0.1 can't be looked up, use a random other one
        examples = '125.239.15.42,114.199.97.224,68.190.165.25,208.75.100.212,'\
                   '61.29.84.154,72.49.16.234,66.57.228.64,196.25.255.250,'\
                   '141.117.6.97,85.68.18.183,90.157.186.202'.split(',')
        shuffle(examples)
        ip_address = examples[0]

    Search.objects.create(
        search_word=search_word,
        user_agent=user_agent.strip(),
        ip_address=ip_address.strip(),
        found_word=found_word,
        language=language,
        search_type=search_type,
    )

def _get_recent_search_word(request):
    # _today = datetime.datetime.today()
    _today = timezone.now()
    _since = datetime.datetime(_today.year, _today.month, 1)

    _extra_exclude = dict(found_word__word__in=list(SEARCH_SUMMARY_SKIPS))
    if request.META.get('HTTP_USER_AGENT'):
        _extra_exclude['user_agent'] = request.META.get('HTTP_USER_AGENT')
    if request.META.get('REMOTE_ADDR'):
        _extra_exclude['ip_address'] = request.META.get('REMOTE_ADDR')

    _extra_filter = dict()
    # Special hack! Since the search summary has a cache of 1 hour,
    # don't include things that are too recent
    _extra_filter['add_date__lt'] = _today - datetime.timedelta(hours=1)

    return _find_recent_search_word(
        get_language(),
        since=_since,
        random=True,
        extra_exclude=_extra_exclude,
        **_extra_filter,
    )

def _find_recent_search_word(
    language,
    since=None,
    random=False,
    extra_exclude={},
    **extra_filter
):
    searches = Search.objects.filter(
        language=language,
        found_word__isnull=False,
        **extra_filter
    ).select_related('found_word')

    if since:
        searches = searches.filter(add_date__gte=since)
    searches = searches.exclude(**extra_exclude)

    if random:
        # For some bizzare reason it seems that even if the exclude above
        # has found_word__word__in=SEARCH_SUMMARY_SKIPS it still returns
        # words from that list!!!!
        # Hence this list comprehension.
        found_words = [x.found_word for x in searches
                       if x.found_word.word not in SEARCH_SUMMARY_SKIPS]
        shuffle(found_words)
        try:
            return found_words[0]
        except IndexError:
            return None
    else:
        searches = searches.order_by('-add_date')
        return searches[0].found_word
    return None

def get_search_stats(language):
    # Total no words in our database
    cache_key = 'no_total_words_%s' % language
    no_total_words = cache.get(cache_key)
    if no_total_words is None:
        no_total_words = Word.objects.filter(language=language).count()
        cache.set(cache_key, no_total_words, 60 * 60 * 24 * 30)

    today = timezone.now()

    # Searches today
    # today_midnight = datetime.datetime(
    #     today.year,
    #     today.month,
    #     today.day, 0, 0, 0)
    today_midnight = today - datetime.timedelta(days=1)
    cache_key = 'no_searches_today_%s' % language
    no_searches_today = cache.get(cache_key)
    if no_searches_today is None:
        no_searches_today = Search.objects.filter(
            language=language,
            add_date__gte=today_midnight
        ).count()
        cache.set(cache_key, no_searches_today, 60 * 60)

    # Searches yesterday
    cache_key = 'no_searches_yesterday_%s' % language
    no_searches_yesterday = cache.get(cache_key)
    if no_searches_yesterday is None:
        yesterday_midnight = today_midnight - datetime.timedelta(days=1)
        no_searches_yesterday = Search.objects.filter(language=language,
            add_date__range=(yesterday_midnight, today_midnight)
        ).count()
        cache.set(cache_key, no_searches_yesterday, 60 * 60 * 24)

    # Searches this week
    cache_key = 'no_searches_this_week_%s' % language
    no_searches_this_week = cache.get(cache_key)
    if no_searches_this_week is None:
        # find the first monday
        monday_midnight = today_midnight
        while monday_midnight.strftime('%A') != 'Monday':
            monday_midnight = monday_midnight - datetime.timedelta(days=1)

        no_searches_this_week = Search.objects.filter(
            language=language,
            add_date__gt=monday_midnight
        ).count()
        cache.set(cache_key, no_searches_this_week, 60 * 60 * 24)

    # Searches this month
    cache_key = 'no_searches_this_month_%s' % language
    no_searches_this_month = cache.get(cache_key)
    if no_searches_this_month is None:
        first_day_month = today.replace(day=1)
        no_searches_this_month = Search.objects.filter(
            language=language,
            add_date__gte=first_day_month
        ).count()
        cache.set(cache_key, no_searches_this_month, 60 * 60)

    # Searches this year
    cache_key = 'no_searches_this_year_%s' % language
    no_searches_this_year = cache.get(cache_key)
    if no_searches_this_year is None:
        # first_day_year = datetime.datetime(today.year, 1, 1, 0, 0, 0)
        first_day_year = today.replace(month=1, day=1)
        no_searches_this_year = Search.objects.filter(
            language=language,
            add_date__gte=first_day_year
        ).count()
        cache.set(cache_key, no_searches_this_year, 60 * 60)

    return {
        'no_total_words': no_total_words,
        'no_searches_today': no_searches_today,
        'no_searches_yesterday': no_searches_yesterday,
        'no_searches_this_week': no_searches_this_week,
        'no_searches_this_month': no_searches_this_month,
        'no_searches_this_year': no_searches_this_year,
    }

MONTH_NAMES = []
for i in range(1, 13):
    d = datetime.date(2009, i, 1)
    MONTH_NAMES.append(d.strftime('%B'))

def searches_summary(request, year, month, atleast_count=2,
                     lookup_definitions=False):

    first_search_date = Search.objects.all().order_by('add_date')[0].add_date
    last_search_date = Search.objects.all().order_by('-add_date')[0].add_date

    year = int(year)
    try:
        month_nr = [x.lower() for x in MONTH_NAMES].index(month.lower()) + 1
    except ValueError:
        raise http.Http404("Unrecognized month name")
    # turn that into a date
    since = datetime.date(year, month_nr, 1)

    if (month_nr + 1) > 12:
        since_month_later = datetime.date(year+1, 1, 1)
    else:
        since_month_later = datetime.date(year, month_nr+1, 1)

    today = timezone.now()
    since_month_later_datetime = today.replace(
        year=since_month_later.year,
        month=since_month_later.month,
        day=since_month_later.day
    )

    next_month_link = None

    if since_month_later_datetime < first_search_date:
        raise http.Http404("Too far back in time")
    if since_month_later_datetime < last_search_date:
        next_month_link = since_month_later.strftime("/searches/%Y/%B/")

    since_datetime = today.replace(
        year=since.year,
        month=since.month,
        day=since.day
    )

    previous_month_link = None

    if since_datetime > last_search_date:
        raise http.Http404("Too far into the future")

    elif since_datetime > first_search_date:
        if (month_nr - 1) < 1:
            since_month_earlier = datetime.date(year-1, 12, 1)
        else:
            since_month_earlier = datetime.date(year, month_nr-1, 1)

        previous_month_link = since_month_earlier.strftime("/searches/%Y/%B/")

    base_searches = Search.objects.filter(
        add_date__gte=since,
        add_date__lt=since_month_later
    )

    found_searches = base_searches.exclude(
        found_word=None
    ).select_related(
        'found_word'
    ).exclude(
        found_word__word__in=list(SEARCH_SUMMARY_SKIPS)
    )

    found_words = defaultdict(list)
    definitions = {}
    for each in found_searches:
        found_words[each.language].append(each.found_word.word)

        if each.language not in definitions:
            definitions[each.found_word.language] = {}
        if each.found_word.definition:
            definitions[each.found_word.language][each.found_word.word.lower()]\
              = each.found_word.definition.splitlines()

    found_words = dict(found_words)

    found_words_repeats = {}
    for language, words in found_words.items():
        counts = defaultdict(int)
        for word in words:
            if len(word) < 2:
                # don't want to find single character words
                # It's a bug that they're even in there
                continue
            counts[word.lower()] += 1
            found_words_repeats[language] = sorted(
                [k for (k, v) in counts.items()
                if v >= atleast_count],
                key=lambda x: x[1]
            )

    if lookup_definitions:
        for lang, words in found_words_repeats.items():
            for word in words:
                try:
                    definitions[lang][word]
                except KeyError:
                    if lang in ('en-us','en-gb'):
                        # wordnet
                        definition = _get_word_definition(word, language=lang)
                    else:
                        definition = None

                    if not definition:
                        definition = _get_word_definition_scrape(word, language=lang)
                    if definition:
                        add_word_definition(word, definition, language=lang)

    # bake the definitions into found_words_repeats
    for lang, words in found_words_repeats.items():
        for i, word in enumerate(words):
            words_dict = dict(word=word)
            if lang in definitions:
                if word in definitions[lang]:
                    words_dict = dict(words_dict, definitions=definitions[lang][word])
            found_words_repeats[lang][i] = words_dict

    all_words_plain = set()
    for records in found_words_repeats.values():
        for record in records:
            all_words_plain.add(record['word'].lower())
    all_words_plain = list(all_words_plain)

    context = {
        # 'language': language,
        'month': month,
        'year': year,
        'all_words_plain': all_words_plain,
        'found_words_repeats': found_words_repeats,
        'previous_month_link': previous_month_link,
        'next_month_link': next_month_link,
    }
    return render(request, 'search/searches_summary.html', context)

def about_crosstips(request):
    return render(request, 'search/about-crosstips.html')


"""
This file (test_youbit.py) contains unit tests for the encode.py and decode.py files.
"""
from pathlib import Path
import os
import time

from yt_dlp.utils import DownloadError

from tests.conftest import uploads
from youbit import Encoder, download_and_decode
from youbit.settings import Settings, Browser
from youbit.download import Downloader
from youbit.util import get_md5

@uploads
def test_youbit_round_trip(browser: Browser, tempdir: Path):
    test_file = Path(os.path.dirname(__file__)) / "testdata" / "files" / "test_file.jpg"
    encoder = Encoder(test_file, Settings(browser=browser))
    url = encoder.encode_and_upload()
    time.sleep(
        10
    )  # YouTube needs time to process the video before we can download the correct resolution
    timeout = 0
    while timeout < 60:
        try:
            downloader = Downloader(url)
            if downloader.best_vbr > 6000:
                break
        except DownloadError:
            time.sleep(5)
            timeout += 5
            continue
    if timeout >= 60:
        assert False, "Timeout"

    output_path = download_and_decode(url, tempdir)
    original_md5 = get_md5(test_file)
    output_md5 = get_md5(output_path)
    assert original_md5 == output_md5