import pandas as pd
import numpy as np
import pickle
import serial
import time

#归一化处理
def normalize(data):
    mu=np.mean(data,axis=0)
    sigma=np.std(data,axis=0)
    return (data-mu)/sigma

Classification_libary=['A','B','C','D','E','F','G','H','I',
                       'J','K','L','M','N','O','P','Q','R',
                       'S','T','U','V','W','X','Y','Z'] #26分类字母库
# 设置端口变量和值
serialPosrt = "COM5"
# 设置波特率变量和值
baudRate = 115200
#设置超时时间,单位为s
timeout = 0.5
# 接受串口数据
ser = serial.Serial(serialPosrt, baudRate)
# 循环获取数据(条件始终为真)
size_col=60
test_example = np.zeros(6*size_col).reshape(size_col, 6)
print("Loading SVM model...")
with open('svm.pickle', 'rb') as fr:
    new_svm = pickle.load(fr)
print("Start to record!")

while 1:
    str = ser.readline()
    sample_time = 0
    if len(str) > 5:
        while  sample_time < size_col:
            str = np.array(
                eval(
                    str.strip()))  # strip() is used to remove irrelevant syntax '\r\n' and eval() is used to remove the ''
            np.set_printoptions(formatter={'float': '{: 0.3f}'.format})  # set data precise for 0.001
            test_example[sample_time, :] = str
            sample_time = sample_time + 1
            # 把数据打印到终端显示
            str = ser.readline()
        test_example = normalize(test_example)  # 实时采集到大小为6x60的一组样本数据，并归一化
        #print(test_example)
        print('Data collected. Start classification...')
        print("predicted value:")
        pred_result = new_svm.predict(test_example.reshape(1, 6 * size_col))  # 预测结果
        # print(pred_result)
        print(Classification_libary[int(pred_result[0]) - 1])  # 显示结果
        print('Recognized!')
        time.sleep(0.5)


from typing import List

class Solution:
    def groupAnagrams(self, strs: List[str]) -> List[List[str]]:
        # Strategy 1 - use sorted string as a key, its value is the
        # original string
        hashmap = {}
        for string in strs:
            sorted_string_tuple = tuple(sorted(string))
            if sorted_string_tuple in hashmap.keys():
                hashmap[sorted_string_tuple].append(string)
            else:
                hashmap[sorted_string_tuple] = [string]
        return hashmap.values()
        """
        # Strategy 2 - use tuple of character counts as a key, its value is
        # the original string
        hashmap = {}
        for string in strs:
            key = [0] * 26 # count how many times the character shows up
            for character in string:
                # The difference is the position of index in the list
                # eg. ord('d') - ord('a') is 3
                key[ord(character)-ord('a')] += 1
            if tuple(key) in hashmap.keys():
                hashmap[tuple(key)].append(string)
            else:
                hashmap[tuple(key)] = [string]
        return hashmap.values()
        """

#strs = ["a"]
#strs = [""]
strs = ["eat","tea","tan","ate","nat","bat"]
result = Solution().groupAnagrams(strs)
print("Group anagrams are", result)


from flask import (
    Flask,
    render_template,
    session,
    redirect,
    request,
    url_for
)
import urllib
import requests
import json
import datetime

from datetime import datetime

app = Flask(__name__)

SEATGEEK_SEARCH_URL = 'http://api.seatgeek.com/2/events'

@app.route('/')
def index():
    # print request.args['data2']
    return render_template('index.html')

@app.route('/results', methods=['POST'])
def results():
    location = request.form['location']
    time_menu = request.form['time_menu']
    budget_menu = request.form['budget_menu']

    data = {
        'venue.city': location,
        'datetime_local.gte': '2013-04-14',
        'lowest_price.gte': budget_menu
    }
    query_string = urllib.urlencode(data)
    api_url = SEATGEEK_SEARCH_URL + "?" + query_string

    response = requests.get(api_url)
    json_response = json.loads(response.text)

    start = "http://maps.googleapis.com/maps/api/staticmap";
    i = 0
    urls = {}
    for event in json_response['events']:
        lat = str(event['venue']['location']['lat'])
        lon = str(event['venue']['location']['lon'])
        coords = lat + "," + lon
        data = {
            'center': coords,
            'markers': coords,
            'zoom': '14',
            'size': '200x200',
            'sensor': 'false'
        }
        qstring = urllib.urlencode(data)
        url = start + '?' + qstring
        event['newurl']=url
        i=i+1

    return render_template('results.html',
                            location = location,
                            low_price = budget_menu,
                            events=json_response['events'])

@app.route('/save', methods=["POST"])
def save():
    return str(request.form)

if __name__ == '__main__':
    app.run(debug=True)


# -*- coding:utf-8 -*-
__author__ = 'yangjian'
"""

"""
from sklearn.linear_model import LogisticRegression, LinearRegression

from .base_ensemble import BaseEnsemble

class StackingEnsemble(BaseEnsemble):
    def __init__(self, task, estimators, need_fit=False, n_folds=5, method='soft', meta_model=None, fit_kwargs=None):
        super(StackingEnsemble, self).__init__(task, estimators, need_fit, n_folds, method)
        if meta_model is None:
            if task == 'regression':
                self.meta_model = LinearRegression()
            else:
                self.meta_model = LogisticRegression()
        else:
            self.meta_model = meta_model
        self.fit_kwargs = fit_kwargs if fit_kwargs is not None else {}

    def fit_predictions(self, predictions, y_true):
        X = self.__predictions2X(predictions)
        self.meta_model.fit(X, y_true, **self.fit_kwargs)

    def __predictions2X(self, predictions):
        X = predictions
        if len(X.shape) == 3:
            if self.task == 'binary':
                X = X[:, :, -1]
            elif self.task == 'multiclass':
                np = self.np
                X = np.argmax(X, axis=2)
            else:
                raise ValueError(
                    f"The shape of `predictions` and the `task` don't match. shape:{predictions.shape}, task:{self.task}")
        return X

    def predictions2predict(self, predictions):
        assert self.meta_model is not None
        X = self.__predictions2X(predictions)
        pred = self.meta_model.predict(X)
        if self.task == 'binary':
            np = self.np
            pred = np.clip(pred, 0, 1)
        return pred

    def predictions2predict_proba(self, predictions):
        assert self.meta_model is not None
        X = self.__predictions2X(predictions)
        if hasattr(self.meta_model, 'predict_proba'):
            pred = self.meta_model.predict_proba(X)
        else:
            pred = self.meta_model.predict(X)

        if self.task == 'binary':
            np = self.np
            pred = np.clip(pred, 0, 1)
        return pred


from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
import util
import numpy as np

"""
use logistic regression to classify data,
the feature to be tested with is the count of descirption
"""
# change to your data file path
train_data_file_path = 'data/train.csv'
test_data_file_path = 'data/test.csv'
# logistic, randomforest or svm
classifier_type = 'randomforest'
gen_final_result = True

train_lable, raw_train_data = util.group_by_visit_number(train_data_file_path)
test_label, raw_test_data = util.group_by_visit_number(test_data_file_path, False)

# feature to be tested with
feature_set = [['d']]
feature_result = []

for feature in feature_set:
    # filter the data with the required feature
    pro_train_data = util.process_data(raw_train_data, feature)
    pro_test_data = util.process_data(raw_test_data, feature)

    bag_of_features = util.get_feature_bag(pro_train_data, pro_test_data, {})
    train_data = util.one_hot_encoding(
        pro_train_data,
        bag_of_features,
        verbose=False,
        numerical=False)
    train_lable = np.array(train_lable)

    # 5-fold cross validation
    num_train_data = len(train_data)
    num_fold = 5
    step_size = num_train_data / num_fold
    result = []
    for i in range(0, num_fold):
        start_index = i * step_size
        end_index = (i + 1) * step_size

        train_mask = np.ones(num_train_data, dtype=bool)
        test_mask = np.zeros(num_train_data, dtype=bool)
        train_mask[start_index:end_index] = np.zeros(step_size, dtype=bool)
        test_mask[start_index:end_index] = np.ones(step_size, dtype=bool)

        classifier = None
        if classifier_type == 'randomforest':
            classifier = RandomForestClassifier()
        elif classifier_type == 'logistic':
            classifier = LogisticRegression()
        elif classifier_type == 'svm':
            classifier = SVC()
        else:
            raise ValueError("unsupport classifier type")
        classifier.fit(train_data[train_mask], train_lable[train_mask])

        predicts = classifier.predict(train_data[test_mask])

        correct = 0
        error = 0
        for tup in zip(list(predicts), list(train_lable[test_mask])):
            if tup[0] == tup[1]:
                correct = correct + 1
            else:
                error = error + 1
        result.append(float(correct) / float(error + correct))

    cross_avg_accu = sum(result) / float(num_fold)
    print(
        "Cross validation result for feature " +
        str(feature) + " is " + str(float(cross_avg_accu)))
    feature_result.append(cross_avg_accu)

print(str(zip(feature_set, feature_result)))

"""
use full train data with the best method,
apply on test data to obtain the final result
"""
if gen_final_result is True:
    pro_train_data = util.process_data(raw_train_data, feature)
    pro_test_data = util.process_data(raw_test_data, feature)

    bag_of_features = util.get_feature_bag(pro_train_data, pro_test_data, {})
    train_data = util.one_hot_encoding(
        pro_train_data,
        bag_of_features,
        verbose=False,
        numerical=False)
    train_lable = np.array(train_lable)

    test_data = util.one_hot_encoding(
        data=pro_test_data,
        bag=bag_of_features,
        verbose=False,
        numerical=False)

    classifier.fit(train_data, train_lable)
    predicts = classifier.predict(test_data)
    util.dump_result_to_file(
        filename='reuslt.csv',
        test_data=raw_test_data,
        label=test_label,
        predict=predicts)


import asyncio
import concurrent.futures
import json
import logging
import os
import traceback
from contextlib import AsyncExitStack
from typing import Any, Callable, Dict, List, NoReturn, Optional, Set, Tuple, TypedDict

import aiohttp_session  # type: ignore
import kubernetes_asyncio
import kubernetes_asyncio.client
import kubernetes_asyncio.config
import uvloop  # type: ignore
import yaml
from aiohttp import web
from gidgethub import aiohttp as gh_aiohttp
from gidgethub import routing as gh_routing
from gidgethub import sansio as gh_sansio
from prometheus_async.aio.web import server_stats  # type: ignore

from gear import (
    AuthServiceAuthenticator,
    Database,
    UserData,
    check_csrf_token,
    json_request,
    json_response,
    monitor_endpoints_middleware,
    setup_aiohttp_session,
)
from gear.profiling import install_profiler_if_requested
from hailtop import aiotools, httpx
from hailtop.auth import hail_credentials
from hailtop.batch_client.aioclient import Batch, BatchClient
from hailtop.config import get_deploy_config
from hailtop.hail_logging import AccessLogger
from hailtop.tls import internal_server_ssl_context
from hailtop.utils import collect_aiter, humanize_timedelta_msecs, periodically_call, retry_transient_errors
from web_common import render_template, set_message, setup_aiohttp_jinja2, setup_common_static_routes

from .constants import AUTHORIZED_USERS, TEAMS
from .environment import CLOUD, DEFAULT_NAMESPACE, STORAGE_URI
from .envoy import create_cds_response, create_rds_response
from .github import PR, WIP, FQBranch, MergeFailureBatch, Repo, UnwatchedBranch, WatchedBranch, select_random_teammate
from .utils import gcp_logging_queries

with open(os.environ.get('HAIL_CI_OAUTH_TOKEN', 'oauth-token/oauth-token'), 'r', encoding='utf-8') as f:
    oauth_token = f.read().strip()

log = logging.getLogger('ci')

uvloop.install()

deploy_config = get_deploy_config()

watched_branches: List[WatchedBranch] = []

routes = web.RouteTableDef()

auth = AuthServiceAuthenticator()

class PRConfig(TypedDict):
    number: int
    title: str
    batch_id: Optional[int]
    build_state: Optional[str]
    gh_statuses: Dict[str, str]
    source_branch_name: str
    review_state: Optional[str]
    author: str
    assignees: Set[str]
    reviewers: Set[str]
    labels: Set[str]
    out_of_date: bool

async def pr_config(app, pr: PR) -> PRConfig:
    batch_id = pr.batch.id if pr.batch and isinstance(pr.batch, Batch) else None
    build_state = pr.build_state if await pr.authorized(app['db']) else 'unauthorized'
    if build_state is None and batch_id is not None:
        build_state = 'building'
    return {
        'number': pr.number,
        'title': pr.title,
        # FIXME generate links to the merge log
        'batch_id': batch_id,
        'build_state': build_state,
        'gh_statuses': {k: v.value for k, v in pr.last_known_github_status.items()},
        'source_branch_name': pr.source_branch.name,
        'review_state': pr.review_state,
        'author': pr.author,
        'assignees': pr.assignees,
        'reviewers': pr.reviewers,
        'labels': pr.labels,
        'out_of_date': pr.build_state in ['failure', 'success', None] and not pr.is_up_to_date(),
    }

class WatchedBranchConfig(TypedDict):
    index: int
    branch: str
    sha: Optional[str]
    deploy_batch_id: Optional[int]
    deploy_state: Optional[str]
    repo: str
    prs: List[PRConfig]
    gh_status_names: Set[str]
    merge_candidate: Optional[str]

async def watched_branch_config(app, wb: WatchedBranch, index: int) -> WatchedBranchConfig:
    if wb.prs:
        pr_configs = [await pr_config(app, pr) for pr in wb.prs.values()]
    else:
        pr_configs = []
    # FIXME recent deploy history
    gh_status_names = {k for pr in pr_configs for k in pr['gh_statuses'].keys()}
    return {
        'index': index,
        'branch': wb.branch.short_str(),
        'sha': wb.sha,
        # FIXME generate links to the merge log
        'deploy_batch_id': wb.deploy_batch.id if wb.deploy_batch and isinstance(wb.deploy_batch, Batch) else None,
        'deploy_state': wb.deploy_state,
        'repo': wb.branch.repo.short_str(),
        'prs': pr_configs,
        'gh_status_names': gh_status_names,
        'merge_candidate': wb.merge_candidate.short_str() if wb.merge_candidate else None,
    }

@routes.get('')
@routes.get('/')
@auth.authenticated_developers_only()
async def index(request: web.Request, userdata: UserData) -> web.Response:
    wb_configs = [await watched_branch_config(request.app, wb, i) for i, wb in enumerate(watched_branches)]
    page_context = {'watched_branches': wb_configs, 'frozen_merge_deploy': request.app['frozen_merge_deploy']}
    return await render_template('ci', request, userdata, 'index.html', page_context)

def wb_and_pr_from_request(request: web.Request) -> Tuple[WatchedBranch, PR]:
    watched_branch_index = int(request.match_info['watched_branch_index'])
    pr_number = int(request.match_info['pr_number'])

    if watched_branch_index < 0 or watched_branch_index >= len(watched_branches):
        raise web.HTTPNotFound()
    wb = watched_branches[watched_branch_index]

    if not wb.prs or pr_number not in wb.prs:
        raise web.HTTPNotFound()
    return wb, wb.prs[pr_number]

def filter_jobs(jobs):
    filtered: Dict[str, list] = {
        state: []
        # the order of this list is the order in which the states will be displayed on the page
        for state in ["failed", "error", "cancelled", "running", "pending", "ready", "creating", "success"]
    }
    for job in jobs:
        filtered[job["state"].lower()].append(job)
    return {"jobs": filtered}

@routes.get('/watched_branches/{watched_branch_index}/pr/{pr_number}')
@auth.authenticated_developers_only()
async def get_pr(request: web.Request, userdata: UserData) -> web.Response:
    wb, pr = wb_and_pr_from_request(request)

    page_context: Dict[str, Any] = {}
    page_context['repo'] = wb.branch.repo.short_str()
    page_context['wb'] = wb
    page_context['pr'] = pr
    # FIXME
    batch = pr.batch
    if batch:
        if isinstance(batch, Batch):
            status = await batch.last_known_status()
            jobs = await collect_aiter(batch.jobs())
            for j in jobs:
                j['duration'] = humanize_timedelta_msecs(j['duration'])
            page_context['batch'] = status
            page_context.update(filter_jobs(jobs))
            artifacts_uri = f'{STORAGE_URI}/build/{batch.attributes["token"]}'
            page_context['artifacts_uri'] = artifacts_uri
            page_context['artifacts_url'] = storage_uri_to_url(artifacts_uri)

            if CLOUD == 'gcp':
                start_time = status['time_created']
                end_time = status['time_completed']
                assert start_time is not None
                page_context['logging_queries'] = gcp_logging_queries(
                    batch.attributes['namespace'], start_time, end_time
                )
            else:
                page_context['logging_queries'] = None
        else:
            page_context['exception'] = '\n'.join(
                traceback.format_exception(None, batch.exception, batch.exception.__traceback__)
            )

    batch_client = request.app['batch_client']
    target_branch = wb.branch.short_str()
    batches = batch_client.list_batches(f'test=1 ' f'pr={pr.number} ' f'target_branch={target_branch} ' f'user:ci')
    batches = sorted([b async for b in batches], key=lambda b: b.id, reverse=True)
    page_context['history'] = [await b.last_known_status() for b in batches]

    return await render_template('ci', request, userdata, 'pr.html', page_context)

def storage_uri_to_url(uri: str) -> str:
    if uri.startswith('gs://'):
        path = uri.removeprefix('gs://')
        return f'https://console.cloud.google.com/storage/browser/{path}'
    return uri

async def retry_pr(wb, pr, request):
    app = request.app
    session = await aiohttp_session.get_session(request)

    if pr.batch is None:
        log.info('retry cannot be requested for PR #{pr.number} because it has no batch')
        set_message(session, f'Retry cannot be requested for PR #{pr.number} because it has no batch.', 'error')
        return

    batch_id = pr.batch.id
    db: Database = app['db']
    await db.execute_insertone('INSERT INTO invalidated_batches (batch_id) VALUES (%s);', batch_id)
    await wb.notify_batch_changed(app)

    log.info(f'retry requested for PR: {pr.number}')
    set_message(session, f'Retry requested for PR #{pr.number}.', 'info')

@routes.post('/watched_branches/{watched_branch_index}/pr/{pr_number}/retry')
@auth.authenticated_developers_only(redirect=False)
async def post_retry_pr(request: web.Request, _) -> NoReturn:
    wb, pr = wb_and_pr_from_request(request)

    await asyncio.shield(retry_pr(wb, pr, request))
    raise web.HTTPFound(deploy_config.external_url('ci', f'/watched_branches/{wb.index}/pr/{pr.number}'))

@routes.get('/batches')
@auth.authenticated_developers_only()
async def get_batches(request, userdata):
    batch_client = request.app['batch_client']
    batches = [b async for b in batch_client.list_batches()]
    statuses = [await b.last_known_status() for b in batches]
    page_context = {'batches': statuses}
    return await render_template('ci', request, userdata, 'batches.html', page_context)

@routes.get('/batches/{batch_id}')
@auth.authenticated_developers_only()
async def get_batch(request, userdata):
    batch_id = int(request.match_info['batch_id'])
    batch_client = request.app['batch_client']
    b = await batch_client.get_batch(batch_id)
    status = await b.last_known_status()
    jobs = await collect_aiter(b.jobs())
    for j in jobs:
        j['duration'] = humanize_timedelta_msecs(j['duration'])
    wb = get_maybe_wb_for_batch(b)
    page_context = {'batch': status, 'wb': wb}
    page_context.update(filter_jobs(jobs))
    return await render_template('ci', request, userdata, 'batch.html', page_context)

def get_maybe_wb_for_batch(b: Batch):
    if 'target_branch' in b.attributes and 'pr' in b.attributes:
        branch = b.attributes['target_branch']
        wbs = [wb for wb in watched_branches if wb.branch.short_str() == branch]
        if len(wbs) == 0:
            pr = b.attributes['pr']
            log.exception(f"Attempted to load PR {pr} for unwatched branch {branch}")
        else:
            assert len(wbs) == 1
            return wbs[0].index
    return None

def filter_wbs(wbs: List[WatchedBranchConfig], pred: Callable[[PRConfig], bool]):
    return [{**wb, 'prs': [pr for pr in wb['prs'] if pred(pr)]} for wb in wbs]

def is_pr_author(gh_username: str, pr_config: PRConfig) -> bool:
    return gh_username == pr_config['author']

def is_pr_reviewer(gh_username: str, pr_config: PRConfig) -> bool:
    return gh_username in pr_config['assignees'] or gh_username in pr_config['reviewers']

def pr_requires_action(gh_username: str, pr_config: PRConfig) -> bool:
    build_state = pr_config['build_state']
    review_state = pr_config['review_state']
    return (
        is_pr_author(gh_username, pr_config)
        and (build_state == 'failure' or review_state == 'changes_requested' or WIP in pr_config['labels'])
    ) or (is_pr_reviewer(gh_username, pr_config) and review_state == 'pending')

@routes.get('/me')
@auth.authenticated_developers_only()
async def get_user(request: web.Request, userdata: UserData) -> web.Response:
    for authorized_user in AUTHORIZED_USERS:
        if authorized_user.hail_username == userdata['username']:
            user = authorized_user
            break
    else:
        raise web.HTTPForbidden()

    wbs = [await watched_branch_config(request.app, wb, i) for i, wb in enumerate(watched_branches)]
    pr_wbs = filter_wbs(wbs, lambda pr: is_pr_author(user.gh_username, pr))
    review_wbs = filter_wbs(wbs, lambda pr: is_pr_reviewer(user.gh_username, pr))
    actionable_wbs = filter_wbs(wbs, lambda pr: pr_requires_action(user.gh_username, pr))

    batch_client = request.app['batch_client']
    dev_deploys = batch_client.list_batches(f'user={user.hail_username} dev_deploy=1', limit=10)
    dev_deploys = sorted([b async for b in dev_deploys], key=lambda b: b.id, reverse=True)

    team_random_member = {team: select_random_teammate(team).gh_username for team in TEAMS}

    page_context = {
        'username': user.hail_username,
        'gh_username': user.gh_username,
        'pr_wbs': pr_wbs,
        'review_wbs': review_wbs,
        'actionable_wbs': actionable_wbs,
        'team_member': team_random_member,
        'dev_deploys': [await b.last_known_status() for b in dev_deploys],
    }
    return await render_template('ci', request, userdata, 'user.html', page_context)

@routes.post('/authorize_source_sha')
@auth.authenticated_developers_only(redirect=False)
async def post_authorized_source_sha(request: web.Request, _) -> NoReturn:
    app = request.app
    db: Database = app['db']
    post = await request.post()
    sha = str(post['sha']).strip()
    await db.execute_insertone('INSERT INTO authorized_shas (sha) VALUES (%s);', sha)
    log.info(f'authorized sha: {sha}')
    session = await aiohttp_session.get_session(request)
    set_message(session, f'SHA {sha} authorized.', 'info')
    raise web.HTTPFound(deploy_config.external_url('ci', '/'))

@routes.get('/healthcheck')
async def healthcheck(_) -> web.Response:
    return web.Response(status=200)

gh_router = gh_routing.Router()

@gh_router.register('pull_request')
async def pull_request_callback(event):
    gh_pr = event.data['pull_request']
    number = gh_pr['number']
    target_branch = FQBranch.from_gh_json(gh_pr['base'])
    for wb in watched_branches:
        if (wb.prs and number in wb.prs) or (wb.branch == target_branch):
            await wb.notify_github_changed(event.app)

@gh_router.register('push')
async def push_callback(event):
    data = event.data
    ref = data['ref']
    if ref.startswith('refs/heads/'):
        branch_name = ref.removeprefix('refs/heads/')
        branch = FQBranch(Repo.from_gh_json(data['repository']), branch_name)
        for wb in watched_branches:
            if wb.branch == branch:
                await wb.notify_github_changed(event.app)

@gh_router.register('pull_request_review')
async def pull_request_review_callback(event):
    gh_pr = event.data['pull_request']
    number = gh_pr['number']
    for wb in watched_branches:
        if number in wb.prs:
            await wb.notify_github_changed(event.app)

async def github_callback_handler(request):
    event = gh_sansio.Event.from_http(request.headers, await request.read())
    event.app = request.app  # type: ignore
    await gh_router.dispatch(event)

@routes.post('/github_callback')
async def github_callback(request):
    await asyncio.shield(github_callback_handler(request))
    return web.Response(status=200)

async def remove_namespace_from_db(db: Database, namespace: str):
    assert namespace != 'default'
    await db.just_execute(
        'DELETE FROM active_namespaces WHERE namespace = %s',
        (namespace,),
    )

async def batch_callback_handler(request):
    app = request.app
    db: Database = app['db']
    params = await json_request(request)
    log.info(f'batch callback {params}')
    attrs = params.get('attributes')
    if attrs:
        target_branch = attrs.get('target_branch')
        if target_branch:
            for wb in watched_branches:
                if wb.branch.short_str() == target_branch:
                    log.info(f'watched_branch {wb.branch.short_str()} notify batch changed')

                    if 'test' in attrs and params['complete']:
                        assert 'deploy' not in attrs
                        assert 'dev' not in attrs
                        namespace = attrs['namespace']
                        if DEFAULT_NAMESPACE == 'default':
                            await remove_namespace_from_db(db, namespace)

                    await wb.notify_batch_changed(app)

@routes.get('/api/v1alpha/deploy_status')
@auth.authenticated_developers_only()
async def deploy_status(request: web.Request, _) -> web.Response:
    batch_client = request.app['batch_client']

    async def get_failure_information(batch):
        if isinstance(batch, MergeFailureBatch):
            exc = batch.exception
            return traceback.format_exception(type(exc), value=exc, tb=exc.__traceback__)
        jobs = await collect_aiter(batch.jobs())

        async def fetch_job_and_log(j):
            full_job = await batch_client.get_job(j['batch_id'], j['job_id'])
            log = await full_job.log()
            return {**full_job._status, 'log': log}

        return await asyncio.gather(*[fetch_job_and_log(j) for j in jobs if j['state'] in ('Error', 'Failed')])

    wb_configs = [
        {
            'branch': wb.branch.short_str(),
            'sha': wb.sha,
            'deploy_batch_id': wb.deploy_batch.id if wb.deploy_batch and isinstance(wb.deploy_batch, Batch) else None,
            'deploy_state': wb.deploy_state,
            'repo': wb.branch.repo.short_str(),
            'failure_information': None
            if wb.deploy_state == 'success'
            else await get_failure_information(wb.deploy_batch),
        }
        for wb in watched_branches
    ]
    return json_response(wb_configs)

@routes.post('/api/v1alpha/update')
@auth.authenticated_developers_only()
async def post_update(request: web.Request, _) -> web.Response:
    log.info('developer triggered update')

    async def update_all():
        for wb in watched_branches:
            await wb.update(request.app)

    request.app['task_manager'].ensure_future(update_all())
    return web.Response(status=200)

@routes.post('/api/v1alpha/dev_deploy_branch')
@auth.authenticated_developers_only()
async def dev_deploy_branch(request: web.Request, userdata: UserData) -> web.Response:
    app = request.app
    try:
        params = await json_request(request)
    except asyncio.CancelledError:
        raise
    except Exception as e:
        message = 'could not read body as JSON'
        log.info('dev deploy failed: ' + message, exc_info=True)
        raise web.HTTPBadRequest(text=message) from e

    try:
        branch = FQBranch.from_short_str(params['branch'])
        steps = params['steps']
        excluded_steps = params['excluded_steps']
        extra_config = params.get('extra_config', {})
    except asyncio.CancelledError:
        raise
    except Exception as e:
        message = f'parameters are wrong; check the branch and steps syntax.\n\n{params}'
        log.info('dev deploy failed: ' + message, exc_info=True)
        raise web.HTTPBadRequest(text=message) from e

    gh = app['github_client']
    request_string = f'/repos/{branch.repo.owner}/{branch.repo.name}/git/refs/heads/{branch.name}'

    try:
        branch_gh_json = await gh.getitem(request_string)
        sha = branch_gh_json['object']['sha']
    except asyncio.CancelledError:
        raise
    except Exception as e:
        message = f'error finding {branch} at GitHub'
        log.info('dev deploy failed: ' + message, exc_info=True)
        raise web.HTTPBadRequest(text=message) from e

    unwatched_branch = UnwatchedBranch(branch, sha, userdata, app['developers'], extra_config=extra_config)

    batch_client = app['batch_client']

    try:
        batch_id = await unwatched_branch.deploy(app['db'], batch_client, steps, excluded_steps=excluded_steps)
    except asyncio.CancelledError:
        raise
    except Exception as e:  # pylint: disable=broad-except
        message = traceback.format_exc()
        raise web.HTTPBadRequest(text=f'starting the deploy failed due to\n{message}') from e
    return json_response({'sha': sha, 'batch_id': batch_id})

@routes.post('/api/v1alpha/batch_callback')
async def batch_callback(request):
    await asyncio.shield(batch_callback_handler(request))
    return web.Response(status=200)

@routes.post('/freeze_merge_deploy')
@auth.authenticated_developers_only()
async def freeze_deploys(request: web.Request, _) -> NoReturn:
    app = request.app
    db: Database = app['db']
    session = await aiohttp_session.get_session(request)

    if app['frozen_merge_deploy']:
        set_message(session, 'CI is already frozen.', 'info')
        raise web.HTTPFound(deploy_config.external_url('ci', '/'))

    await db.execute_update(
        '''
UPDATE globals SET frozen_merge_deploy = 1;
'''
    )

    app['frozen_merge_deploy'] = True

    set_message(session, 'Froze all merges and deploys.', 'info')

    raise web.HTTPFound(deploy_config.external_url('ci', '/'))

@routes.post('/unfreeze_merge_deploy')
@auth.authenticated_developers_only()
async def unfreeze_deploys(request: web.Request, _) -> NoReturn:
    app = request.app
    db: Database = app['db']
    session = await aiohttp_session.get_session(request)

    if not app['frozen_merge_deploy']:
        set_message(session, 'CI is already unfrozen.', 'info')
        raise web.HTTPFound(deploy_config.external_url('ci', '/'))

    await db.execute_update(
        '''
UPDATE globals SET frozen_merge_deploy = 0;
'''
    )

    app['frozen_merge_deploy'] = False

    set_message(session, 'Unfroze all merges and deploys.', 'info')

    raise web.HTTPFound(deploy_config.external_url('ci', '/'))

@routes.get('/namespaces')
@auth.authenticated_developers_only()
async def get_active_namespaces(request: web.Request, userdata: UserData) -> web.Response:
    db: Database = request.app['db']
    namespaces = [
        r
        async for r in db.execute_and_fetchall(
            '''
SELECT active_namespaces.*, JSON_ARRAYAGG(service) as services
FROM active_namespaces
LEFT JOIN deployed_services
ON active_namespaces.namespace = deployed_services.namespace
GROUP BY active_namespaces.namespace'''
        )
    ]
    for ns in namespaces:
        ns['services'] = [s for s in json.loads(ns['services']) if s is not None]
    context = {
        'namespaces': namespaces,
    }
    return await render_template('ci', request, userdata, 'namespaces.html', context)

@routes.post('/namespaces/{namespace}/services/add')
@auth.authenticated_developers_only()
async def add_namespaced_service(request: web.Request, _) -> NoReturn:
    db: Database = request.app['db']
    post = await request.post()
    service = post['service']
    namespace = request.match_info['namespace']

    record = await db.select_and_fetchone(
        '''
SELECT 1 FROM deployed_services
WHERE namespace = %s AND service = %s
''',
        (namespace, service),
    )

    if record:
        session = await aiohttp_session.get_session(request)
        set_message(session, 'Service already registered', 'info')
    else:
        await db.execute_insertone(
            'INSERT INTO deployed_services VALUES (%s, %s)',
            (namespace, service),
        )

    raise web.HTTPFound(deploy_config.external_url('ci', '/namespaces'))

@routes.post('/namespaces/add')
@auth.authenticated_developers_only()
async def add_namespace(request: web.Request, _) -> NoReturn:
    db: Database = request.app['db']
    post = await request.post()
    namespace = post['namespace']

    record = await db.execute_and_fetchone(
        'SELECT 1 FROM active_namespaces where namespace = %s',
        (namespace,),
    )

    if record:
        session = await aiohttp_session.get_session(request)
        set_message(session, 'Namespace already registered', 'info')
    else:
        await db.execute_insertone(
            'INSERT INTO active_namespaces (`namespace`) VALUES (%s)',
            (namespace,),
        )

    raise web.HTTPFound(deploy_config.external_url('ci', '/namespaces'))

async def cleanup_expired_namespaces(db: Database):
    assert DEFAULT_NAMESPACE == 'default'
    expired_namespaces = [
        record['namespace']
        async for record in db.execute_and_fetchall(
            'SELECT namespace FROM active_namespaces WHERE expiration_time < UTC_TIMESTAMP()'
        )
    ]
    for namespace in expired_namespaces:
        assert namespace != 'default'
        log.info(f'Cleaning up expired namespace: {namespace}')
        await remove_namespace_from_db(db, namespace)

async def update_envoy_configs(db: Database, k8s_client):
    assert DEFAULT_NAMESPACE == 'default'

    api_response = await k8s_client.list_namespace()
    live_namespaces = tuple(ns.metadata.name for ns in api_response.items)
    namespace_arg_list = "(" + ",".join('%s' for _ in live_namespaces) + ")"

    services_per_namespace = {
        r['namespace']: [s for s in json.loads(r['services']) if s is not None]
        async for r in db.execute_and_fetchall(
            f'''
SELECT active_namespaces.namespace, JSON_ARRAYAGG(service) as services
FROM active_namespaces
LEFT JOIN deployed_services
ON active_namespaces.namespace = deployed_services.namespace
WHERE active_namespaces.namespace IN {namespace_arg_list}
GROUP BY active_namespaces.namespace''',
            live_namespaces,
        )
    }
    assert 'default' in services_per_namespace
    default_services = services_per_namespace.pop('default')
    assert set(['batch', 'auth', 'batch-driver', 'ci']).issubset(set(default_services)), default_services

    for proxy in ('gateway', 'internal-gateway'):
        configmap_name = f'{proxy}-xds-config'
        configmap = await k8s_client.read_namespaced_config_map(
            name=configmap_name,
            namespace=DEFAULT_NAMESPACE,
        )
        cds = create_cds_response(default_services, services_per_namespace, proxy)
        rds = create_rds_response(default_services, services_per_namespace, proxy)
        configmap.data['cds.yaml'] = yaml.dump(cds)
        configmap.data['rds.yaml'] = yaml.dump(rds)
        await k8s_client.patch_namespaced_config_map(
            name=configmap_name,
            namespace=DEFAULT_NAMESPACE,
            body=configmap,
        )

async def update_loop(app):
    wb: Optional[WatchedBranch] = None
    while True:
        try:
            for wb in watched_branches:
                log.info(f'updating {wb.branch.short_str()}')
                await wb.update(app)
        except concurrent.futures.CancelledError:
            raise
        except Exception:  # pylint: disable=broad-except
            if wb:
                log.exception(f'{wb.branch.short_str()} update failed due to exception')
        await asyncio.sleep(300)

async def on_startup(app):
    client_session = httpx.client_session()
    app['client_session'] = client_session
    app['github_client'] = gh_aiohttp.GitHubAPI(app['client_session'], 'ci', oauth_token=oauth_token)
    app['batch_client'] = await BatchClient.create('ci')

    app['db'] = Database()
    await app['db'].async_init()

    row = await app['db'].select_and_fetchone(
        '''
SELECT frozen_merge_deploy FROM globals;
'''
    )

    app['frozen_merge_deploy'] = row['frozen_merge_deploy']

    app['task_manager'] = aiotools.BackgroundTaskManager()

    if DEFAULT_NAMESPACE == 'default':
        kubernetes_asyncio.config.load_incluster_config()
        k8s_client = kubernetes_asyncio.client.CoreV1Api()
        app['task_manager'].ensure_future(periodically_call(10, update_envoy_configs, app['db'], k8s_client))
        app['task_manager'].ensure_future(periodically_call(10, cleanup_expired_namespaces, app['db']))

    async with hail_credentials() as creds:
        headers = await creds.auth_headers()
    users = await retry_transient_errors(
        client_session.get_read_json,
        deploy_config.url('auth', '/api/v1alpha/users'),
        headers=headers,
    )
    app['developers'] = [u for u in users if u['is_developer'] == 1 and u['state'] == 'active']

    global watched_branches
    watched_branches = [
        WatchedBranch(index, FQBranch.from_short_str(bss), deployable, mergeable, app['developers'])
        for (index, [bss, deployable, mergeable]) in enumerate(
            json.loads(os.environ.get('HAIL_WATCHED_BRANCHES', '[]'))
        )
    ]

    app['task_manager'].ensure_future(update_loop(app))

async def on_cleanup(app):
    async with AsyncExitStack() as cleanup:
        cleanup.push_async_callback(app['db'].async_close)
        cleanup.push_async_callback(app['client_session'].close)
        cleanup.push_async_callback(app['batch_client'].close)
        cleanup.callback(app['task_manager'].shutdown)

def run():
    install_profiler_if_requested('ci')

    app = web.Application(middlewares=[check_csrf_token, monitor_endpoints_middleware])
    setup_aiohttp_jinja2(app, 'ci')
    setup_aiohttp_session(app)

    app.on_startup.append(on_startup)
    app.on_cleanup.append(on_cleanup)

    setup_common_static_routes(routes)
    app.add_routes(routes)
    app.router.add_get("/metrics", server_stats)

    web.run_app(
        deploy_config.prefix_application(app, 'ci'),
        host='0.0.0.0',
        port=5000,
        access_log_class=AccessLogger,
        ssl_context=internal_server_ssl_context(),
    )


#!/usr/bin/env python
# coding: utf-8

# In[27]:

from keras.applications import EfficientNetB0
from keras.layers import Dense, Conv2D, GlobalAveragePooling2D
from keras.layers import Dropout, Input, Softmax, RandomFlip
from tensorflow.keras.callbacks import ReduceLROnPlateau
from keras.models import Sequential
from keras.losses import binary_crossentropy
from keras.utils import image_dataset_from_directory

from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

import numpy as np
import matplotlib.pyplot as plt

# In[29]:

img_rows = 200
img_cols = 200
channels = 3

def load_dataset(path):
    dataset = image_dataset_from_directory(directory = path,
                                           label_mode = 'categorical',
                                           color_mode = 'rgb',
                                           shuffle = False,
                                           batch_size = None,
                                           image_size = (img_rows, img_cols),
                                           crop_to_aspect_ratio = True)
    return dataset

dataset1 = load_dataset('C:/Users/HP/Downloads/Brain-Tumor-Classification-DataSet-master/Brain-Tumor-Classification-DataSet-master/Training')
dataset2 = load_dataset('C:/Users/HP/Downloads/Brain-Tumor-Classification-DataSet-master/Brain-Tumor-Classification-DataSet-master/Testing')

images = []
labels = []
for (image, label) in dataset1:
    images.append(image)
    labels.append(label)

for (image, label) in dataset2:
    images.append(image)
    labels.append(label)

images = np.asarray(images)
labels = np.asarray(labels)
labels = labels.astype(np.uint8)

images, labels = shuffle(images, labels, random_state = 10)
x_train, x_test, y_train, y_test = train_test_split(images, labels, test_size = 0.1, random_state = 36)

# In[30]:

plt.figure(figsize = (50 * 3, 50))

for i in range(3):
    image = x_train[i].astype(np.uint8)
    plt.subplot(1, 3, i + 1)
    plt.imshow(image)

plt.tight_layout()
plt.show()

# In[31]:

def create_model(dropout_rate = 0.5, flip_mode = None, lr = 0.001):
    model = Sequential()
    model.add(Input(shape = (img_rows, img_cols, channels)))
    if(flip_mode != None):
        model.add(RandomFlip(flip_mode))

    model.add(EfficientNetB0(weights = 'imagenet', include_top = False))

    model.add(GlobalAveragePooling2D())
    model.add(Dropout(dropout_rate))
    model.add(Dense(4, activation = 'softmax'))

    model.build()
    model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])

    return model

# In[32]:

reduce_lr = ReduceLROnPlateau(monitor = 'val_accuracy', factor = 0.3, patience = 2,
                              min_delta = 0.01, mode = 'auto', verbose = 1)

# In[33]:

model = create_model(dropout_rate = 0.5, flip_mode = 'horizontal')
history = model.fit(x = x_train, y = y_train, validation_data = (x_test, y_test),
                    epochs = 2, batch_size = 32, callbacks = [reduce_lr])

# In[34]:

plt.rcParams.update({'font.size': 12})
def plot_performance(history, epochs, metric):
    fig, ax = plt.subplots(1, 1)

    ax.set_xlabel('Epochs')
    ax.set_ylabel(metric.title())

    x = list(range(1, epochs + 1))
    vy = history.history['val_' + metric]
    ty = history.history[metric]

    ax.plot(x, vy, 'b', label = 'Validation')
    ax.plot(x, ty, 'r', label = 'Train')

    plt.legend()
    plt.grid()
    fig.canvas.draw()

# In[36]:

plot_performance(history, 2, 'accuracy')
plot_performance(history, 2, 'loss')

# In[ ]:


import argparse
import cv2 as cv

ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required = True, help = "PAth to load image from")
args = vars(ap.parse_args())

image = cv.imread(args["image"])
print("Width: {} pixels, Height: {} pixels, Channels: {}".format(image.shape[1], image.shape[0], image.shape[2]))
cv.imshow("Original", image)
cv.waitKey(0)


"""
Inclusion tag for Organizer
"""

from django import template

register = template.Library()

@register.inclusion_tag('competition/templatetags/organizer/participant_list.html')
def participant_list(organizer):
    participants = organizer.competition.participant_set.all()

    return {"participants": participants}


from methods.regular.regular_api import *

import stripe

from shared.database.account.account import Account

from shared.permissions.account_permissions import Permission_Account

stripe.api_key = settings.STRIPE_API_KEY

@routes.route('/api/v1/project/<string:project_string_id>/account/billing/stripe/token',
              methods = ['POST'])
@Project_permissions.user_has_project(["admin",
                                       "Editor"], with_permission = "project_create_billing_account")
@limiter.limit("5 per day")
def stripe_new_customer_api(project_string_id):
    spec_list = [{"account_id": int},
                 {"token": dict}]

    log, input, untrusted_input = regular_input.master(request = request,
                                                       spec_list = spec_list)
    if len(log["error"].keys()) >= 1:
        return jsonify(log = log), 400

    if not settings.ALLOW_STRIPE_BILLING:
        return jsonify(log = {'error': {'ALLOW_STRIPE_BILLING': 'Not Allowed for this version'}}), 400

    with sessionMaker.session_scope() as session:
        member = get_member(session = session)
        user = member.user
        account_id = input.get('account_id')
        if account_id is None and user:
            account_list = Account.get_list(
                session = session,
                user_id = user.id,
                mode_trainer_or_builder = "builder",
                account_type = "billing",
                by_primary_user = True)
            if account_list:
                account_id = account_list[0].id
                logger.info(f'No account provided. Fallback to use Account ID: {new_account}')
            else:
                new_account = Account.account_new_core(
                    session = session,
                    primary_user = user,
                    mode_trainer_or_builder = "builder",
                    account_type = "billing",
                    nickname = "My Account")
                logger.info(f'User has no billing accounts attached. Creating a new one...')
                logger.info(f'New account created for payment: Account ID: {new_account}')
                account_id = new_account.id

        result, log = stripe_new_customer_core(session = session,
                                               token = input['token'],
                                               account_id = account_id,
                                               log = log)
        if result is False:
            return jsonify(log = log), 400

        # We should not need to do this here, but something funny
        # with account thing.
        project = Project.get(session, project_string_id)
        project.api_billing_enabled = True
        session.add(project)

        log['success'] = True
        return jsonify(log = log), 200

@Permission_Account.by_id()
def stripe_new_customer_core(session,
                             token,
                             account_id,
                             log):
    # TODO check if billing is already enabled for project?

    account = Account.get_by_id(session, account_id)

    project_list = Project.list(
        session = session,
        account_id = account.id,
        mode = "from_account_id")

    email = account.primary_user.email

    try:
        stripe_customer = stripe.Customer.create(source = token['id'],
                                                 email = email)
    except Exception as exception:
        print(exception)
        log['error']['stripe'] = "Invalid Stripe Token"
        return False, log

    session.add(account)
    account.stripe_id = stripe_customer.id
    account.payment_method_on_file = True

    # CAUTION assumes user
    user = User.get(session = session)

    # applies to many projects...
    # could also create many events instead...
    Event.new(
        kind = "billing_enabled",
        session = session,
        member_id = user.member_id,
        success = True,
        email = user.email
    )

    # Enable billing
    for project in project_list:
        # TODO this should be own function? had started a
        # billing_enable file

        session.add(project)
        project.api_billing_enabled = True

    account.credit_limit = 100 * 100  # $100

    return True, log


import os
import json
import sys

errors_sub = ["Term_Resource", "Term_Inconsistent", "Term_Wrong", "Acc_Mistrans", "Acc_Overtrans", "Acc_Undertrans",
          "Acc_Add", "Acc_Omi", "Acc_DNT", "Acc_Untrans", "Ling_Grammar",
          "Ling_Punct", "Ling_Spelling", "Ling_Unintelligible", "Ling_Encoding", "Style_Org", "Style_Third",
          "Style_Register", "Style_Awkward", "Style_Unidimoatic", "Style_Inconsistent", "Loc_Number",
          "Loc_Currency", "Loc_Measure", "Loc_Time", "Loc_Date", "Loc_Addr", "Loc_Tel", "Loc_Shortc",
          "AudienceAppropriateness", "DesignMarkup"]

errors_main = ["Terminology", "Accuracy", "Linguistic Conventions", "Style", "Locale", "Audience Appropriateness", "Design and Markup"]

def write_error_dict(error_dict, outfile):
    with open(outfile, "w", encoding="utf-8") as fo:
        for key, value in error_dict.items():
            new_list = [key] + [str(item) for item in value]
            fo.write("\t".join(new_list)+"\n")

def make_error_dict(errors_list):
    error_dict = {}

    for e in errors_list:
        # severity values: normal, minor, major, critical
        error_dict[e] = [0, 0, 0, 0]

    return error_dict

def calculate_sents_with_errors(sents_with_errors_dict, error_dict_file):
    opennmt_sents = []
    modernmt_sents = []
    deepl_sents = []
    for key, value in error_dict_file.items():
        if key == "SRC":continue
        for v in value:
            if v[-1].startswith("sent"):
                sent_id = int(v[-1].replace("sent", ""))
                if key == "deepl": deepl_sents.append(sent_id)
                elif key == "opennmt": opennmt_sents.append(sent_id)
                elif key == "modernmt": modernmt_sents.append(sent_id)
                else: continue

    opennmt_sents = list(set(opennmt_sents))
    modernmt_sents = list(set(modernmt_sents))
    deepl_sents = list(set(deepl_sents))
    sents_with_errors_dict["deepl"] += len(deepl_sents)
    sents_with_errors_dict["opennmt"] += len(opennmt_sents)
    sents_with_errors_dict["modernmt"] += len(modernmt_sents)

def calculate_mqm(deepl_dict, opennmt_dict, modernmt_dict, src_dict, error_dict_file):
    for key, value in error_dict_file.items():
        print()
        print(key)

        for e in errors_sub:
            for v in value:
                if v[0] == e:
                    try:
                        severity = int(v[1])
                        # Increment error for each specific severity type
                        if key == "deepl": deepl_dict[e][severity]+=1
                        elif key == "opennmt": opennmt_dict[e][severity]+=1
                        elif key == "modernmt": modernmt_dict[e][severity]+=1
                        elif key == "SRC": src_dict[e][severity]+=1
                    except:
                        if key == "SRC":
                            # for SRC segments only increment neutral severity type
                            src_dict[e][0] += 1
                        else:
                            print("No severity found for:")
                            print(key, value)
                            sys.exit()

def analyse(folder_input):

    deepl_dict = make_error_dict(errors_sub)
    opennmt_dict = make_error_dict(errors_sub)
    modernmt_dict = make_error_dict(errors_sub)
    src_dict = make_error_dict(errors_sub)

    # Create a dictionary to count sentences with errors per system
    sents_with_errors_dict = {"deepl":0, "opennmt":0, "modernmt": 0}

    # Loop over each file in the folder
    for filename in os.listdir(folder_input):
        # Check if the file is a JSON file
        if filename.endswith(".json"):
            print()
            print(filename)
            print("___________________________")
            # Open the file and read its contents
            with open(os.path.join(folder_input, filename), "r") as file:
                error_dict_file = json.load(file)
                print(error_dict_file)
                calculate_mqm(deepl_dict, opennmt_dict, modernmt_dict, src_dict, error_dict_file)
                calculate_sents_with_errors(sents_with_errors_dict, error_dict_file)

    out_opennmt = os.path.join(folder_input, "opennmt_mqm.txt")
    out_deepl = os.path.join(folder_input, "deepl_mqm.txt")
    out_modernmt = os.path.join(folder_input, "modernmt_mqm.txt")
    print("OpenNMT:")
    print(opennmt_dict)
    write_error_dict(opennmt_dict, out_opennmt)
    print("")
    print("DeepL:")
    print(deepl_dict)
    write_error_dict(deepl_dict, out_deepl)
    print("")
    print("ModernMT:")
    print(modernmt_dict)
    write_error_dict(modernmt_dict, out_modernmt)
    print("")
    print("SRC:")
    print(src_dict)
    print("")
    print("Sentences with errors:")
    print(sents_with_errors_dict)

if __name__ == '__main__':
    # Check if the folder path and the term list are provided as a command-line argument
    if len(sys.argv) != 2:
        print('Usage: python labelstudio_analyse_annotations.py <folder_input>')
        sys.exit(1)

    folder_input = sys.argv[1]

    # Call the analyse function
    analyse(folder_input)


# -*- coding: utf-8 -*-
import glob
import json
import os
import warnings

from .exceptions import DataLoaderException
from .grains import Grain
from .grains import GrainAddition
from .hops import Hop
from .hops import HopAddition
from .recipes import Recipe
from .yeasts import Yeast

__all__ = [
    u"DataLoader",
    u"JSONDataLoader",
    u"parse_cereals",
    u"parse_hops",
    u"parse_yeast",
    u"parse_recipe",
]

class DataLoader(object):
    """
    Base class for loading data from data files inside the data_dir.
    """

    #: A local cache of loaded data
    DATA = {}
    #: The expected file extension (json, xml, csv)
    EXT = ""

    def __init__(self, data_dir):
        """
        :param str data_dir: The directory where the data resides
        """
        if not os.path.isdir(data_dir):
            raise DataLoaderException(
                u"Directory '{}' does not exist".format(data_dir)
            )  # noqa
        self.data_dir = data_dir

    @classmethod
    def format_name(cls, name):
        """
        Reformat a given name to match the filename of a data file.
        """
        return name.lower().replace(u" ", u"_").replace(u"-", u"_")

    @classmethod
    def read_data(cls, filename):
        """
        :param str filename: The filename of the file to read
        :raises NotImplementedError: Must be supplied in inherited class
        """
        raise NotImplementedError

    def get_item(self, dir_suffix, item_name):
        """
        :param str dir_suffix: The directory name suffix
        :param str item_name: The name of the item to load
        :return: The item as a python dict
        :raises DataLoaderException: If item directory does not exist
        :raises Warning: If item not found in the directory
        """
        item_dir = os.path.join(self.data_dir, dir_suffix)
        if not os.path.isdir(item_dir):
            raise DataLoaderException(
                u"Item directory '{}' does not exist".format(item_dir)
            )  # noqa

        # Cache the directory
        if dir_suffix not in self.DATA:
            self.DATA[dir_suffix] = {}
            for item in glob.glob("{}*.{}".format(item_dir, self.EXT)):
                ext_len = len(self.EXT) + 1
                filename = os.path.basename(item)[:-ext_len]
                self.DATA[dir_suffix][filename] = {}

        name = self.format_name(item_name)
        if name not in self.DATA[dir_suffix]:
            warnings.warn(
                u"Item from {} dir not found: {}".format(dir_suffix, name)  # noqa
            )
            return {}

        # Cache file data
        if not self.DATA[dir_suffix][name]:
            item_filename = os.path.join(
                item_dir, "{}.{}".format(name, self.EXT)
            )  # noqa
            data = self.read_data(item_filename)
            self.DATA[dir_suffix][name] = data
            return data
        else:
            return self.DATA[dir_suffix][name]

class JSONDataLoader(DataLoader):
    """
    Load data from JSON files inside the data_dir.
    """

    #: The JSON file extension
    EXT = "json"

    @classmethod
    def read_data(cls, filename):
        """
        :param str filename: The filename of the file to read
        :return: The data loaded from a JSON file
        """
        data = None
        with open(filename, "r") as data_file:
            data = json.loads(data_file.read())
        return data

def parse_cereals(cereal, loader, dir_suffix="cereals/"):
    """
    Parse grains data from a recipe

    :param dict cereal: A representation of a cereal
    :param DataLoader loader: A class to load additional information

    Grain must have the following top level attributes:

    * name   (str)
    * weight (float)
    * data   (dict) (optional)

    Additionally grains may contain override data in the 'data'
    attribute with the following keys:

    * color (float)
    * ppg   (int)
    """
    GrainAddition.validate(cereal)

    cereal_data = loader.get_item(dir_suffix, cereal[u"name"])

    name = cereal_data.get(u"name", cereal[u"name"])
    color = None
    ppg = None

    if u"data" in cereal:
        color = cereal[u"data"].get(u"color", None)
        ppg = cereal[u"data"].get(u"ppg", None)

    if not color:
        color = cereal_data.get(u"color", None)

    if not ppg:
        ppg = cereal_data.get(u"ppg", None)

    grain_obj = Grain(name, color=color, ppg=ppg)

    grain_add_kwargs = {u"weight": float(cereal[u"weight"])}
    if u"grain_type" in cereal:
        grain_add_kwargs[u"grain_type"] = cereal[u"grain_type"]
    if u"units" in cereal:
        grain_add_kwargs[u"units"] = cereal[u"units"]
    return GrainAddition(grain_obj, **grain_add_kwargs)

def parse_hops(hop, loader, dir_suffix="hops/"):
    """
    Parse hops data from a recipe

    :param dict hops: A representation of a hop
    :param DataLoader loader: A class to load additional information

    Hops must have the following top level attributes:

    * name      (str)
    * weight    (float)
    * boil_time (float)
    * data      (dict) (optional)

    Additionally hops may contain override data in the 'data' attribute
    with the following keys:

    * percent_alpha_acids (float)
    """
    HopAddition.validate(hop)

    hop_data = loader.get_item(dir_suffix, hop[u"name"])

    name = hop_data.get(u"name", hop[u"name"])
    alpha_acids = None

    if u"data" in hop:
        alpha_acids = hop[u"data"].get(u"percent_alpha_acids", None)

    if not alpha_acids:
        alpha_acids = hop_data.get(u"percent_alpha_acids", None)

    hop_obj = Hop(name, percent_alpha_acids=alpha_acids)
    hop_add_kwargs = {u"weight": float(hop[u"weight"]), u"boil_time": hop[u"boil_time"]}
    if u"hop_type" in hop:
        hop_add_kwargs[u"hop_type"] = hop[u"hop_type"]
    if u"units" in hop:
        hop_add_kwargs[u"units"] = hop[u"units"]
    return HopAddition(hop_obj, **hop_add_kwargs)

def parse_yeast(yeast, loader, dir_suffix="yeast/"):
    """
    Parse yeast data from a recipe

    :param dict hops: A representation of a yeast
    :param DataLoader loader: A class to load additional information

    Yeast must have the following top level attributes:

    * name (str)
    * data (dict) (optional)

    Additionally yeast may contain override data in the 'data' attribute
    with the following keys:

    * percent_attenuation (float)
    """
    Yeast.validate(yeast)

    yeast_data = loader.get_item(dir_suffix, yeast[u"name"])  # noqa

    name = yeast_data.get(u"name", yeast[u"name"])
    attenuation = None

    if u"data" in yeast:
        attenuation = yeast[u"data"].get(u"percent_attenuation", None)

    if not attenuation:
        attenuation = yeast_data.get(u"percent_attenuation", None)

    return Yeast(name, percent_attenuation=attenuation)

def parse_recipe(
    recipe,
    loader,
    cereals_loader=None,
    hops_loader=None,
    yeast_loader=None,
    cereals_dir_suffix="cereals/",
    hops_dir_suffix="hops/",
    yeast_dir_suffix="yeast/",
):
    """
    Parse a recipe from a python Dict

    :param dict recipe: A representation of a recipe
    :param DataLoader loader: A class to load additional information
    :param DataLoader cereal_loader: A class to load additional information specific to cereals
    :param DataLoader hops_loader: A class to load additional information specific to hops
    :param DataLoader yeast_loader: A class to load additional information specific to yeast

    A recipe must have the following top level attributes:

    * name         (str)
    * start_volume (float)
    * final_volume (float)
    * grains       (list(dict))
    * hops         (list(dict))
    * yeast        (dict)

    Additionally the recipe may contain override data in the 'data'
    attribute with the following keys:

    * brew_house_yield (float)
    * units            (str)

    All other fields will be ignored and may be used for other metadata.

    The dict objects in the grains, hops, and yeast values are required to have
    the key 'name' and the remaining attributes will be looked up in the data
    directory if they are not provided.
    """  # noqa
    if cereals_loader is None:
        cereals_loader = loader
    if hops_loader is None:
        hops_loader = loader
    if yeast_loader is None:
        yeast_loader = loader

    Recipe.validate(recipe)

    grain_additions = []
    for grain in recipe[u"grains"]:
        grain_additions.append(
            parse_cereals(grain, cereals_loader, dir_suffix=cereals_dir_suffix)
        )

    hop_additions = []
    for hop in recipe[u"hops"]:
        hop_additions.append(parse_hops(hop, hops_loader, dir_suffix=hops_dir_suffix))

    yeast = parse_yeast(recipe[u"yeast"], yeast_loader, dir_suffix=yeast_dir_suffix)

    recipe_kwargs = {
        u"grain_additions": grain_additions,
        u"hop_additions": hop_additions,
        u"yeast": yeast,
        u"start_volume": recipe[u"start_volume"],
        u"final_volume": recipe[u"final_volume"],
    }
    if u"data" in recipe:
        if u"brew_house_yield" in recipe[u"data"]:
            recipe_kwargs[u"brew_house_yield"] = recipe[u"data"][u"brew_house_yield"]
        if u"units" in recipe[u"data"]:
            recipe_kwargs[u"units"] = recipe[u"data"][u"units"]

    beer = Recipe(recipe[u"name"], **recipe_kwargs)
    return beer


import numpy as np
import cv2
import csv
import sys
import os

# open setting csv to record the images
setting_file = open('setting.csv', 'w')
setting_writer = csv.writer(setting_file)
setting_file2 = open('setting2.csv', 'w')
setting_writer2 = csv.writer(setting_file2)

# iterate all images
for filename in os.listdir("./image"):
    if(filename[-4:] != ".jpg" and filename[-4:] != ".png"):
        continue
    with open(os.path.join("./image", filename), 'r') as f:
        # read img
        print("opening " + filename)
        img = cv2.imread("./image/" + filename)

        # turn gray
        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # turn blur
        blur_img = cv2.GaussianBlur(gray_img, (11, 11), 0)

        # Edge detection
        edges = cv2.Canny(blur_img, 20, 30)

        # thicken the edges
        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
        edges = cv2.dilate(edges, kernel, iterations=1)

        # write img
        #cv2.imwrite("image/_" + filename, edges)

        # transfer coordinates to csv
        indices = np.where(edges != [0])

        # shuffle the order of coordinate
        lst = list(range(len(indices[0])))
        np.random.shuffle(lst)

        # csv name
        new_file_name = filename[:-4]
        print("writing " + new_file_name + ".csv")

        # open input csv
        file = open('csv/'+ new_file_name+'.csv', 'w')
        writer = csv.writer(file)

        # img setting into csv
        h, w = edges.shape
        width_height = [int(w),int(h)]
        data = width_height
        writer.writerow(data)

        # img coordinate into csv
        for i in lst:
            data = [indices[0][i], indices[1][i]]
            bgr = img[int(indices[0][i]), int(indices[1][i])]
            data.append(bgr[0])
            data.append(bgr[1])
            data.append(bgr[2])
            writer.writerow(data)
        file.close()

        # adding image and songs to setting.csv
        count = 0
        data = [filename[:-4]]
        for song_name in os.listdir("./music"):
            if(song_name[0:len(filename[:-4])] == filename[:-4] ):
                data.append(song_name)
                count += 1
        data.insert(1,count)
        setting_writer.writerow(data)

# adding image to each songs in setting2.csv
for song_name in os.listdir("./music"):
    data = [song_name]
    x = song_name.split("_")
    if(x[1]=="JingleBell.wav"):
        data.append("x_"+x[0])
    else:
        data.append(x[0])
    setting_writer2.writerow(data)

setting_file.close()
setting_file2.close()


import fontawesome as fa
import datetime
import pytz
from astral import Astral
from ...Fibaro.Model.room import OpeningState
from ...Fibaro.Model.room import MonoxideState
from ...utils.helper import get_season
from ...utils.helper import get_day_of_week

class MapDrawer:
    def __init__(self, city, log=None):
        self._astral = Astral()
        self._astral.solar_depression = 'civil'
        self._city = city
        self._astral = self._astral[self._city]
        self._room_info = log

    def draw(self, handle):
        handle = self.draw_room_wall(handle)
        handle = self.draw_sensors(handle)
        return handle

    def draw_sensors(self, handle):
        handle = self.draw_motion_sensor(handle)
        handle = self.draw_noise_sensor(handle)
        handle = self.draw_light_sensor(handle)
        handle = self.draw_thermometer(handle)
        handle = self.draw_humidity(handle)
        handle = self.draw_vaccum_activity(handle)
        handle = self.draw_smoke_detector(handle)
        handle = self.draw_openings(handle)
        return handle

    def draw_openings(self, handle):
        if self._room_info.door == OpeningState.OPEN:
            handle = self.draw_door_sensor(handle)
        if self._room_info.shutter == OpeningState.OPEN:
            handle = self.draw_shutter_sensor(handle)
        return handle

    def draw_date(self, handle, x=90, y=246, size='15', color='black'):
        hour = datetime.datetime.now().strftime('%H:%M:%S')
        day_of_week = get_day_of_week()
        season = get_season()

        season_icon = fa.icons["snowflake"]
        if season == 'spring':
            season_icon = fa.icons["yelp"]
        elif season == 'summer':
            season_icon = fa.icons["sun"]
        elif season == 'fall':
            season_icon = fa.icons["envira"]

        utc = pytz.UTC
        sun = self._astral.sun(date=datetime.datetime.now(), local=True)
        sun_icon = fa.icons["moon"]
        if sun['sunrise'].replace(tzinfo=utc) < datetime.datetime.now().replace(tzinfo=utc) < sun['sunset'].replace(tzinfo=utc):
            sun_icon = fa.icons["sun"]

        text = day_of_week + '  ' + hour + '  ' + sun_icon + '  ' + season_icon
        handle = self.draw_text(handle, text, x, y, size, color)
        return handle

    def draw_door_sensor(self, handle):
        return handle

    def draw_shutter_sensor(self, handle):
        return handle

    def draw_vaccum_activity(self, handle):
        return handle

    def draw_light_icon(self, handle, x, y, size='15', color='yellow'):
        if self._room_info.light:
            handle = self.draw_icon(handle, 'lightbulb', x, y, size, color)
        return handle

    def draw_motion_icon(self, handle, x, y, size='15', color='black'):
        if self._room_info.motion:
            handle = self.draw_icon(handle, 'eye', x, y, size, color)
        return handle

    def draw_noise_icon(self, handle, x, y, size='15', color='black'):
        if self._room_info.noise:
            handle = self.draw_icon(handle, 'volume-up', x, y, size, color)
        return handle

    def draw_vaccum_icon(self, handle, x, y, size='15', color='black'):
        if self._room_info.vaccum:
            handle = self.draw_icon(handle, 'codiepie', x, y, size, color)
        return handle

    def draw_drop_icon(self, handle, x, y, size='15', color='blue'):
        if self._room_info.humidity:
            handle = self.draw_icon(handle, 'tint', x, y, size, color)
        return handle

    def draw_smoke_icon(self, handle, x, y, size='15', color='lightgray'):
        if self._room_info.gaz == MonoxideState.MODERATE:
            handle = self.draw_icon(handle, 'cloud', x, y, size, color)
        elif self._room_info.gaz == MonoxideState.DANGEROUS:
            handle = self.draw_icon(handle, 'cloud', x, y, size, 'grey')
        else:
            handle = self.draw_icon(handle, 'cloud', x, y, size, 'white')
        return handle

    def draw_thermometer_icon(self, handle, x, y, size='15'):
        if self._room_info.temperature is not None:
            temperature = int(self._room_info.temperature)
            if temperature < 4:
                color = 'cyan'
                icon = 'snowflake'
                x -= 4
            elif temperature < 18:
                color = 'blue'
                icon = 'thermometer-empty'
            elif temperature <= 21:
                color = 'orange'
                icon = 'thermometer-half'
            else:
                color = 'red'
                icon = 'thermometer-full'
            handle = self.draw_icon(handle, icon, x, y, size, color)
        return handle

    def draw_pet_icon(self, handle, x, y, size='15', color='black'):
        handle = self.draw_icon(handle, 'paw', x, y, size, color)
        return handle

    def draw_context(self, handle, icon, x=10, y=246, size='30', color='black'):
        handle = self.draw_icon(handle, icon, x, y, size, color)
        return handle

    def draw_icon(self, handle, icon, x, y, size='15', color='black'):
        handle.add(handle.text(fa.icons[icon], insert=(x, y), fill=color, font_size=size+'px'))
        return handle

    def draw_text(self, handle, text, x, y, size='15', color='black'):
        handle.add(handle.text(text, insert=(x, y), fill=color, font_size=size+'px'))
        return handle

    def draw_door(self, handle, x1, y1, x2, y2, width="2", color="white"):
        handle.add(handle.line((x1, y1), (x2, y2), stroke_width=width, stroke=color))
        return handle

    def print_all_fontawesome(self):
        for icon in fa.icons:
            print(icon + '    ' + fa.icons[icon])


#!/root/prediccionService/venv/bin/python

import os
import datetime
#import matplotlib as mpl
#import matplotlib.pyplot as plt
import numpy as np
import pandas
import tensorflow as tf
import sqlalchemy
import pymysql

def createTimeFeatures (dfToExpand):
    # A pesar de estar definida como una funcion que crea y retorna una copia del dataset que se le es proveida
    # aun así el dataset original es modificado, no se por que
    dfToExpand['hour'] = pandas.to_datetime(dfToExpand['utc']).dt.hour
    dfToExpand['day'] = pandas.to_datetime(dfToExpand['utc']).dt.day
    dfToExpand['month'] = pandas.to_datetime(dfToExpand['utc']).dt.month
    dfToExpand['year'] = pandas.to_datetime(dfToExpand['utc']).dt.year
    return dfToExpand

# El modelo DNN existente solo necesita horas, días y minutos arbitrarios para poder generar un pronóstico relativamente fiable
# -MSE de 41, 5 y 1.5 para HR, QFE y TS respectivamente-
# por lo tanto si se desea hacer pronostico del clima para las proximas 24 horas, basta con obtener la fecha y hora actual, generar los pasos de tiempo
# de una hora y descomponerla en caracteristicas de tiempo significativas
# aunque dado el input con el que fue entrenado el modelo, se le puede ser ingresado cualquier momento en el tiempo arbitrario sin necesidad de ser
# consecutivo

# hour day month in that order
try:
    credentials = np.genfromtxt("../pass",dtype='str')
    engine = sqlalchemy.create_engine("mysql+pymysql://"+credentials[0]+":"+credentials[1]+"@"+credentials[2]+"/"+credentials[3] )
    mydb = engine.connect()
    query = "SELECT * FROM WEATHER_MEASUREMENT ORDER BY ID DESC LIMIT 1;"
    df = pandas.read_sql(query,mydb)
except:
    mydb.close()
    print("error conexion a db")

df["utc"] = pandas.to_datetime(df["serverDate"],format='%Y-%m-%d %H:%M:%S')
df = df[['AMBIENT_TEMPERATURE','AIR_PRESSURE','HUMIDITY','utc']]
# Agrupando parametros segun hora para obtener un dataset de 24 filas
df2 = df.groupby(pandas.Grouper(key="utc",freq='H')).mean()
df2 = df2.reset_index()

# Generando fechas futuras
# Se extremadamente cuidados con los tipos de datos de now y delta
# delta hereda de now, y now dependiendo del metodo por el que se obtiene puede ser un pandas.timestamp o un datetime.datetime
# ambos tienen forma de acceso a sus atributos distintos, por lo tanto cuidar esto
# now = datetime.datetime.now()
now = pandas.to_datetime(df2[-1:]['utc'])
stackPredictors = pandas.DataFrame()
deltaStack = pandas.DataFrame()
for i in range(72):
    delta = now + datetime.timedelta(0,i*3600)
    deltaStack = deltaStack.append(pandas.DataFrame(delta))
    # deltaStack = deltaStack.append([delta])
    # temp = np.array([delta.hour,delta.day,delta.month],dtype="float32")
    temp = np.array([delta.dt.hour,delta.dt.day,delta.dt.month],dtype="float32")
    stackPredictors = stackPredictors.append(pandas.DataFrame(temp).transpose())

stackPredictors = stackPredictors.reset_index(drop=True)
deltaStack = deltaStack.reset_index(drop=True)
stackPredictors.columns = ['hour','day','month']

# Normalizando manualmente
# el modelo fue entrenado con valores normalizados generados en base a metricas -promedio, desviacion estandar- del dataset de
# entrenamiento, por lo tanto, estas metricas han de calcularse también para respetar la normalizacion (promedios y dev_est)
# con los que fue entrenado el modelo

df = pandas.read_csv("dataPreprocessed.csv")
df = createTimeFeatures(df)
dates_df = df.pop('year')
dates_df = df.pop('utc')
train_mean = df.mean()
train_std = df.std()

train_mean.index = ['AMBIENT_TEMPERATURE','HUMIDITY','AIR_PRESSURE','hour','day','month']
train_std.index = ['AMBIENT_TEMPERATURE','HUMIDITY','AIR_PRESSURE','hour','day','month']
stackPredictors = (stackPredictors - train_mean[3:]) / train_std[3:]

# Cargado de modelo
# dnnMultivar = tf.keras.models.load_model('../../pyCNN/deepNN-multiV2')
dnnMultivar = tf.keras.models.load_model('../../../models/dnn/')
# Realizando pronosticos
stackPreds = dnnMultivar.predict(stackPredictors)
stackPreds = pandas.DataFrame(stackPreds)
stackPreds.columns = ['AMBIENT_TEMPERATURE','HUMIDITY','AIR_PRESSURE']
# Construyendo tabla de predicciones final, conteniendo predictores y Predicciones
stackPreds = pandas.concat([stackPreds,stackPredictors],axis=1)
# De-normalizando valores
stackPreds = stackPreds * train_std + train_mean
# El año no se considero en el entrenamiento pues disminuia la varianza de las predicciones demasiado
stackPreds['utc'] = deltaStack

#stackPreds.plot(subplots=True)
#plt.plot(df4[0,0:24,1])
#plt.plot(df4[0,0:24,2])
#plt.grid()
#plt.show()

stackPreds.AMBIENT_TEMPERATURE = stackPreds.AMBIENT_TEMPERATURE.round(3)
stackPreds.HUMIDITY = stackPreds.HUMIDITY.round(3)
stackPreds.AIR_PRESSURE = stackPreds.AIR_PRESSURE.round(3)
#stackPreds.hour = stackPreds.hour.round(0).astype(int)
#stackPreds.day = stackPreds.day.round(0).astype(int)
#stackPreds.month = stackPreds.month.round(0).astype(int)
stackPreds.pop('hour')
stackPreds.pop('day')
stackPreds.pop('month')

# stackPreds['utc'] =stackPreds.year.astype(str)+'-'+stackPreds.month.astype(str)+'-'+stackPreds.day.astype(str)+' '+stackPreds.hour.astype(str)+':00:00'
stackPreds['utc'] = pandas.to_datetime(stackPreds['utc'])

try:
    #credentials = np.genfromtxt("../viz/scripts/pass",dtype='str')
    # credentials = np.genfromtxt("../pass",dtype='str')
    # engine = sqlalchemy.create_engine("mysql+pymysql://"+credentials[0]+":"+credentials[1]+"@"+credentials[2]+"/"+credentials[3] )
    # mydb = engine.connect()
    stackPreds.to_sql('dnnPredictions',mydb,if_exists='append',index=False)
    query = "SHOW COLUMNS FROM `weather` LIKE 'id';"
    a = mydb.execute(query)
    if a.fetchall():
        print("Columna id existente")
    else:
        print("Añadiendo columna id")
        query = "ALTER TABLE arimaPredictions ADD id INT PRIMARY KEY AUTO_INCREMENT;"
        mydb.execute(query)
except:
    mydb.close() #close the connectionexcept Exception as e:
    print('Error en conexion a base de datos')

mydb.close()
engine.dispose()
## script ready to be callbable


from collections import namedtuple
from datetime import date, timedelta
from functools import wraps
import pdb
from dateutil.relativedelta import relativedelta
from flask import (
    abort,
    jsonify,
    redirect,
    render_template,
    request,
    Response,
    url_for,
)
from .sitemap import create_sitemap
from .converter import LuceneQueryConverter, ReviewServiceConverter, ListConverter
from .queries import (
    STATS, BY_DOIS, BY_SLUG, FIG_BY_DOI_IDX,
    DESCRIBE_REVIEWING_SERVICES,
    REVIEW_PROCESS_BY_DOI, REVIEW_MATERIAL_BY_ID,
    DOCMAP_BY_DOI, DOCMAPS_FROM_REVSERVICE_IN_INTERVAL,
    DOCMAPS_IN_INTERVAL,
    BY_AUTO_TOPICS, BY_REVIEWING_SERVICE, AUTOMAGIC,
    LUCENE_SEARCH, SEARCH_DOI,
    COVID19, REFEREED_PREPRINTS,
    COLLECTION_NAMES,
    SUBJECT_COLLECTIONS,
)
from neotools.db import Query
from typing import Dict
import re
from . import app, cache, get_db

DOI_REGEX = re.compile(r'10.\d{4,9}/[-._;()/:A-Z0-9]+$', flags=re.IGNORECASE)

app.url_map.converters['escape_lucene'] = LuceneQueryConverter
app.url_map.converters['service_name'] = ReviewServiceConverter
app.url_map.converters['list'] = ListConverter

def n_months_ago(n):
    n_months_ago = date.today() + relativedelta(months=-n)
    first_day_of_that_month = n_months_ago.replace(day=1)
    return str(first_day_of_that_month)

def ask_neo(query: Query, **kwargs) -> Dict:
    """
    Run a query and return the database results as dictionary with the keys specified in the query.returns list.
    """
    def tx_funct(tx, code, params):
        results = tx.run(code, params)
        data = [r.data(*query.returns) for r in results]  # consuming the data inside the transaction https://neo4j.com/docs/api/python-driver/current/transactions.html
        return data
    # use the map of name of substitution variable in cypher to the name and default value of the var in the request
    for var_in_cypher, var_in_request in query.map.items():
        query.params[var_in_cypher] = kwargs.get(var_in_request['req_param'], var_in_request['default'])
    data = get_db().query_with_tx_funct(tx_funct, query)
    return data

def get_all_dois_and_slugs():
    refereed_preprints = ask_neo(BY_REVIEWING_SERVICE(), limit_date='1900-01-01')
    by_auto_topics = ask_neo(BY_AUTO_TOPICS(), limit_date='1900-01-01')
    automagic = ask_neo(AUTOMAGIC(), limit_date='1900-01-01')
    app.logger.info("gathering all dois")

    dois_and_slugs = []
    for collection in refereed_preprints, by_auto_topics, automagic:
        for sub_collection in collection:
            papers = sub_collection['papers']
            new_dois_and_slugs = [(paper['doi'], paper.get('slug', None)) for paper in papers]
            dois_and_slugs += new_dois_and_slugs
    dois_and_slugs = set(dois_and_slugs)  # remove duplicates
    return dois_and_slugs

@app.route('/')
def root():
    return render_template('index.html')

@app.route('/doi/<path:doi>', methods=['GET'])
@cache.cached()
def doi_redirect(doi):
    papers = ask_neo(BY_DOIS(), dois=[doi])
    if len(papers) == 0:
        app.logger.debug("tried to redirect DOI %s to slug, no matching paper found", doi)
        return abort(404)
    if len(papers) > 1:
        app.logger.warning("found multiple VizPapers for DOI %s, using first one: %s", doi, papers)

    paper = papers[0]
    slug = paper.get('slug', None)
    if slug:
        app.logger.debug("redirecting DOI %s to slug %s", doi, slug)
        return redirect('/p/' + slug, code=301)
    return render_template('index.html')

# @app.route('/doc')
# def doc():
#     return render_template('doc.html', name='me')

@app.route('/sitemap.xml', methods=['GET'])
@cache.cached()
def sitemap():
    """
    Generate dynamically a sitemap.
    """
    dois_and_slugs = get_all_dois_and_slugs()
    app.logger.info(f"generating sitemap with {len(dois_and_slugs)} links.")
    sitemap = create_sitemap(dois_and_slugs)
    return Response(sitemap, mimetype='text/xml')

@app.route('/api/v1/stats', methods=['GET', 'POST'])
@cache.cached()
def stats():
    app.logger.info(f"show db stats")
    return jsonify(ask_neo(STATS()))

# using routing rather than parameters to provide limit_date so that cache.cached() works properly; memoize would need function params
@app.route('/api/v1/by_auto_topics/', defaults={'limit_date': None}, methods=['GET', 'POST'])
@app.route('/api/v1/by_auto_topics/<limit_date>', methods=['GET', 'POST'])
@cache.cached()
def by_hyp(limit_date):
    if limit_date is None:
        limit_date = n_months_ago(4)
    app.logger.info(f"list by automatic topics with limit {limit_date}")
    return jsonify(ask_neo(BY_AUTO_TOPICS(), limit_date=limit_date))

@app.route('/api/v1/reviewing_services/', methods=['GET', 'POST'])
@cache.cached()
def reviewing_services():
    app.logger.info(f"descriptions of reviewing services")
    return jsonify(ask_neo(DESCRIBE_REVIEWING_SERVICES()))

@app.route('/api/v1/by_reviewing_service/', defaults={'limit_date': '1900-01-01'}, methods=['GET', 'POST'])
@app.route('/api/v1/by_reviewing_service/<limit_date>', methods=['GET', 'POST'])
@cache.cached()
def by_reviewing_service(limit_date):
    app.logger.info(f"list by by_reviewing_service")
    return jsonify(ask_neo(BY_REVIEWING_SERVICE(), limit_date=limit_date))

@app.route('/api/v1/automagic/', defaults={'limit_date': '1900-01-01'}, methods=['GET', 'POST'])
@app.route('/api/v1/automagic/<limit_date>', methods=['GET', 'POST'])
@cache.cached()
def automagic(limit_date):
    app.logger.info(f"list by automagic score")
    return jsonify(ask_neo(AUTOMAGIC(), limit_date=limit_date))

@app.route('/api/v1/doi/<path:doi>', methods=['GET', 'POST'])
@cache.cached()
def by_doi(doi: str):
    app.logger.debug(f"lookup doi: {doi}")
    result = ask_neo(BY_DOIS(), dois=[doi])
    return jsonify(result)

@app.route('/api/v1/slug/<path:slug>', methods=['GET', 'POST'])
@cache.cached()
def by_slug(slug: str):
    app.logger.debug(f"lookup slug: {slug}")
    result = ask_neo(BY_SLUG(), slug=slug)
    return jsonify(result)

@app.route('/api/v1/dois/', methods=['POST'])
def by_dois():
    if not request.is_json:
        abort(415) # unsupported media type
    if not 'dois' in request.json:
        abort(400) # required parameter is missing
    dois = request.json.get('dois', [])
    published_in = request.json.get('published_in', '')
    num_dois = len(dois)
    dois_info = f'{dois}' if num_dois < 4 else f'["{dois[0]}", "{dois[1]}", ..., "{dois[-1]}"] ({num_dois} in total)'
    app.logger.debug(f"lookup dois: {dois_info} {' published in '+published_in if published_in else ''}")

    cache_key = f'/api/v1/dois/{hash(frozenset(dois))}'
    doi_data = cache.get(cache_key)
    if doi_data is None:
        app.logger.debug(f"\t\t cache miss: {cache_key}")
        doi_data = ask_neo(BY_DOIS(), dois=dois, published_in=published_in)
        cache.add(cache_key, doi_data)
    else:
        app.logger.debug(f"\t\t  cache hit: {cache_key}")

    return jsonify(doi_data)

@app.route('/api/v1/reviews/<path:doi>', methods=['GET', 'POST'])
@cache.cached()
def review_by_doi(doi: str):
    app.logger.info(f"review process for doi:{doi}")
    return jsonify(ask_neo(REVIEW_PROCESS_BY_DOI(), doi=doi))

@app.route('/api/v1/review/<path:doi>/<int:n>', methods=['GET', 'POST'])
@cache.cached()
def review_by_doi_n(doi: str, n: int):
    app.logger.info(f"review #{n} for doi:{doi}")
    j = ask_neo(REVIEW_PROCESS_BY_DOI(), doi=doi)
    try:
        j = j[0].get('review_process', []).get('reviews', [])
        j = j[n]
    except IndexError:
        j = {}
    return jsonify(j)

@app.route('/api/v1/response/<path:doi>', methods=['GET', 'POST'])
@cache.cached()
def response_by_doi(doi: str):
    app.logger.info(f"response for doi:{doi}")
    j = ask_neo(REVIEW_PROCESS_BY_DOI(), doi=doi)
    j = j[0].get('review_process', []).get('response', [])
    return jsonify(j)

@app.route('/api/v1/figure', methods=['GET', 'POST'])
def fig_by_doi_idx():
    app.logger.info(f"figure {request.args.get('position_idx')} from {request.args.get('doi')}")
    return jsonify(ask_neo(FIG_BY_DOI_IDX(), **request.args))

# <escape_lucene:search_string> triggers the use of converter.LuceneQueryConverter to quotes Lucene search strings
@app.route('/api/v1/search/<escape_lucene:search_string>', methods=['GET'])
# using <path:search_string> to match route when doi is searched
@app.route('/api/v1/search/<path:search_string>', methods=['GET'])
@cache.cached()
def search(search_string: str):
    search_string = search_string.strip()
    doi = DOI_REGEX.search(search_string)
    if doi is not None:
        doi = doi.group(0)
        app.logger.info(f"search doi: '{doi}'")
        results = ask_neo(SEARCH_DOI(), search_string=doi)
    else:
        app.logger.info(f"search text: '{search_string}'")
        results = ask_neo(LUCENE_SEARCH(), search_string=search_string)
    return jsonify(results)

@app.route('/api/v1/collection/covid19', methods=['GET', 'POST'])
@cache.cached()
def covid19():
    return jsonify(ask_neo(COVID19()))

def _fetch_refereed_preprints(reviewing_service, published_in, pagesize, page):
    app.logger.info(
        "refereed preprints from '%s' published in '%s' (page %s of size %s)",
        reviewing_service,
        published_in,
        page,
        pagesize,
    )
    return jsonify(
        ask_neo(
            REFEREED_PREPRINTS(),
            reviewing_service=reviewing_service,
            published_in=published_in,
            pagesize=int(pagesize),
            page=int(page),
        )
    )

PagingParameters = namedtuple('PagingParameters', ['page', 'pagesize'])
"""The container for the paging parameters, i.e. the page number and page size."""

DEFAULT_PAGESIZE = 20
DEFAULT_PAGE = 0
def paged(view_func):
    """
    Decorator that simplifies paged views.

    Reads parameters named `pagesize` and `page` from the request data and puts their
    values into a dict called `paging` on the request object. The values can be
    retrieved like so: `request.paging['pagesize'], request.paging['page']`.
    If one or both parameters are not present, default values are used.

    For GET requests, the parameters are retrieved from the query string.
    For POST requests with JSON request bodies, they are read from the request body.
    In any other case a warning is logged and the defaults are used.
    """

    param_name_pagesize = 'pagesize'
    param_name_page = 'page'

    @wraps(view_func)
    def inner(*args, **kwargs):
        if request.method == 'POST' and request.is_json:
            param_dict = request.json
        else:
            param_dict = request.args

        request.paging = PagingParameters(
            page=param_dict.get(param_name_page, DEFAULT_PAGE),
            pagesize=param_dict.get(param_name_pagesize, DEFAULT_PAGESIZE),
        )

        return view_func(*args, **kwargs)

    return inner

def paging_aware_cache_key():
    """
    Function to generate a paging-aware cache key.

    Use it as the `make_cache_key` parameter to @cached and in conjunction with @paged.

    Adds the paging parameters set by the @paged decorator to the cache key. Otherwise,
    requesting the same URL with different paging parameters returns the first, cached,
    result.
    """
    return f'view/{request.path}/page-{hash(request.paging)}'

DEFAULT_REVIEWING_SERVICE = ''
DEFAULT_PUBLISHER = ''

@app.route(
    '/api/v1/collection/refereed-preprints/<service_name:reviewing_service>/<published_in>',
    methods=['GET'],
)
@paged
@cache.cached(key_prefix=paging_aware_cache_key)
def refereed_preprints_get(reviewing_service, published_in):
    """
    Returns all refereed preprints that were reviewed by `reviewing_service` and
    published in `published_in`.

    Results are sorted by publication date of the preprint and paged. Use the query
    parameters `pagesize` and `page` to adjust the paging:
    /api/v1/collection/refereed-preprints/reviewcommons/elife?pagesize=100&page=3

    For more control, e.g. not filtering by reviewing service or publisher, use the POST
    version of this route.
    """
    return _fetch_refereed_preprints(
        reviewing_service=reviewing_service,
        published_in=published_in,
        pagesize=request.paging.pagesize,
        page=request.paging.page,
    )

@app.route('/api/v1/collection/refereed-preprints', methods=['GET'])
@cache.cached()
def refereed_preprints_get_all():
    """
    Returns all refereed preprints sorted by publication date of the preprint.

    For more control, e.g. not filtering by reviewing service or publisher, use the POST
    version of this route.
    """
    return _fetch_refereed_preprints(
        reviewing_service=DEFAULT_REVIEWING_SERVICE,
        published_in=DEFAULT_PUBLISHER,
        pagesize=10 ** 7,
        page=DEFAULT_PAGE,
    )

@app.route('/api/v1/collection/refereed-preprints', methods=['POST'])
@paged
def refereed_preprints_post():
    """
    Returns all refereed preprints. Results can be filtered by reviewing service and
    publisher.

    The request body must be JSON, or empty. If empty, all refereed preprints are returned.
    The results can be filtered by which reviewing service reviewed them (`reviewing_service`)
    and/or where they were published (`published_in`):
    `{ "reviewing_service": "review commons", "published_in": "Life Science Alliance" }`.

    Results are sorted by publication date of the preprint and paged. Use the parameters
    `pagesize` and `page` to adjust the paging: `{ "pagesize": 100, "page": 3 }`
    """
    if not request.is_json:
        abort(415) # unsupported media type
    return _fetch_refereed_preprints(
        reviewing_service=request.json.get('reviewing_service', DEFAULT_REVIEWING_SERVICE),
        published_in=request.json.get('published_in', DEFAULT_PUBLISHER),
        pagesize=request.paging.pagesize,
        page=request.paging.page,
    )

@app.route('/api/v1/collection/<subject>', methods=['GET', 'POST'])
@cache.cached()
def subject_collection(subject: str):
    app.logger.info(f"subject collection for subject: '{subject}'")
    return jsonify(ask_neo(SUBJECT_COLLECTIONS(), subject=subject))

@app.route('/api/v1/subjects', methods=['GET', 'POST'])
@cache.cached()
def subjects():
    app.logger.info(f"subjects names")
    return jsonify(ask_neo(COLLECTION_NAMES()))

@app.route('/api/v2/review_material/<int:node_id>', methods=['GET', 'POST'])
@cache.cached()
def review_material_by_id(node_id: int):
    app.logger.info(f"review material for id {node_id}")
    root = url_for('root', _external=True)
    j = ask_neo(REVIEW_MATERIAL_BY_ID(), node_id=node_id, root=root)
    return jsonify(j)

@app.route('/api/v2/review_process/<path:doi>', methods=['GET', 'POST'])
@app.route('/api/v2/docmap/<path:doi>', methods=['GET', 'POST'])
@cache.cached()
def docmap_semantic_doi(doi: str):
    app.logger.debug(f"docmap for id {doi}")
    root = url_for('root', _external=True)
    j = ask_neo(DOCMAP_BY_DOI(), doi=doi, root=root)
    return jsonify(j)

def do_paginated_docmap_query(query, page=0, page_size=100, **kwargs):
    if page < 0:
        abort(400) # pagination paramater must be 0 or greater

    offset = page * page_size
    root = url_for('root', _external=True)

    result = ask_neo(
        query,
        root=root,
        offset=offset,
        page_size=page_size,
        **kwargs,
    )

    return jsonify(result)

@app.route('/api/v2/<service_name:reviewing_service>/docmap/<start_date>/<end_date>/<int:pagination>', methods=['GET', 'POST'])
def docmaps_from_revservice_in_interval(reviewing_service: str, start_date: str, end_date: str, pagination: int):
    app.logger.info(f"Getting docmaps for reviewing service \"{reviewing_service}\" from {start_date} to {end_date}, page {pagination}")
    return do_paginated_docmap_query(
        DOCMAPS_FROM_REVSERVICE_IN_INTERVAL(),
        reviewing_service=reviewing_service,
        start_date=start_date,
        end_date=end_date,
        page=pagination,
    )

@app.route('/api/v2/docmap/<start_date>/<end_date>/<int:pagination>', methods=['GET', 'POST'])
def docmaps_in_interval(start_date: str, end_date: str, pagination: int):
    app.logger.info(f"Getting docmaps from {start_date} to {end_date}, page {pagination}")
    return do_paginated_docmap_query(
        DOCMAPS_IN_INTERVAL(),
        start_date=start_date,
        end_date=end_date,
        page=pagination,
    )

@app.route('/api/v2/<service_name:reviewing_service>/docmap/<int:days>d/<int:pagination>', methods=['GET', 'POST'])
def docmaps_from_revservice_in_last_days(reviewing_service: str, days: int, pagination: int):
    app.logger.info(f"Getting docmaps for reviewing service \"{reviewing_service}\" with reviews in the last {days} days, page {pagination}")
    # The Docmap query is exclusive for the end of the interval. We want any reviews published today so add a day here.
    end_date = date.today() + timedelta(days=1)
    # With this start date we get any reviews published today and in the last n-1 days, so reviews from a total of n days.
    start_date = end_date - timedelta(days=days)
    return do_paginated_docmap_query(
        DOCMAPS_FROM_REVSERVICE_IN_INTERVAL(),
        reviewing_service=reviewing_service,
        start_date=str(start_date),
        end_date=str(end_date),
        page=pagination,
    )

# @app.route('/api/v2/<reviewing_service>/docmap/<int:n_most_recent>/<int:pagination>', methods=['GET', 'POST'])

# date of docmap creation and date of docmap update. filter on publishing data or on docmap creation date.


import src.main as rs
import matplotlib.pyplot as plt
import pandas as pd
import ee

# Definitions

jan19 = '2019-01-01'
march19 = '2019-03-01'
may19 = '2019-05-01'

jan21 = '2021-01-01'

lat_n=-34.52
lat_s=-34.73
lon_w=-58.56
lon_e=-58.33

reds = [ee.Reducer.mean(), ee.Reducer.min()] # the spatial reducers we'll be using
names = ['NO2_trop_mean','NO2_trop_min'] # their names

arg_shapefile = "data/arg/departamento.shp"

def test_time_series(start, end, roi):

    df = rs.time_series_df(roi,start,end,filename='raw.csv',reducers = reds, red_names = names)
    df_daily = rs.ts_dailydf(df, filename= 'daily.csv')
    df_weekly = rs.ts_weeklydf(df, filename= 'weekly.csv')
    df_monthly = rs.ts_monthlydf(df, filename= 'monthly.csv')

    return df, df_daily, df_weekly, df_monthly

def test_tseries_median(df):
    median_daily = rs.ts_dailydf(df, statistic = 'median')
    median_weekly = rs.ts_weeklydf(df, statistic = 'median')
    median_monthly = rs.ts_monthlydf(df, statistic = 'median')

    return median_daily, median_weekly, median_monthly

print("Create geometry of Buenos Aires")
ba_roi = rs.geometry_rectangle(lon_w,lat_s,lon_e,lat_n)

# Basics

print("Obtain dataframes with NO2 time series. Aggregate using mean")
df, df_d, df_w, df_m = test_time_series(march19, may19, ba_roi)

print("Show and save plots for these series")
fig, ax = rs.plot_series(df_d,show = True) #save with default name: series.png
rs.plot_series(df_w, filename = 'weekly.png', show = True)
rs.plot_series(df_m, filename = 'monthy.png', show = True)

print("Change title to matplotlib object and save again")
fig.suptitle("Daily series of tropospheric NO2")
fig.savefig('daily_again.png',bbox_inches='tight',dpi=500)

print("Recalculate NO2 time series, now using median to aggregate")
median_d, median_w, median_m = test_tseries_median(df)

print("Show or save plots for these series")
fig, ax = rs.plot_series(median_d,show = True) #show and save with default name: series.png
rs.plot_series(median_w, filename = 'median_weekly.png') #save, don't show
rs.plot_series(median_m, show = True, save = False) #show, don't save

print("Show autocorrelogram for daily series")
rs.plot_autocorr(df_d, lags = 22, show = True, save = False)

# Long case

print("Get series for 2019 and 2020")
long_df = rs.time_series_df(ba_roi,jan19,jan21,filename='long_raw.csv',reducers = reds, red_names = names)
long_monthly = rs.ts_monthlydf(long_df, filename= 'long_monthly.csv')

print("Load monthly series from csv and plot a bar plot comparing years")
df_monthly = pd.read_csv('long_monthly.csv')
rs.barplot_year_cmp(df_monthly, 2019, 2020, show = True)

var = rs.interanual_variation(df_monthly, 2019, 2020, month_num = 4)
print("Interannual variation for april 2019-2020: ",var)

# Spatial data

print("Show and save plots for spatial distribution of NO2")
values, lon, lat = rs.space_data_meshgrid(ba_roi, jan19, may19, export = False)
map_fig, _ = rs.plot_map(values, lon, lat, arg_shapefile, show=True)
map_fig.suptitle("Concentrations of NO2 in Buenos Aires")
map_fig.savefig("map.png")