"""This is Slate's Linear Algebra Compiler. This module is
responsible for generating C++ kernel functions representing
symbolic linear algebra expressions written in Slate.

This linear algebra compiler uses both Firedrake's form compiler,
the Two-Stage Form Compiler (TSFC) and COFFEE's kernel abstract
syntax tree (AST) optimizer. TSFC provides this compiler with
appropriate kernel functions (in C) for evaluating integral
expressions (finite element variational forms written in UFL).
COFFEE's AST base helps with the construction of code blocks
throughout the kernel returned by: `compile_expression`.

The Eigen C++ library (http://eigen.tuxfamily.org/) is required, as
all low-level numerical linear algebra operations are performed using
this templated function library.
"""
from coffee import base as ast

from firedrake.constant import Constant
from firedrake.tsfc_interface import SplitKernel, KernelInfo
from firedrake.slate.slac.kernel_builder import LocalKernelBuilder
from firedrake import op2

from itertools import chain

from pyop2.utils import get_petsc_dir
from pyop2.datatypes import as_cstr

from tsfc.parameters import SCALAR_TYPE

import firedrake.slate.slate as slate
import numpy as np

__all__ = ['compile_expression']

PETSC_DIR = get_petsc_dir()

cell_to_facets_dtype = np.dtype(np.int8)

def compile_expression(slate_expr, tsfc_parameters=None):
    """Takes a Slate expression `slate_expr` and returns the appropriate
    :class:`firedrake.op2.Kernel` object representing the Slate expression.

    :arg slate_expr: a :class:'TensorBase' expression.
    :arg tsfc_parameters: an optional `dict` of form compiler parameters to
                          be passed onto TSFC during the compilation of
                          ufl forms.

    Returns: A `tuple` containing a `SplitKernel(idx, kinfo)`
    """
    if not isinstance(slate_expr, slate.TensorBase):
        raise ValueError("Expecting a `TensorBase` object, not %s" % type(slate_expr))

    # TODO: Get PyOP2 to write into mixed dats
    if slate_expr.is_mixed:
        raise NotImplementedError("Compiling mixed slate expressions")

    if len(slate_expr.ufl_domains()) > 1:
        raise NotImplementedError("Multiple domains not implemented.")

    # If the expression has already been symbolically compiled, then
    # simply reuse the produced kernel.
    if slate_expr._metakernel_cache is not None:
        return slate_expr._metakernel_cache

    # Create a builder for the Slate expression
    builder = LocalKernelBuilder(expression=slate_expr,
                                 tsfc_parameters=tsfc_parameters)

    # Keep track of declared temporaries
    declared_temps = {}
    statements = []

    # Declare terminal tensor temporaries
    terminal_declarations = terminal_temporaries(builder, declared_temps)
    statements.extend(terminal_declarations)

    # Generate assembly calls for tensor assembly
    subkernel_calls = tensor_assembly_calls(builder)
    statements.extend(subkernel_calls)

    # Create coefficient temporaries if necessary
    if builder.action_coefficients:
        coefficient_temps = coefficient_temporaries(builder, declared_temps)
        statements.extend(coefficient_temps)

    # Create auxiliary temporaries if necessary
    if builder.aux_exprs:
        aux_temps = auxiliary_temporaries(builder, declared_temps)
        statements.extend(aux_temps)

    # Generate the kernel information with complete AST
    kinfo = generate_kernel_ast(builder, statements, declared_temps)

    # Cache the resulting kernel
    idx = tuple([0]*slate_expr.rank)
    kernel = (SplitKernel(idx, kinfo),)
    slate_expr._metakernel_cache = kernel

    return kernel

def generate_kernel_ast(builder, statements, declared_temps):
    """Glues together the complete AST for the Slate expression
    contained in the :class:`LocalKernelBuilder`.

    :arg builder: The :class:`LocalKernelBuilder` containing
                  all relevant expression information.
    :arg statements: A list of COFFEE objects containing all
                     assembly calls and temporary declarations.
    :arg declared_temps: A `dict` containing all previously
                         declared temporaries.

    Return: A `KernelInfo` object describing the complete AST.
    """
    slate_expr = builder.expression
    if slate_expr.rank == 0:
        # Scalars are treated as 1x1 MatrixBase objects
        shape = (1,)
    else:
        shape = slate_expr.shape

    # Now we create the result statement by declaring its eigen type and
    # using Eigen::Map to move between Eigen and C data structs.
    statements.append(ast.FlatBlock("/* Map eigen tensor into C struct */\n"))
    result_sym = ast.Symbol("T%d" % len(declared_temps))
    result_data_sym = ast.Symbol("A%d" % len(declared_temps))
    result_type = "Eigen::Map<%s >" % eigen_matrixbase_type(shape)
    result = ast.Decl(SCALAR_TYPE, ast.Symbol(result_data_sym, shape))
    result_statement = ast.FlatBlock("%s %s((%s *)%s);\n" % (result_type,
                                                             result_sym,
                                                             SCALAR_TYPE,
                                                             result_data_sym))
    statements.append(result_statement)

    # Generate the complete c++ string performing the linear algebra operations
    # on Eigen matrices/vectors
    statements.append(ast.FlatBlock("/* Linear algebra expression */\n"))
    cpp_string = ast.FlatBlock(metaphrase_slate_to_cpp(slate_expr,
                                                       declared_temps))
    statements.append(ast.Incr(result_sym, cpp_string))

    # Generate arguments for the macro kernel
    args = [result, ast.Decl("%s **" % SCALAR_TYPE, builder.coord_sym)]

    # Orientation information
    if builder.oriented:
        args.append(ast.Decl("int **", builder.cell_orientations_sym))

    # Coefficient information
    expr_coeffs = slate_expr.coefficients()
    for c in expr_coeffs:
        if isinstance(c, Constant):
            ctype = "%s *" % SCALAR_TYPE
        else:
            ctype = "%s **" % SCALAR_TYPE
        args.extend([ast.Decl(ctype, csym) for csym in builder.coefficient(c)])

    # Facet information
    if builder.needs_cell_facets:
        args.append(ast.Decl("%s *" % as_cstr(cell_to_facets_dtype),
                             builder.cell_facet_sym))

    # NOTE: We need to be careful about the ordering here. Mesh layers are
    # added as the final argument to the kernel.
    if builder.needs_mesh_layers:
        args.append(ast.Decl("int", builder.mesh_layer_sym))

    # Macro kernel
    macro_kernel_name = "compile_slate"
    stmts = ast.Block(statements)
    macro_kernel = ast.FunDecl("void", macro_kernel_name, args,
                               stmts, pred=["static", "inline"])

    # Construct the final ast
    kernel_ast = ast.Node(builder.templated_subkernels + [macro_kernel])

    # Now we wrap up the kernel ast as a PyOP2 kernel and include the
    # Eigen header files
    include_dirs = builder.include_dirs
    include_dirs.extend(["%s/include/eigen3/" % d for d in PETSC_DIR])
    op2kernel = op2.Kernel(kernel_ast,
                           macro_kernel_name,
                           cpp=True,
                           include_dirs=include_dirs,
                           headers=['#include <Eigen/Dense>',
                                    '#define restrict __restrict'])

    # Send back a "TSFC-like" SplitKernel object with an
    # index and KernelInfo
    kinfo = KernelInfo(kernel=op2kernel,
                       integral_type=builder.integral_type,
                       oriented=builder.oriented,
                       subdomain_id="otherwise",
                       domain_number=0,
                       coefficient_map=tuple(range(len(expr_coeffs))),
                       needs_cell_facets=builder.needs_cell_facets,
                       pass_layer_arg=builder.needs_mesh_layers)

    return kinfo

def auxiliary_temporaries(builder, declared_temps):
    """Generates statements for assigning auxiliary temporaries
    for nodes in an expression with "high" reference count.
    Expressions which require additional temporaries are provided
    by the :class:`LocalKernelBuilder`.

    :arg builder: The :class:`LocalKernelBuilder` containing
                  all relevant expression information.
    :arg declared_temps: A `dict` containing all previously
                         declared temporaries. This dictionary
                         is updated as auxiliary expressions
                         are assigned temporaries.
    """
    statements = [ast.FlatBlock("/* Auxiliary temporaries */\n")]
    results = [ast.FlatBlock("/* Assign auxiliary temps */\n")]
    for exp in builder.aux_exprs:
        if exp not in declared_temps:
            t = ast.Symbol("auxT%d" % len(declared_temps))
            result = metaphrase_slate_to_cpp(exp, declared_temps)
            tensor_type = eigen_matrixbase_type(shape=exp.shape)
            statements.append(ast.Decl(tensor_type, t))
            statements.append(ast.FlatBlock("%s.setZero();\n" % t))
            results.append(ast.Assign(t, result))
            declared_temps[exp] = t

    statements.extend(results)

    return statements

def coefficient_temporaries(builder, declared_temps):
    """Generates coefficient temporary statements for assigning
    coefficients to vector temporaries.

    :arg builder: The :class:`LocalKernelBuilder` containing
                  all relevant expression information.
    :arg declared_temps: A `dict` keeping track of all declared
                         temporaries. This dictionary is updated
                         as coefficients are assigned temporaries.

    Action computations require creating coefficient temporaries to
    compute the matrix-vector product. The temporaries are created by
    inspecting the function space of the coefficient to compute node
    and dof extents. The coefficient is then assigned values by looping
    over both the node extent and dof extent (double FOR-loop). A double
    FOR-loop is needed for each function space (if the function space is
    mixed, then a loop will be constructed for each component space).
    The general structure of each coefficient loop will be:

         FOR (i1=0; i1<node_extent; i1++):
             FOR (j1=0; j1<dof_extent; j1++):
                 wT0[offset + (dof_extent * i1) + j1] = w_0_0[i1][j1]
                 wT1[offset + (dof_extent * i1) + j1] = w_1_0[i1][j1]
                 .
                 .
                 .

    where wT0, wT1, ... are temporaries for coefficients sharing the
    same node and dof extents. The offset is computed based on whether
    the function space is mixed. The offset is always 0 for non-mixed
    coefficients. If the coefficient is mixed, then the offset is
    incremented by the total number of nodal unknowns associated with
    the component spaces of the mixed space.
    """
    statements = [ast.FlatBlock("/* Coefficient temporaries */\n")]
    i_sym = ast.Symbol("i1")
    j_sym = ast.Symbol("j1")
    loops = [ast.FlatBlock("/* Loops for coefficient temps */\n")]
    for (nodes, dofs), cinfo_list in builder.action_coefficients.items():
        # Collect all coefficients which share the same node/dof extent
        assignments = []
        for cinfo in cinfo_list:
            fs_i = cinfo.space_index
            offset = cinfo.offset_index
            c_shape = cinfo.shape
            actee = cinfo.coefficient

            if actee not in declared_temps:
                # Declare and initialize coefficient temporary
                c_type = eigen_matrixbase_type(shape=c_shape)
                t = ast.Symbol("wT%d" % len(declared_temps))
                statements.append(ast.Decl(c_type, t))
                statements.append(ast.FlatBlock("%s.setZero();\n" % t))
                declared_temps[actee] = t

            # Assigning coefficient values into temporary
            coeff_sym = ast.Symbol(builder.coefficient(actee)[fs_i],
                                   rank=(i_sym, j_sym))
            index = ast.Sum(offset,
                            ast.Sum(ast.Prod(dofs, i_sym), j_sym))
            coeff_temp = ast.Symbol(t, rank=(index,))
            assignments.append(ast.Assign(coeff_temp, coeff_sym))

        # Inner-loop running over dof extent
        inner_loop = ast.For(ast.Decl("unsigned int", j_sym, init=0),
                             ast.Less(j_sym, dofs),
                             ast.Incr(j_sym, 1),
                             assignments)

        # Outer-loop running over node extent
        loop = ast.For(ast.Decl("unsigned int", i_sym, init=0),
                       ast.Less(i_sym, nodes),
                       ast.Incr(i_sym, 1),
                       inner_loop)

        loops.append(loop)

    statements.extend(loops)

    return statements

def tensor_assembly_calls(builder):
    """Generates a block of statements for assembling the local
    finite element tensors.

    :arg builder: The :class:`LocalKernelBuilder` containing
                  all relevant expression information and
                  assembly calls.
    """
    statements = [ast.FlatBlock("/* Assemble local tensors */\n")]

    # Cell integrals are straightforward. Just splat them out.
    statements.extend(builder.assembly_calls["cell"])

    if builder.needs_cell_facets:
        # The for-loop will have the general structure:
        #
        #    FOR (facet=0; facet<num_facets; facet++):
        #        IF (facet is interior):
        #            *interior calls
        #        ELSE IF (facet is exterior):
        #            *exterior calls
        #
        # If only interior (exterior) facets are present,
        # then only a single IF-statement checking for interior
        # (exterior) facets will be present within the loop. The
        # cell facets are labelled `1` for interior, and `0` for
        # exterior.

        statements.append(ast.FlatBlock("/* Loop over cell facets */\n"))
        int_calls = list(chain(*[builder.assembly_calls[it_type]
                                 for it_type in ("interior_facet",
                                                 "interior_facet_vert")]))
        ext_calls = list(chain(*[builder.assembly_calls[it_type]
                                 for it_type in ("exterior_facet",
                                                 "exterior_facet_vert")]))

        # Compute the number of facets to loop over
        domain = builder.expression.ufl_domain()
        if domain.cell_set._extruded:
            num_facets = domain.ufl_cell()._cells[0].num_facets()
        else:
            num_facets = domain.ufl_cell().num_facets()

        if_ext = ast.Eq(ast.Symbol(builder.cell_facet_sym,
                                   rank=(builder.it_sym,)), 0)
        if_int = ast.Eq(ast.Symbol(builder.cell_facet_sym,
                                   rank=(builder.it_sym,)), 1)
        body = []
        if ext_calls:
            body.append(ast.If(if_ext, (ast.Block(ext_calls,
                                                  open_scope=True),)))
        if int_calls:
            body.append(ast.If(if_int, (ast.Block(int_calls,
                                                  open_scope=True),)))

        statements.append(ast.For(ast.Decl("unsigned int",
                                           builder.it_sym, init=0),
                                  ast.Less(builder.it_sym, num_facets),
                                  ast.Incr(builder.it_sym, 1), body))

    if builder.needs_mesh_layers:
        # In the presence of interior horizontal facet calls, an
        # IF-ELIF-ELSE block is generated using the mesh levels
        # as conditions for which calls are needed:
        #
        #    IF (layer == bottom_layer):
        #        *bottom calls
        #    ELSE IF (layer == top_layer):
        #        *top calls
        #    ELSE:
        #        *top calls
        #        *bottom calls
        #
        # Any extruded top or bottom calls for extruded facets are
        # included within the appropriate mesh-level IF-blocks. If
        # no interior horizontal facet calls are present, then
        # standard IF-blocks are generated for exterior top/bottom
        # facet calls when appropriate:
        #
        #    IF (layer == bottom_layer):
        #        *bottom calls
        #
        #    IF (layer == top_layer):
        #        *top calls
        #
        # The mesh level is an integer provided as a macro kernel
        # argument.

        # FIXME: No variable layers assumption
        statements.append(ast.FlatBlock("/* Mesh levels: */\n"))
        num_layers = builder.expression.ufl_domain().topological.layers - 1
        int_top = builder.assembly_calls["interior_facet_horiz_top"]
        int_btm = builder.assembly_calls["interior_facet_horiz_bottom"]
        ext_top = builder.assembly_calls["exterior_facet_top"]
        ext_btm = builder.assembly_calls["exterior_facet_bottom"]

        bottom = ast.Block(int_top + ext_btm, open_scope=True)
        top = ast.Block(int_btm + ext_top, open_scope=True)
        rest = ast.Block(int_btm + int_top, open_scope=True)
        statements.append(ast.If(ast.Eq(builder.mesh_layer_sym, 0),
                                 (bottom,
                                  ast.If(ast.Eq(builder.mesh_layer_sym,
                                                num_layers - 1),
                                         (top, rest)))))

    return statements

def terminal_temporaries(builder, declared_temps):
    """Generates statements for assigning auxiliary temporaries
    for nodes in an expression with "high" reference count.
    Expressions which require additional temporaries are provided
    by the :class:`LocalKernelBuilder`.

    :arg builder: The :class:`LocalKernelBuilder` containing
                  all relevant expression information.
    :arg declared_temps: A `dict` keeping track of all declared
                         temporaries. This dictionary is updated
                         as terminal tensors are assigned temporaries.
    """
    statements = [ast.FlatBlock("/* Declare and initialize */\n")]
    for exp in builder.temps:
        t = builder.temps[exp]
        statements.append(ast.Decl(eigen_matrixbase_type(exp.shape), t))
        statements.append(ast.FlatBlock("%s.setZero();\n" % t))
        declared_temps[exp] = t

    return statements

def parenthesize(arg, prec=None, parent=None):
    """Parenthesizes an expression."""
    if prec is None or parent is None or prec >= parent:
        return arg
    return "(%s)" % arg

def metaphrase_slate_to_cpp(expr, temps, prec=None):
    """Translates a Slate expression into its equivalent representation in
    the Eigen C++ syntax.

    :arg expr: a :class:`slate.TensorBase` expression.
    :arg temps: a `dict` of temporaries which map a given expression to its
                corresponding representation as a `coffee.Symbol` object.
    :arg prec: an argument dictating the order of precedence in the linear
               algebra operations. This ensures that parentheticals are placed
               appropriately and the order in which linear algebra operations
               are performed are correct.

    Returns
        This function returns a `string` which represents the C/C++ code
        representation of the `slate.TensorBase` expr.
    """
    # If the tensor is terminal, it has already been declared.
    # Coefficients in action expressions will have been declared by now,
    # as well as any other nodes with high reference count.
    if expr in temps:
        return temps[expr].gencode()

    elif isinstance(expr, slate.Transpose):
        tensor, = expr.operands
        return "(%s).transpose()" % metaphrase_slate_to_cpp(tensor, temps)

    elif isinstance(expr, slate.Inverse):
        tensor, = expr.operands
        return "(%s).inverse()" % metaphrase_slate_to_cpp(tensor, temps)

    elif isinstance(expr, slate.Negative):
        tensor, = expr.operands
        result = "-%s" % metaphrase_slate_to_cpp(tensor, temps, expr.prec)
        return parenthesize(result, expr.prec, prec)

    elif isinstance(expr, (slate.Add, slate.Sub, slate.Mul)):
        op = {slate.Add: '+',
              slate.Sub: '-',
              slate.Mul: '*'}[type(expr)]
        A, B = expr.operands
        result = "%s %s %s" % (metaphrase_slate_to_cpp(A, temps, expr.prec),
                               op,
                               metaphrase_slate_to_cpp(B, temps, expr.prec))

        return parenthesize(result, expr.prec, prec)

    elif isinstance(expr, slate.Action):
        tensor, = expr.operands
        c, = expr.actee
        result = "(%s) * %s" % (metaphrase_slate_to_cpp(tensor,
                                                        temps,
                                                        expr.prec), temps[c])
        return parenthesize(result, expr.prec, prec)

    else:
        raise NotImplementedError("Type %s not supported.", type(expr))

def eigen_matrixbase_type(shape):
    """Returns the Eigen::Matrix declaration of the tensor.

    :arg shape: a tuple of integers the denote the shape of the
                :class:`slate.TensorBase` object.

    Returns: Returns a string indicating the appropriate declaration of the
             `slate.TensorBase` object in the appropriate Eigen C++ template
             library syntax.
    """
    if len(shape) == 0:
        rows = 1
        cols = 1
    elif len(shape) == 1:
        rows = shape[0]
        cols = 1
    else:
        if not len(shape) == 2:
            raise NotImplementedError(
                "%d-rank tensors are not supported." % len(shape)
            )
        rows = shape[0]
        cols = shape[1]
    if cols != 1:
        order = ", Eigen::RowMajor"
    else:
        order = ""

    return "Eigen::Matrix<double, %d, %d%s>" % (rows, cols, order)


from glob import glob
from os import makedirs
from shutil import copy2

from tqdm import tqdm

SINGER = 'mixed'
RELEASE_DIR = 'release/mixed_---'
PATH_QUESTION = 'conf/jp_qst001_nnsvs_simple_4-4_mix.hed'
NAME_EXPERIMENT = 'simple_4-4_mix'

def copy_question(path_question, release_dir):
    """
    hedファイル(question)をコピー
    """
    makedirs(f'{release_dir}/conf', exist_ok=True)
    print('copying question')
    copy2(path_question, f'{release_dir}/{path_question}')

def copy_scaler(singer, release_dir):
    """
    dumpフォルダにあるファイルをコピー
    """
    makedirs(f'{release_dir}/dump/{singer}/norm', exist_ok=True)
    list_path_scaler = glob(f'dump/{singer}/norm/*_scaler.joblib')

    print('copying scaler')
    for path_scaler in tqdm(list_path_scaler):
        copy2(path_scaler, f'{release_dir}/{path_scaler}')

def copy_model(singer, name_exp, release_dir):
    """
    name_exp: 試験のID
    """
    name_exp = singer + '_' + name_exp
    makedirs(f'{release_dir}/exp/{name_exp}/acoustic', exist_ok=True)
    makedirs(f'{release_dir}/exp/{name_exp}/duration', exist_ok=True)
    makedirs(f'{release_dir}/exp/{name_exp}/timelag', exist_ok=True)
    list_path_model = glob(f'exp/{name_exp}/*/best_loss.pth')
    list_path_model += glob(f'exp/{name_exp}/*/latest.pth')
    list_path_model += glob(f'exp/{name_exp}/*/model.yaml')

    print('copying model')
    for path_model in tqdm(list_path_model):
        copy2(path_model, f'{release_dir}/{path_model}')

def main():
    """
    各種ファイルをコピーする
    """
    copy_question(PATH_QUESTION, RELEASE_DIR)
    copy_scaler(SINGER, RELEASE_DIR)
    copy_model(SINGER, NAME_EXPERIMENT, RELEASE_DIR)

if __name__ == '__main__':
    main()


from django.views.generic.simple import direct_to_template
from django.db.models import get_app, get_models, get_model
from django.http import HttpResponse
from django.template.loader import render_to_string
from django.forms import ModelForm
from django.forms.models import modelform_factory
from django.forms import ModelForm, Textarea, TextInput,HiddenInput

def index(request,app):
        if request.is_ajax():
            model = get_model(app, request.GET.get('model'))
            for field in model._meta.fields:
                if field.get_internal_type()=='DateField':
                    wdg={field.get_attname(): TextInput(attrs={'class': 'datepicker1'})}
                else:
                    wdg=None
            form=modelform_factory(model,widgets=wdg)#{'date_joined': TextInput(attrs={'class': 'datepicker1'})})
            id=None
            f = form(request.POST or None,request.FILES or None,instance=id and model.objects.get(id=id))
            if request.method == 'POST' and f.is_valid() :
                f.save()
            data=[]
            data = model.objects.all().values_list()
            fields = [field.verbose_name for field in model._meta.fields]
            html = render_to_string('table.html', {'data': data,'fields':fields,'form':form,'model':model,'modelnm':model.__name__})
            return HttpResponse(html)
        else:
            list_models = []
            for model in get_models(get_app(app)):
                list_models.append([model.__name__, model._meta.verbose_name_plural])
            return direct_to_template(request, 'index.html', {'models': list_models,})


import os
import sys
import yt_dlp
from tqdm import tqdm

def download_videos(links_file):
    if not os.path.exists(links_file):
        print("Error: The links file '{}' does not exist.".format(links_file))
        return

    ydl_opts = {
        'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
        'outtmpl': '%(autonumber)s - %(title)s.%(ext)s',
        'merge_output_format': 'mp4',
    }

    with open(links_file, 'r') as f:
        video_links = f.read().splitlines()

    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
        for link in video_links:
            try:
                info_dict = ydl.extract_info(link, download=True)
                print("\n{}: Download completed!".format(info_dict['title']))
            except Exception as e:
                print("\nError downloading video: {}".format(e))
                continue

if __name__ == "__main__":
    links_file = "links.txt"
    download_videos(links_file)


import os
import pytest
import requests_mock
from newsApi.models.request import EverythingRequestModel, Language, TopHeadlinesRequestModel, SourcesRequestModel
from newsApi.models.response import EverythingResponseModel, TopHeadlinesResponseModel, SourcesResponseModel
from newsApi.service import NewsAPIService
@pytest.fixture
def news_api_service():
    return NewsAPIService(os.environ.get('NEWS_API_KEY'))

def test_everything(news_api_service):
    request_model = EverythingRequestModel(q="test")
    with requests_mock.Mocker() as m:
        m.get('https://newsapi.org/v2/everything', json={"status": "ok", "totalResults": 10, "articles": []})
        response = news_api_service.everything(request_model)
    assert isinstance(response, EverythingResponseModel)
    assert response.status == 'ok'
    assert response.totalResults == 10

def test_top_headlines(news_api_service):
    request_model = TopHeadlinesRequestModel(q="test")
    with requests_mock.Mocker() as m:
        m.get('https://newsapi.org/v2/top-headlines', json={"status": "ok", "totalResults": 10, "articles": []})
        response = news_api_service.top_headlines(request_model)
    assert isinstance(response, TopHeadlinesResponseModel)
    assert response.status == 'ok'
    assert response.totalResults == 10

def test_sources(news_api_service):
    request_model = SourcesRequestModel(language=Language.EN)
    with requests_mock.Mocker() as m:
        m.get('https://newsapi.org/v2/sources', json={"status": "ok", "sources": []})
        response = news_api_service.sources(request_model)
    assert isinstance(response, SourcesResponseModel)
    assert response.status == 'ok'


import requests
import json
from pprint import pprint

access_token='<put your access token here>'
page_id='<put your page id here>'

url='https://graph.facebook.com/v2.0/'+page_id+'?feed&access_token='+access_token
r = requests.get(url)

try:
	response_json = json.loads(r.text)
except (ValueError, KeyError, TypeError):
	print("JSON error")

pprint(response_json)


import time
import json
import math
import csv
import serial # conda install pyserial
import sys
import glob
# pygame needs python 3.6, not available for 3.7
import pygame # conda install -c cogsci pygame; maybe because it only is supplied for earlier python, might need conda install -c evindunn pygame ; sudo apt-get install libsdl-ttf2.0-0
import pygame.joystick as joystick # https://www.pygame.org/docs/ref/joystick.html
from datetime import datetime
# our imports
import kbhit
from pendulum import Pendulum

POLOLU_MOTOR            = False # set true to set options for this motor, which has opposite sign for set_motor TODO needs fixing in firmware or wiring of motor

SERIAL_PORT             = "COM4" #"/dev/ttyUSB0" # might move if other devices plugged in
SERIAL_BAUD             = 230400  # default 230400, in firmware.   Alternatives if compiled and supported by USB serial intervace are are 115200, 128000, 153600, 230400, 460800, 921600, 1500000, 2000000
PRINT_PERIOD_MS         = 100  # shows state every this many ms
CONTROL_PERIOD_MS       = 5
CALIBRATE               = False #False # important to calibrate if running standalone to avoid motor burnout because limits are determined during this calibration
MOTOR_FULL_SCALE        =  7199 # 7199 # with pololu motor and scaling in firmware #7199 # with original motor
MOTOR_MAX_PWM           = int(round(0.95 * MOTOR_FULL_SCALE))

JOYSTICK_SCALING        = MOTOR_MAX_PWM # how much joystick value -1:1 should be scaled to motor command
JOYSTICK_DEADZONE       = 0.05 # deadzone around joystick neutral position that stick is ignored

ANGLE_TARGET            = 3129 # 3383  # adjust to exactly vertical angle value, read by inspecting angle output
ANGLE_CTRL_PERIOD_MS    = 5         # Must be a multiple of CONTROL_PERIOD_MS
ANGLE_AVG_LENGTH        = 4 # adc routine in firmware reads ADC this many times quickly in succession to reduce noise
ANGLE_SMOOTHING         = 1      # 1.0 turns off smoothing
ANGLE_KP                = 400
ANGLE_KD                = 400

POSITION_TARGET         = 0 # 1200
POSITION_CTRL_PERIOD_MS = 25        # Must be a multiple of CONTROL_PERIOD_MS
POSITION_SMOOTHING      = 1       # 1.0 turns off smoothing
POSITION_KP             = 20
POSITION_KD             = 300

def serial_ports(): # from https://stackoverflow.com/questions/12090503/listing-available-com-ports-with-python
    """ Lists serial port names

        :raises EnvironmentError:
            On unsupported or unknown platforms
        :returns:
            A list of the serial ports available on the system
    """
    if sys.platform.startswith('win'):
        ports = ['COM%s' % (i + 1) for i in range(256)]
    elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
        # this excludes your current terminal "/dev/tty"
        # if cannot open, check permissions
        ports = glob.glob('/dev/ttyUSB[0-9]*')
    elif sys.platform.startswith('darwin'):
        ports = glob.glob('/dev/tty.*')
    else:
        raise EnvironmentError('Unsupported platform')

    result = []
    for port in ports:
        try:
            s = serial.Serial(port)
            s.close()
            result.append(port)
        except (OSError, serial.SerialException):
            pass
    return result

def saveparams():
    print("\nSaving parameters")
    p={}
    p['ANGLE_TARGET']=ANGLE_TARGET
    p['ANGLE_KP']=ANGLE_KP
    p['ANGLE_KD']=ANGLE_KD
    p['POSITION_TARGET']=POSITION_TARGET
    p['POSITION_KP']=POSITION_KP
    p['POSITION_KD']=POSITION_KD
    p['ANGLE_SMOOTHING']=ANGLE_SMOOTHING
    p['POSITION_SMOOTHING']=POSITION_SMOOTHING
    with open('control.json','w') as f:
        json.dump(p,f)

def loadparams():
    print("\nLoading parameters")
    f=open('control.json')
    try:
        p=json.load(f)
        global ANGLE_TARGET, ANGLE_KP,ANGLE_KD,POSITION_TARGET,POSITION_KP,POSITION_KD,ANGLE_SMOOTHING,POSITION_SMOOTHING
        ANGLE_TARGET=p['ANGLE_TARGET']
        ANGLE_KP=p['ANGLE_KP']
        ANGLE_KD=p['ANGLE_KD']
        POSITION_TARGET=p['POSITION_TARGET']
        POSITION_KP=p['POSITION_KP']
        POSITION_KD=p['POSITION_KD']
        ANGLE_SMOOTHING=p['ANGLE_SMOOTHING']
        POSITION_SMOOTHING=p['POSITION_SMOOTHING']
    except:
        print("something went wrong loading parameters")
    printparams()

def help():
    print("\n***********************************")
    print("keystroke commands")
    print("ESC quit")
    print("k toggle control on/off (initially off)")
    print("K trigger motor position calibration")
    print("=/- increase/decrease angle target")
    print("[/] increase/decrease position target")
    print("w/q angle proportional gain")
    print("s/a angle derivative gain")
    print("z/x angle smoothing")
    print("r/e position proportional gain")
    print("f/d position derivative gain")
    print("c/v position smoothing")
    print("l toggle logging data")
    print("S/L Save/Load param values from disk")
    print("D Toggle dance mode")
    print("***********************************")

def printparams():
    print("\nAngle PD Control Parameters")
    print("    Set point       {0}".format(ANGLE_TARGET))
    print("    Average Length  {0}".format(ANGLE_AVG_LENGTH))
    print("    Smoothing       {0:.2f}".format(ANGLE_SMOOTHING))
    print("    P Gain          {0:.2f}".format(ANGLE_KP))
    print("    D Gain          {0:.2f}".format(ANGLE_KD))

    print("Position PD Control Parameters")
    print("    Set point       {0}".format(POSITION_TARGET))
    print("    Control Period  {0} ms".format(POSITION_CTRL_PERIOD_MS))
    print("    Smoothing       {0:.2f}".format(POSITION_SMOOTHING))
    print("    P Gain          {0:.2f}".format(POSITION_KP))
    print("    D Gain          {0:.2f}".format(POSITION_KD))

ratio=1.05

def inc(param):
    if param < 2:
        param = round(param + 0.1, 1)
    else:
        old=param
        param = round(param*ratio)
        if param==old:
            param+=1
    return param

def dec(param):
    if param < 2:
        param = max(0,round(param - 0.1, 1))
    else:
        old=param
        param = round(param/ratio)
        if param==old:
            param-=1
    return param

if ANGLE_CTRL_PERIOD_MS < CONTROL_PERIOD_MS or POSITION_CTRL_PERIOD_MS <CONTROL_PERIOD_MS:
    raise Exception("angle or position control periods too short compared to CONTROL_PERIOD_MS")

# check that we are running from terminal, otherwise we cannot control it
if sys.stdin.isatty():
    # running interactively
    print('running interactively from an interactive terminal, ok')
else:
   print('run from an interactive terminal to allow keyboard input')
   quit()

################################################################################
# OPEN SERIAL PORT
################################################################################
p = Pendulum()
serialPorts=serial_ports()
print('Available serial ports: '+str(serialPorts))
if len(serialPorts)==0:
    print('no serial ports available, or cannot open it; check linux permissions\n Under linux, sudo chmod a+rw [port] transiently, or add user to dialout or tty group')
    quit()

if len(serialPorts)>1:
    print(str(len(serialPorts))+' serial ports, taking first one which is '+str(serialPorts[0]))
SERIAL_PORT=str(serialPorts[0])
try:
    p.open(SERIAL_PORT, SERIAL_BAUD)
except:
    print('cannot open port '+str(SERIAL_PORT)+': available ports are '+str(serial_ports()))
    quit()

print('opened '+str(SERIAL_PORT)+' successfully')
p.control_mode(False)
p.stream_output(False)

joystickExists=False
pygame.init()
joystick.init()
if joystick.get_count()==1:
    stick = joystick.Joystick(0)
    stick.init()
    axisNum = stick.get_numaxes()
    buttonNum = stick.get_numbuttons()
    joystickExists=True
    print('joystick found with '+str(axisNum)+' axes and '+str(buttonNum)+' buttons')
else:
    print('no joystick found, only PD control or no control possible')

if CALIBRATE:
    print("Calibrating motor position....")
    if not p.calibrate():
        print("Failed to connect to device")
        p.close()
        exit()
    print("Done calibrating")

loadparams()
time.sleep(1)

################################################################################
# SET PARAMETERS
################################################################################
p.set_angle_config(     ANGLE_TARGET,
                        ANGLE_AVG_LENGTH,
                        ANGLE_SMOOTHING,
                        ANGLE_KP,
                        ANGLE_KD)

p.set_position_config(  POSITION_TARGET,
                        POSITION_CTRL_PERIOD_MS,
                        POSITION_SMOOTHING,
                        POSITION_KP,
                        POSITION_KD)

################################################################################
# GET PARAMETERS
################################################################################
(   ANGLE_TARGET,
    ANGLE_AVG_LENGTH,
    ANGLE_SMOOTHING,
    ANGLE_KP,
    ANGLE_KD)           = p.get_angle_config()

(   POSITION_TARGET,
    POSITION_CTRL_PERIOD_MS,
    POSITION_SMOOTHING,
    POSITION_KP,
    POSITION_KD)        = p.get_position_config()

################################################################################
# CONTROL LOOP (PC BASED)
################################################################################
printCount          = 0

angleErrPrev        = 0
angleCmd            = 0
positionErrPrev     = 0
positionCmd         = 0

controlEnabled=False

danceEnabled=False
danceAmpl=500
dancePeriodS=8

loggingEnabled=False

kbAvailable=True
try:
    kb = kbhit.KBHit() # can only use in posix terminal; cannot use from spyder ipython console for example
except:
    kbAvailable=False

printparams()
help()
startTime=time.time()
lastTime=startTime
lastAngleControlTime=lastTime
lastPositionControlTime=lastTime
angleErr=0
positionErr=0 # for printing even if not controlling
p.stream_output(True)  # now start streaming state
while True:

    # Adjust Parameters
    if kbAvailable & kb.kbhit():
        c = kb.getch()

        if c=='D':
            danceEnabled=~danceEnabled
            print("\ndanceEnabled= {0}".format(danceEnabled))
        elif c == 'l':
            loggingEnabled=~loggingEnabled
            print("\nloggingEnabled= {0}".format(loggingEnabled))
            if loggingEnabled:
                try:
                    csvfilename=datetime.now().strftime("cartpole-%Y-%m-%d-%H-%M-%S.csv")
                    csvfile=open(csvfilename, 'w', newline='')
                    csvwriter = csv.writer(csvfile, delimiter=',')
                    csvwriter.writerow(['time'] + ['deltaTimeMs']+['angle'] + ['position'] + ['angleTarget']  + ['angleErr'] + ['positionTarget'] + ['positionErr'] + ['angleCmd'] + ['positionCmd'] + ['motorCmd']+['actualMotorCmd'])
                    print("\n Started logging data to "+csvfilename)
                except Exception as e:
                    loggingEnabled=False
                    print("\n" + str(e) + ": Exception opening csvfile; logging disabled")
            else:
                csvfile.close()
                print("\n Stopped logging data to "+csvfilename)

        elif c == 'k':
            controlEnabled=~controlEnabled
            print("\ncontrolEnabled= {0}".format(controlEnabled))
        elif c == 'K':
            controlEnabled=False
            print("\nCalibration triggered")
            p.calibrate()
            print("\nCalibration finished")
        elif c == 'h' or c=='?':
            help()
        elif c == 'p' :
            printparams()
        # Increase Target Angle
        elif c == '=':
            ANGLE_TARGET += 1
            print("\nIncreased target angle to {0}".format(ANGLE_TARGET))
        # Decrease Target Angle
        elif c == '-':
            ANGLE_TARGET -= 1
            print("\nDecreased target angle to {0}".format(ANGLE_TARGET))

        # Increase Target Position
        elif c == ']':
            POSITION_TARGET += 200
            print("\nIncreased target position to {0}".format(POSITION_TARGET))
        # Decrease Target Position
        elif c == '[':
            POSITION_TARGET -= 200
            print("\nDecreased target position to {0}".format(POSITION_TARGET))

        # Angle Gains
        elif c == 'w':
            ANGLE_KP=inc(ANGLE_KP)
            print("\nIncreased angle KP {0}".format(ANGLE_KP))
        elif c == 'q':
            ANGLE_KP=dec(ANGLE_KP)
            print("\nDecreased angle KP {0}".format(ANGLE_KP))
        elif c == 's':
            ANGLE_KD=inc(ANGLE_KD)
            print("\nIncreased angle KD {0}".format(ANGLE_KD))
        elif c == 'a':
            ANGLE_KD=dec(ANGLE_KD)
            print("\nDecreased angle KD {0}".format(ANGLE_KD))
        elif c == 'x':
            ANGLE_SMOOTHING=dec(ANGLE_SMOOTHING)
            if ANGLE_SMOOTHING>1:
                ANGLE_SMOOTHING=1
            print("\nIncreased ANGLE_SMOOTHING {0}".format(ANGLE_SMOOTHING))
        elif c == 'z':
            ANGLE_SMOOTHING=inc(ANGLE_SMOOTHING)
            if ANGLE_SMOOTHING>1:
                ANGLE_SMOOTHING=1
            print("\nDecreased ANGLE_SMOOTHING {0}".format(ANGLE_SMOOTHING))

        # Position Gains
        elif c == 'r':
            POSITION_KP=inc(POSITION_KP)
            print("\nIncreased position KP {0}".format(POSITION_KP))
        elif c == 'e':
            POSITION_KP=dec(POSITION_KP)
            print("\nDecreased position KP {0}".format(POSITION_KP))
        elif c == 'f':
            POSITION_KD=inc(POSITION_KD)
            print("\nIncreased position KD {0}".format(POSITION_KD))
        elif c == 'd':
            POSITION_KD=dec(POSITION_KD)
            print("\nDecreased position KD {0}".format(POSITION_KD))
        elif c == 'v':
            POSITION_SMOOTHING=dec(POSITION_SMOOTHING)
            if POSITION_SMOOTHING>1:
                POSITION_SMOOTHING=1
            print("\nIncreased POSITION_SMOOTHING {0}".format(POSITION_SMOOTHING))
        elif c == 'c':
            POSITION_SMOOTHING=inc(POSITION_SMOOTHING)
            if POSITION_SMOOTHING>1:
                POSITION_SMOOTHING=1
            print("\nDecreased POSITION_SMOOTHING {0}".format(POSITION_SMOOTHING))
        elif c=='S':
            saveparams()
        elif c=='L':
            loadparams()

        # Exit
        elif ord(c) == 27 : # ESC
            print("\nquitting....")
            break

    # This function will block at the rate of the control loop
#    p.clear_read_buffer() # if we don't clear read buffer, state output piles up in serial buffer
    (angle, position, command) = p.read_state()
    # angle count is more positive CCW facing cart, position encoder count is more positive to right facing cart

    timeNow=time.time()
    deltaTime=timeNow-lastTime
    if deltaTime==0:
        deltaTime=1e-6
    lastTime=timeNow
    elapsedTime=timeNow-startTime
    diffFactor=(CONTROL_PERIOD_MS/(deltaTime*1000))

    positionTargetNow=POSITION_TARGET
    if controlEnabled and danceEnabled:
        positionTargetNow=POSITION_TARGET+danceAmpl*math.sin(2*math.pi*(elapsedTime/dancePeriodS))

    # Balance PD Control
    # Position PD Control
    if timeNow -lastPositionControlTime >= POSITION_CTRL_PERIOD_MS*.001:
        lastPositionControlTime=timeNow
        positionErr         = POSITION_SMOOTHING*(position - positionTargetNow) + (1.0 - POSITION_SMOOTHING)*positionErrPrev # First order low-P=pass filter
        positionErrDiff     = (positionErr - positionErrPrev)*diffFactor
        positionErrPrev     = positionErr
        # Naive solution: if too positive (too right), move left (minus on positionCmd),
        # but this does not produce correct control.
        # The correct strategy is that if cart is too positive (too right),
        # produce lean to the left by introducing a positive set point angle leaning slightly to left,
        # i.e. more positve positionErr makes more positive effective ANGLE_TARGET
        # End result is that sign of positionCmd is flipped
        # Also, if positionErr is increasing more, then we want even more lean, so D sign is also positive
        positionCmd         = +(POSITION_KP*positionErr + POSITION_KD*positionErrDiff)

    if timeNow-lastAngleControlTime >= ANGLE_CTRL_PERIOD_MS*.001:
        lastAngleControlTime=timeNow
        angleErr            = ANGLE_SMOOTHING*(angle - ANGLE_TARGET) + (1.0 - ANGLE_SMOOTHING)*angleErrPrev # First order low-pass filter
        angleErrDiff        = (angleErr - angleErrPrev)*diffFactor # correct for actual sample interval; if interval is too long, reduce diff error
        angleErrPrev        = angleErr
        angleCmd            = -(ANGLE_KP*angleErr + ANGLE_KD*angleErrDiff) # if too CCW (pos error), move cart left

    motorCmd = int(round(angleCmd + positionCmd)) # change to plus for original, check that when cart is displayed, the KP term for cart position leans cart the correct direction
    motorCmd =  MOTOR_MAX_PWM if motorCmd >  MOTOR_MAX_PWM else motorCmd
    motorCmd = -MOTOR_MAX_PWM if motorCmd < -MOTOR_MAX_PWM else motorCmd

    stickPos=0.0
    if joystickExists:
        # for event in pygame.event.get(): # User did something.
        #     if event.type == pygame.QUIT: # If user clicked close.
        #         done = True # Flag that we are done so we exit this loop.
        #     elif event.type == pygame.JOYBUTTONDOWN:
        #         print("Joystick button pressed.")
        #     elif event.type == pygame.JOYBUTTONUP:
        #         print("Joystick button released.")
        pygame.event.get() # must call get() to handle internal queue
        stickPos=stick.get_axis(0) # 0 left right, 1 front back 2 rotate

    if abs(stickPos)>JOYSTICK_DEADZONE:
        actualMotorCmd=int(round(stickPos*JOYSTICK_SCALING))
    elif controlEnabled:
        actualMotorCmd=motorCmd
    else:
        actualMotorCmd=0

    if POLOLU_MOTOR==False:
        p.set_motor(-actualMotorCmd) # positive motor cmd moves cart right
    else:
        p.set_motor(actualMotorCmd)  # positive motor cmd moves cart right

    if loggingEnabled:
#      csvwriter.writerow(['time'] + ['deltaTimeMs']+['angle'] + ['position']  + ['angleErr'] + ['positionErr'] + ['angleCmd'] + ['positionCmd'] + ['motorCmd'])
       csvwriter.writerow([elapsedTime,deltaTime*1000,angle, position, ANGLE_TARGET, angleErr, positionTargetNow, positionErr, angleCmd,positionCmd,motorCmd,actualMotorCmd])

    # Print output
    printCount += 1
    if printCount >= (PRINT_PERIOD_MS/CONTROL_PERIOD_MS):
        printCount = 0
        print("\r angle {:+4d} angleErr {:+6.1f} position {:+6d} positionErr {:+6.1f} angleCmd {:+6d} positionCmd {:+6d} motorCmd {:+6d} dt {:.3f}ms  stick {:.3f}         \r".format(int(angle), angleErr, int(position),  positionErr, int(round(angleCmd)), int(round(positionCmd)), actualMotorCmd, deltaTime*1000, stickPos), end = '')
# if we pause like below, state info piles up in serial input buffer
        # instead loop at max possible rate to get latest state info
#    time.sleep(CONTROL_PERIOD_MS*.001)  # not quite correct since there will be time for execution below

# when x hit during loop or other loop exit
p.set_motor(0) # turn off motor
p.close()
joystick.quit()

if loggingEnabled:
    csvfile.close()


import enum
from typing import Dict, List, Tuple

# --------------------------
# Basic (string, float, int)
# --------------------------
name: str = "Tay May"
weight: float = 60.2
age: int = 16

print(name)
print(weight)
print(age)

# --------------------------
# List
# --------------------------
thanhvien_cs102: List[str] = ["Tay May", "To Mo", "Robot"]
# có thể thay đổi giá trị
thanhvien_cs102[1] = "Hello"
print(thanhvien_cs102)

# --------------------------
# Tuple
# --------------------------
mytuple: Tuple[str, str, int] = ("Pygame", "with", 102)
# Không thể thay đổi giá trị
# mytuple[2] = 103
print(mytuple)

# --------------------------
# Dictionary
# --------------------------
card: Dict[str, str] = {"course": "CS102", "main": "Pygame"}
print(card["main"])

# --------------------------
# enum
# --------------------------
class GameStateType(enum.Enum):
    RUNNING = 0
    WON = 1
    LOST = 2

state: GameStateType = GameStateType.RUNNING
print(state == GameStateType.RUNNING)
print(state == GameStateType.WON)

# --------------------------
# Function
# --------------------------
def sum(items: List[float]) -> float:
    total: float = 0.0
    for item in items:
        total += item
    return total

danh_sach_diem: List[float] = [2.5, 1.5, 2, 3.25]
tong: float = sum(danh_sach_diem)

print(tong)


#from exam_word import word_04
from .exam_func import TEST, MEAN,EXAMPLE, EXAMPLE_test, SIMILAR
import json
from collections import OrderedDict
from .W2V_word import W2V
import random

# word = word_04

def REMOVE(st):
    row = ' '.join(s for s in st)
    remove = "}"
    for x in range(len(remove)):
      row1 = row.replace(remove[x],"")
    row2 = row1.replace("'","")
    row3 = row2.split('.')
    strip_li = []
    for i in row3:
        i = i.strip()
        if i:
          strip_li.append(i)

    return strip_li

def W2V_MEAN(w2v_word):
    n = len(w2v_word)
    w2v_mean = []
    for i in range(0,n):
        t = TEST(w2v_word[i])
        result_parse = t[0]
        row1 = MEAN(result_parse)
        row2 = REMOVE(row1)
        w2v_mean.append(row2)

    return w2v_mean

def t04(word):

    w2v_word = W2V(word)

    t = TEST(word)
    result_parse = t[0]
    result_similar = t[2]
    result_similar = random.choice(result_similar)

    mean = result_similar['예시']
    similar = result_similar['유의어']
    similar_list = []
    similar_list.append(similar)

    w2v_mean = W2V_MEAN(w2v_word) + W2V_MEAN(similar_list)
    w2v_word = w2v_word + similar_list

    choice = { name:value for name, value in zip(w2v_word, w2v_mean) }

    file_exam4 = OrderedDict()
    file_exam4["TYPE4"] = "다음 문장 속 "+word+"의 의미와 가장 관련이 깊은 단어를 고르시오."
    #file_exam4["WORD"] = word    #단어
    file_exam4["ANSWER"] = similar    #유의어
    file_exam4["MEAN"] = mean    #뜻
    file_exam4["CHOICE"] = choice

    EXAM4 = json.dumps(file_exam4, ensure_ascii=False, indent="\t")
    print(EXAM4)
    return EXAM4


from easy_pyechart import _funnel_base_config, constants
from pyecharts import options as opts
from pyecharts.commons.utils import JsCode
from typing import Any, Optional
from pyecharts.charts import Funnel
import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

'''漏斗图'''
class eFunnel():
    def __init__(
            self,
            title: Optional[str] = None,
            subTitle: Optional[str] = None,
            lableList: Optional[list] = [],
            valueList: Optional[list] = [],
            themeType=constants.defualt_theme,
            backgroundImageUrl: Optional[str] = None):
        self.opts: dict = {
            "lengend": Funnel,
            "xList": lableList,
            "yList": valueList,
            "themeType": themeType,
            "backgroundImageUrl": backgroundImageUrl,
            "title": title,
            "subTitle": subTitle,
        }
    '''三角形的漏斗图设置'''
    def _funnel_chart(self):
        return _funnel_base_config(self)


import sys, dpkt, socket
from dpkt.compat import compat_ord

class Statistics:               #Statistic Class: Used just to store global stats of the following info
    connCount = 0
    rstCount = 0
    openCount = 0
    closeCount = 0
    duration = 0
    minDuration = 0
    meanDuration = 0
    maxDuration = 0
    RTTCount = 0
    RTT = []
    minRTT = -1
    meanRTT = -1
    maxRTT = -1
    pktCount = 0
    minPacket = 0
    meanPacket = 0
    maxPacket = 0
    window = []
    minWindow = -1
    meanWindow = -1
    maxWindow = -1

class Packet(object):       #Packet Class: used to store packet info. A packet class is created for each packet, and destroyed after being analyzed
    srcMac = ""
    dstMac = ""
    srcIP = ""
    dstIP = ""
    IPLen = -1
    id = -1
    seq = -1
    ack = -1
    windowSize = -1
    flagsBin = -1
    flags = []
    srcPort = -1
    dstPort = -1
    time = -1

class Connection:                           #Connection Info. Used to store
    def __init__(self, packet):
        self.srcAdd = packet.srcIP
        self.dstAdd = packet.dstIP
        self.srcPort = packet.srcPort
        self.dstPort = packet.dstPort
        self.status = [0, 0, 0]             #SYN Count, FIN Count, RST Count
        self.startTime = packet.time
        self.endTime = packet.time
        self.srcDstPacketCount = 0
        self.dstSrcPacketCount = 0
        self.packetCount = 0
        self.srcDstByteCount = 0
        self.dstSrcByteCount = 0
        self.byteCount = 0
        self.initialClientSeq = packet.seq + 1
        self.initialServerSeq = 0
        self.pastClientSeq = -50
        self.pastServerSeq = 0
        self.pastClientPacketTime = packet.time
        self.pastServerPacketTime = 0
        self.RTTCount = 0
        self.calRTT = 0

        self.duration = 0
        self.RTT = []
        self.window = []

class Connections:
    def __init__(self):
        self.links = []
        self.size = 0

    def add(self, connection):
        self.links.append(connection)
        self.size = self.size + 1

    def printConnections(self):
        count = 1
        for link in self.links:
            print("Connection " + str(count) + ":")
            print("Source Address: " + link.srcAdd)
            print("Destination Address: " + link.dstAdd)
            print("Source Port: " + str(link.srcPort))
            print("Destination Port: " + str(link.dstPort))
            print("Status: " + "S" + str(link.status[0]) + "F" + str(link.status[1]) + "R" + str(link.status[2]))
            if link.status[0] >= 1:
                if link.status[1] >= 1:
                    print("Start Time: " + str(link.startTime) + "ms")
                    print("End Time: " + str(link.endTime) + "ms")
                    print("Duration: " + str(link.duration) + "ms")
                    print("Number of packets send from Source to Destination: " + str(link.srcDstPacketCount))
                    print("Number of packets send from Destination to Source: " + str(link.dstSrcPacketCount))
                    print("Total number of packets: " + str(link.packetCount))
                    print("Number of data bytes send from Source to Destination: " + str(link.srcDstByteCount))
                    print("Number of data bytes send from Destination to Source: " + str(link.dstSrcByteCount))
                    print("Total number of data bytes: " + str(link.byteCount))
                    print("END")
            count = count + 1
            if count <= (self.size): print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")

def printPacket(packet):
    print("Source MAC: " + packet.srcMac)
    print("Destination MAC: " + packet.dstMac)
    print("Source IP: " + packet.srcIP)
    print("Destination IP: " + packet.dstIP)
    print("IP Header Length: " + str(packet.IPLen))
    print("Packet ID: " + str(packet.id))
    print("Sequence: " + str(packet.seq))
    print("Acknowledgement: " + str(packet.ack))
    print("Window Size: " + str(packet.windowSize))
    print("Flag Binary: " + bin(packet.flagsBin))
    print("Flags: " + str(packet.flags))
    print("Source Port: " + str(packet.srcPort))
    print("Destination Port: " + str(packet.dstPort))
    print("Time: " + str(packet.time))

def mac_addr(address):      #Refer to Reference                     #Used to convert binary to mac addresses
    return ":".join("%02x" % compat_ord(b) for b in address)

def inet_to_str(inet):      #Refer to Reference                     #Used to convert binary to Ip Addresses
    return socket.inet_ntop(socket.AF_INET, inet)

def binToFlags(packet):                                             #Binary Flag parsing
    packet.flags = []
    if packet.flagsBin & 0b1: packet.flags.append("FIN")
    if packet.flagsBin & 0b10: packet.flags.append("SYN")
    if packet.flagsBin & 0b100: packet.flags.append("RST")
    if packet.flagsBin & 0b1000: packet.flags.append("PSH")
    if packet.flagsBin & 0b10000: packet.flags.append("ACK")
    if packet.flagsBin & 0b100000: packet.flags.append("URG")
    return packet

def clientInitialRTT(stats, connection, packet):                    #The initial time for RTT
    connection.pastClientSeq = packet.seq                           #Initial sequence number sent
    connection.pastClientPacketTime = packet.time                   #Initial packet time
    return 0

def clientFinalRTT(stats, connection, packet):                      #Client final RTT
    if connection.pastClientSeq <= packet.ack:                      #Ensure that the ack receieved corresponds to an ack
        RTT = packet.time - connection.pastClientPacketTime         #Calculate RTT time between the matching seq to ack
        connection.RTT.append(RTT)                                  #Append RTT calculation to connection for mean and other purposes
    return 0

def updateDstSrcCount(connection, packet):                          #Calculation to update the byte and packet count from the destunation to source
    connection.packetCount = connection.packetCount + 1
    connection.srcDstPacketCount = connection.srcDstPacketCount + 1
    connection.dstSrcByteCount = packet.ack - connection.initialServerSeq - 1
    connection.byteCount = connection.srcDstByteCount + connection.dstSrcByteCount
    return packet.ack - connection.initialServerSeq - 1

def updateSrcDstCount(connection, packet):                          #Method to update the byte and packet count from the source to destionation
    connection.packetCount = connection.packetCount + 1
    connection.dstSrcPacketCount = connection.dstSrcPacketCount + 1
    if connection.initialServerSeq == 0:                            #Initial server / client 3 way hand shake scenario handling
        connection.initialServerSeq = packet.seq + 1
    connection.srcDstByteCount = packet.ack - connection.initialClientSeq
    connection.byteCount = connection.srcDstByteCount + connection.dstSrcByteCount
    return packet.ack - connection.initialClientSeq

def printFinal(stats, connections):
    print("A) Total number of connections: " + str(connections.size))
    print("___________________________________________________________________________________")
    print("")
    print("B) Connection's details:")
    print("")
    connections.printConnections()
    print("___________________________________________________________________________________")
    print("")
    print("C) General:")
    print("")
    print("Total number of complete TCP connections: " + str(stats.closeCount))
    print("Number of reset TCP connections: " + str(stats.rstCount))
    print("Number of TCP connections that were still open when the trace capture ended: " + str(stats.openCount))
    print("___________________________________________________________________________________")
    print("")
    print("D) Complete TCP connections:")
    print("")
    print("Minimum time durations: " + str(stats.minDuration) + "ms")
    print("Mean time durations: " + str(stats.meanDuration) + "ms")
    print("Maximum time duration: " + str(stats.maxDuration) + "ms")
    print("")
    print("Minimum RTT values: " + str(stats.minRTT) + "ms")
    print("Mean RTT values: " + str(stats.meanRTT) + "ms")
    print("Maximum RTT values: " + str(stats.maxRTT) + "ms")
    print("")
    print("Minimum number of packets including both send/received: " + str(stats.minPacket))
    print("Mean number of packets including both send/received: " + str(stats.meanPacket))
    print("Maximum number of packets including both send/received: " + str(stats.maxPacket))
    print("")
    print("Minimum receive window sizes including both send/received: " + str(stats.minWindow))
    print("Mean receive window sizes including both send/received: " + str(stats.meanWindow))
    print("Maximum receive window sizes including both send/receive: " + str(stats.maxWindow))
    print("___________________________________________________________________________________")

def analyzePacket(stats, connections, packet):              #Series of function calls that analyzes all the packets
    for connection in connections.links:                    #Checks whether a connection exists in file for the packet being analyzed
        if (connection.srcAdd == packet.srcIP) and (connection.dstAdd == packet.dstIP) and (connection.srcPort == packet.srcPort) and (connection.dstPort == packet.dstPort):
            if "SYN" in packet.flags:
                connection.status[0] = connection.status[0] + 1         #Update SYN Count
            if "FIN" in packet.flags:
                connection.status[1] = connection.status[1] + 1         #Update FIN Count
                connection.endTime = packet.time                        #Update END TIME
            if "RST" in packet.flags:
                connection.status[2] = connection.status[2] + 1         #Update RST Count

            connection.window.append(packet.windowSize)                 #Store Window Size
            byteTransfered = updateDstSrcCount(connection, packet)      #Calculate if any bytes were sent /received, and store

            if "SYN" in packet.flags or "FIN" in packet.flags:          #Calculate the RTT if it is SYN or FIN
                connection.calRTT = 1
                clientInitialRTT(stats, connection, packet)

            return 1

                                        #Serires of function calls that analyzes all the packets
                                        #Similar as to above, but for server -> destionation packets
        if (connection.dstAdd == packet.srcIP) and (connection.srcAdd == packet.dstIP) and (connection.dstPort == packet.srcPort) and (connection.srcPort == packet.dstPort):
            if "SYN" in packet.flags:
                connection.status[0] = connection.status[0] + 1
            if "FIN" in packet.flags:
                connection.status[1] = connection.status[1] + 1
                connection.endTime = packet.time
            if "RST" in packet.flags:
                connection.status[2] = connection.status[2] + 1

            connection.window.append(packet.windowSize)
            byteTransfered = updateSrcDstCount(connection, packet)

                                            #Only calculate RTT is, it is SYN, has data transfered
            if ((byteTransfered > 0 and "ACK" in packet.flags) or "SYN" in packet.flags) and connection.calRTT == 1:
                connection.calRTT = 0
                clientFinalRTT(stats, connection, packet)

            return 1

    connection = Connection(packet)                                                 #If there isn't any existing connection, create a connection
    connection.srcDstPacketCount = connection.srcDstPacketCount + 1                 #Update packet count
    connection.packetCount = connection.packetCount + 1                             #
    connection.status[0] = connection.status[0] + 1                                 #Add syn count
    stats.openCount = stats.openCount + 1                                           #Add open count
    stats.connCount = stats.connCount + 1                                           #Add connection cvount
    connection.window.append(packet.windowSize)                                     #Store c
    connections.add(connection)

    if "SYN" in packet.flags:                                                       #If SYN, prepare values for RTT
        connection.calRTT = 1
        clientInitialRTT(stats, connection, packet)
    return 0

def finalStatCheck(stats, connections):                                             #After analyzing all the packets
    for connection in connections.links:                                            #For all connections
        if connection.status[0] >= 1:                                               #If SYN
            if connection.status[1] >= 1:                                           #If FIN
                stats.openCount = stats.openCount - 1                               #Complete connection
                stats.closeCount = stats.closeCount + 1

                connection.duration = connection.endTime - connection.startTime     #min mean max duration for all compelete connections
                stats.duration = stats.duration + connection.duration
                if stats.minDuration == 0:
                    stats.minDuration = connection.duration
                if stats.maxDuration == 0:
                    stats.maxDuration = connection.duration
                if connection.duration <= stats.minDuration:
                    stats.minDuration = connection.duration
                if connection.duration >= stats.maxDuration:
                    stats.maxDuration = connection.duration

                stats.pktCount = stats.pktCount + connection.packetCount        #min mean max packet count for all complete connections
                if stats.minPacket == 0:
                    stats.minPacket = connection.packetCount
                if stats.maxPacket == 0:
                    stats.maxPacket = connection.packetCount
                if connection.packetCount <= stats.minPacket:
                    stats.minPacket = connection.packetCount
                if connection.packetCount >= stats.maxPacket:
                    stats.maxPacket = connection.packetCount

                stats.window.extend(connection.window)                          #update connection window for min, mean max calculations

                stats.RTT.extend(connection.RTT)                                #add rtt for min mean max valculations

            if connection.status[2] >= 1:
                stats.rstCount = stats.rstCount + 1

    stats.meanDuration = stats.duration / stats.closeCount                      #mean duration calculation
    stats.meanPacket = stats.pktCount / stats.closeCount                        #mean packet count duration

    stats.minWindow = min(stats.window)                                         #min mean max window size calculation
    stats.maxWindow = max(stats.window)
    stats.meanWindow = sum(stats.window)/stats.pktCount

    stats.minRTT = min(stats.RTT)                                               #min mean max RTT calculation
    stats.maxRTT = max(stats.RTT)
    stats.meanRTT = sum(stats.RTT) / len(stats.RTT)

    return 1

def main():
    traceFileName = sys.argv[1]                         #name of file to read from

    traceFile = open(traceFileName, "rb")               #open the file to read in binary
    tracePcap = dpkt.pcap.Reader(traceFile)             #use a reader to parse

    stats = Statistics()
    connections = Connections()
    count = 0

    for timeStamp, buf in tracePcap:                    #Refer to reference. Parts of the referenced code has been deleted or modified.
        eth = dpkt.ethernet.Ethernet(buf)
        ip = eth.data                                   #IP Header
        tcp = ip.data                                   #TCP Header

        packet = Packet()                               #Storing various values into a packet class
        packet.srcMac = mac_addr(eth.src)
        packet.dstMac = mac_addr(eth.dst)

        packet.srcIP = inet_to_str(ip.src)
        packet.dstIP = inet_to_str(ip.dst)
        packet.IPLen = ip.len
        packet.id = ip.id

        packet.seq = tcp.seq
        packet.ack = tcp.ack
        packet.windowSize = tcp.win
        packet.flagsBin = tcp.flags
        packet.srcPort = tcp.sport
        packet.dstPort = tcp.dport
        packet.time = timeStamp
        packet = binToFlags(packet)

        analyzePacket(stats, connections, packet)       #For each packet, analyze
        del packet

    finalStatCheck(stats, connections)
    printFinal(stats, connections)

main()

# Parsing is taken from the link below, in particluar the mac_addr, and inet_to_str.
# Opening the file, and obtaining the buffer and timestamp is from
# http://dpkt.readthedocs.io/en/latest/_modules/examples/print_packets.html?highlight=print%20ip


__doc__ = "this module contains varoous tools"
from datetime import date, datetime

# built in modules:
# import sys
# import os

# modules from pypi (install using `pip install module_name`)
# paramiko
# requests

def input_int(num_range: tuple):
    """
    `range`: tuple like (from, to)
    """
    frm, to = num_range
    option = input(f'enter number between {frm} and {to} ')
    while (not option.isdigit()) or (int(option) < frm or int(option) > to):
        print("Error: invalid option")
        option = input()
    return int(option)

def cool():
    print("la la la ")

def print_this_time():
    """should be used only when this modle is the main module"""
    print(datetime.now())

if __name__ == '__main__':
    print_this_time()


from typing import Literal

ver_num = "3.2.2"
online_message = "Oh no, pas encore..."

mods = {}

def enable_module(mod):
    mods[mod] = "✅"

def disable_module(mod):
    mods[mod] = "❌"

def get_modules():
    return mods

ban_domain = ["twitter", "deezer", "spotify"]

values = Literal[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50]

sites_dict = {
    "dailymotion": {
        "icon_url": "https://upload.wikimedia.org/wikipedia/commons/2/27/Logo_dailymotion.png",
        "color": 0x00bff9,
        "message": None
    },
    "soundcloud": {
        "icon_url": "https://play-lh.googleusercontent.com/lvYCdrPNFU0Ar_lXln3JShoE-NaYF_V-DNlp4eLRZhUVkj00wAseSIm-60OoCKznpw=w240-h480",
        "color": 0xff6800,
        "message": None
    },
    "tiktok": {
        "icon_url": "https://cdn.pixabay.com/photo/2021/06/15/12/28/tiktok-6338432_960_720.png",
        "color": 0xee1d52,
        "message": None
    },
    "twitch": {
        "icon_url": "https://static-00.iconduck.com/assets.00/twitch-icon-2048x2048-tipdihgh.png",
        "color": 0x9146ff,
        "message": None
    },
    "twitter": {
        "icon_url": "https://e7.pngegg.com/pngimages/804/985/png-clipart-social-media-logo-computer-icons-information-twitter-logo-media.png",
        "color": 0x05acf0,
        "message": None
    },
    "youtube": {
        "icon_url": "https://cdn.icon-icons.com/icons2/1099/PNG/512/1485482355-youtube_78661.png",
        "color": 0xfe0000,
        "message": None
    },
    "reddit": {
        "icon_url": "https://freelogopng.com/images/all_img/1658834272reddit-logo-transparent.png",
        "color": 0xff4500,
        "message": None
    },
    "générique": {
        "thumbnail": "https://images.frandroid.com/wp-content/uploads/2018/08/guide-apps-video-android.jpg",
        "icon_url": "https://cdn0.iconfinder.com/data/icons/basic-uses-symbol-vol-2/100/Help_Need_Suggestion_Question_Unknown-512.png",
        "color": 0xffffff,
        "message": None
    }
}


import os
import gc
import re
import sys
import copy
import time
import random
import tempfile
import logging
import cPickle as cp
import multiprocessing
import subprocess
import deepity
import numpy as np
import numpy.random as npr
import smat as sm
import scipy
import scipy.stats
from . import std
from .        import hpsearch as hp
from .        import _io_
from .        import util
from .data    import datasource
from .        import globals
from .report  import training_report, calc_auc, bootstrap_auc
import node    as _node
import trainer as _trainer

class _object_factory_from_file(object):
    def __init__(self,filename,fieldname=None):
        self.filename = filename
        self.fieldname = fieldname

    def __call__(self):
        obj = _io_.load(self.filename)
        if self.fieldname and isinstance(obj,dict):
            return obj[self.fieldname]
        return obj

def _create_model(model_proto, hparams):
    model = copy.deepcopy(model_proto)
    for key,val in hparams.iteritems():
        prefix,path = key.split(":")  # look for hparams named "model:..."
        if prefix == "model":
            nodepath,attrname = path.rsplit(".",1)
            node = model.find(nodepath)
            if hasattr(node,"set_"+attrname):
                getattr(node,"set_"+attrname)(model,val)   # call model.set_xxx(val)
            else:
                setattr(node,attrname,val)
    return model

def _create_trainer(trainer_proto, hparams):
    trainer = copy.deepcopy(trainer_proto)
    for key,val in hparams.iteritems():
        prefix,attrname = key.split(":")  # look for hparams named "trainer:..."
        if prefix == "trainer":
            if hasattr(trainer,"set_"+attrname):
                getattr(trainer,"set_"+attrname)(model,val)   # call trainer.set_xxx(val)
            else:
                setattr(trainer,attrname,val)
    return trainer

def _slice_hparams(hparams, inst):
    h = copy.deepcopy(hparams)
    for key in h.keys():
        h[key] = h[key][inst]
    return h

def load_hparams_result(filename):
    with open(filename,'r') as f:
        lines = f.readlines()
    params = {}
    result = 0.0
    for line in lines:
        # Look for validation performance
        matches = re.findall("# metric = (\S+)", line)
        if len(matches) > 0:
            result = float(matches[0])
            continue

        # Add hparam
        name, value = re.findall(" *(\S+) += (\S+)", line)[0]
        if name in [":cfgname"]:
            params[name] = value
        else:
            params[name] = float(value)

    return hp.sample(params, result)

def save_hparams_result(filename, hparams_result, metric_key):
    util.makepath(os.path.dirname(filename))
    with open(filename,'w') as f:
        if metric_key:
            f.write("# metric = %f (%s)\n" % (hparams_result.metrics[metric_key], metric_key))
        f.write(hparams2str(hparams_result.params))

def _save_model_inst(filename, inst, model, hparams):
    m = copy.deepcopy(model)
    sm.sync()

    # Slice the trainable weights
    m.slice_inst(inst)

    # Also slice the hyperparams, and replace corresponding 'arrayed'
    # attributes in the model with their scalar (sliced element) counterpart.
    h = _slice_hparams(hparams,inst)
    for key,val in h.iteritems():
        prefix,path = key.split(":")  # look for hparams named "model:..."
        if prefix != "model":
            continue
        nodepath,attrname = path.rsplit(".",1)
        node = m.find(nodepath)
        if hasattr(node,"set_"+attrname):
            getattr(node,"set_"+attrname)(model,val)   # call model.set_xxx(val)
        else:
            setattr(node,attrname,val)

    # Dump the model
    util.makepath(os.path.dirname(filename))
    with open(filename,'wb') as f:
        cp.dump(m,f)

    sm.sync() # Make sure we wait until the sarrays are all dumped

def gen_predictions(model, data):
    # We must feed each sequence through the model several times
    # by applying the model repeatedly on sliding a window along the sequence.
    # That generates a prediction map, from which we can take max, sum, etc.
    predictions = []
    gmaps = {}
    batches = data.asbatches(batchsize=128, reshuffle=False)
    for batch in batches:
        args = batch.input_data()
        args["want_bprop_inputs"] = False
        if isinstance(model.Z.origin().node,std.softmaxnode):
            args["bprop_inputs_loss"] = std.nll()
        else:
            args["bprop_inputs_loss"] = std.mse()
        outputs = model.eval(**args)
        Z = outputs['Z'].asnumpy()
        Zmask = outputs.get('Zmask',None)
        if Zmask is not None:
            Zmask = Zmask.asnumpy()
            Z = Z[Zmask.ravel()]
        predictions.append(Z)

    # Concatenate all numpy arrays if they're the same size
    predictions = np.vstack(predictions)

    return predictions

def getinstdir(outdir, targetname, trialnum, foldid):
    if isinstance(outdir,str):
        return outdir
    outdir = [_ for _ in outdir] # Make a copy that we can substitute elements in
    args = {"target" : targetname,
            "trial"  : trialnum,
            "fold"   : foldid}
    for i,item in enumerate(outdir):
        if isinstance(item, tuple):
            name, patt = item
            if args[name] is None:
                outdir[i] = None
            else:
                outdir[i] = patt % args[name]
    instdir = "/".join([part for part in outdir if part is not None])
    return instdir

def load_metrics(filename):
    metrics = {}
    with open(filename) as f:
        groupnames = f.readline().rstrip().split()
        for line in f:
            line = line.rstrip().split()
            for i,val in enumerate(line[1:]):
                metrics.setdefault(groupnames[i],{})[line[0]] = val
    return metrics

def save_metrics(outfile, metrics):
    with open(outfile,"w") as f:
        groupnames = sorted(metrics.keys())
        fieldnames = set()
        for groupname in groupnames:
            for fieldname in metrics[groupname].keys():
                fieldnames.add(fieldname)
        fieldnames = sorted(list(fieldnames))

        f.write(" "*14+"\t".join(groupnames) + "\n")
        rows = {}
        for groupname in groupnames:
            for fieldname in fieldnames:
                fieldval = metrics[groupname].setdefault(fieldname, np.nan)
                if not isinstance(fieldval,np.ndarray):
                    if isinstance(fieldval, float):
                        fmt = "%.2e" if fieldname.endswith(".p") else "%.6f"
                        fieldval = fmt%fieldval
                    rows.setdefault(fieldname,[]).append(str(fieldval))
        f.writelines([fieldname + " "*max(0,14-len(fieldname)) + "\t".join(rows[fieldname]) +"\n" for fieldname in fieldnames])

def call_dumpviz(dumpdir):
    subprocess.Popen(["python", os.path.dirname(__file__)+"/dumpviz.py", dumpdir])

##########################################

class hypertrain_worker(object):
    """
    Given a dataset and specific hyperparameters, this object will
    simply train a model (an array of models) and return the
    validation error (array of validation errors).
    """
    def __init__(self, worker_id, model_proto, trainer_proto, datasrc,
                                  nfold, allfolds, outdir, report_class, devices, verbose,
                                  default_dtype, global_flags, auxfilter, mode, dumpviz):

        self.worker_id = worker_id
        # All the data subsets in 'trainset' will be merged into a single fold.
        self.model_proto   = model_proto
        self.trainer_proto = trainer_proto
        self.datasrc = datasrc    # Load a copy of the dataset into this worker process.
        self.nfold  = nfold
        self.allfolds = allfolds
        self.outdir  = outdir
        self.mode = mode
        self.aucrange = (0.5,0.5)  # threshold for making AUCs out of non-binary targets, presumed to be in range [0,1]
        self.report_class = report_class
        self.auxfilter = auxfilter
        self.dumpviz = dumpviz

        globals.flags.copy_from(global_flags)

        # If we've been called from a new process, create a separate log file.
        # Otherwise everything is logged into the original log file.
        if multiprocessing.current_process().name != "MainProcess":
            logdir = getinstdir(outdir,None,None,None)
            worker_logfile = os.path.join(logdir,"hypertrain_worker%d.log" % worker_id)
            globals.set_logging(worker_logfile,level=verbose,echo=False)
            logging.info("\n----------------------------- %s -----------------------------" % time.strftime("%y-%m-%d %H-%M-%S",time.localtime()))

        # Configure deepity to use this worker's GPU device.
        logging.info("worker %d starting on device %d using %s" % (worker_id,devices[worker_id],sm.get_default_dtype().__name__))
        rseed = int((time.time()*100000 + worker_id)  % 2000)
        globals.reset_backend(device=devices[worker_id], seed=rseed)
        random.seed(rseed)
        sm.set_default_dtype(default_dtype)
        npr.seed(rseed)

        # Seed this process's random number generator, for reproducibility
        sm.sync()

        # Prepare the datasource to serve data.
        self.datasrc.open()

    def __del__(self):
        self.datasrc.close()
        self.datasrc = None
        gc.collect()  # Clear out the cruft and make sure the backend can be destroyed
        sm.sync()
        sm.destroy_backend()

    def __call__(self, hparams, task_ids, sample_ids):

        # Determine what kind of targets we want to train on
        data = self.datasrc.astargets(task_ids) # Copies of arbitrary targets
        data = data[:] # Copy so that when we normalize etc we don't affect the original data

        # Normalize the targets. For logisitic-output models this means
        # scaling targets to [0,1]. For other models this means scaling
        # targets to have mean=0, variance=1.
        data.requirements = self.model_proto.data_requirements()
        #print np.percentile(data.Y[data.Ymask].ravel(), [99, 99.99, 99.995, 99.999])
        #print data.Y.size, int(data.Y.size*(100-99.95)/100)
        if "clamp_targets" in globals.flags:
            data.clamp_extremes(0.0,99.95)
        if "normalize_targets" in globals.flags:
            data.normalize_targets()
            #data.arcsinhtransform_targets()

        if self.mode != 'calib':
            # If we're not in calibration mode, then there's no need for multiple checkpoints
            #  -- just keep the last checkpoint so that it can be dumped to disk
            #del hparams["trainer:checkpoints"]
            self.trainer_proto.checkpoints = 1

        # Shuffle the individual rows of data, always the same random shuffle
        # and therefore always the same random split each time the code is run.
        data.shuffle()

        # Create a callback handler to collect predictions and evaluate final performance
        checkpoints = self.report_class()

        # Perform k-fold cross validation (k=nfold), training with one fold held out at a time.
        for foldid in range(self.nfold):
            checkpoints.setfold(foldid)  # Tell the checkpoint

            # Create a new model and trainer with the given hyperparams
            model   = _create_model(self.model_proto,     hparams)
            trainer = _create_trainer(self.trainer_proto, hparams)

            # Split the data into training and validation sets
            trdata, vadata = data.split(foldid, self.nfold-1)
            trdata = trdata.augmented(trdata)
            datasets = { "train" : trdata }
            if vadata:
                vadata = vadata.augmented(vadata)
                datasets["validate"] = vadata
                if self.auxfilter:
                    datasets["validate_aux"] = vadata[[i for i in range(len(vadata)) if vadata.foldids[i] in self.auxfilter]]
            for dset in datasets.values():
                dset.requirements = model.data_requirements()

            #if not checkpoint_callback:
            #    trainer.viz_steps = False # Disable periodic updates if no reports

            # Train the model and remember how well it performed.
            trainer.train(model, datasets, checkpoints)

            if self.mode == 'train' and self.nfold > 1:
                entries = checkpoints.curr()
                metrics = self.calc_metrics(entries)
                self.save_model(model, hparams, task_ids, sample_ids, foldid)
                self.save_metrics(metrics, task_ids, sample_ids, foldid)
                self.save_predictions(entries, task_ids, sample_ids, foldid)
                self.call_dumpviz(task_ids, sample_ids, foldid)

            # If we`re only supposed to try one fold, then don`t bother looping over the other splits
            if not self.allfolds:
                break

        # Consolidate the separate folds, and dump them if need be
        entries = checkpoints.combined()

        # Calculate the performance stats associated with each target
        metrics = self.calc_metrics(entries)

        # Save the current model and predictions
        if self.mode == 'train':
            self.save_predictions(entries, task_ids, sample_ids, None)
            self.save_metrics(metrics, task_ids, sample_ids, None)
            if self.nfold == 1:
                self.save_model(model, hparams, task_ids, sample_ids, None)
                self.save_preprocessors(data, task_ids, sample_ids, None)
            #self.call_dumpviz(task_ids, sample_ids, None)

        # Return a new hparams object with the performance incorporated
        hpsearch_result = self.add_hparam_metrics(hparams, metrics)
        return hpsearch_result

    def save_model(self, model, hparams, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            util.makepath(dumpdir)

            # Slice out model i and save it to disk
            _save_model_inst(dumpdir+"/model.pkl", i, model, hparams)

    def save_predictions(self, entries, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            util.makepath(dumpdir)

            # Save out the predictions for model i
            assert len(entries[i]) == 1, "Bug. Expected only a single unique 'step' in the list of entries"
            groups = entries[i].values()[0]
            np.savez_compressed(dumpdir+"/predict.npz",
                                targetname=np.asarray(taskid, dtype=object),
                                groups=np.asarray(groups, dtype=object))

    def save_metrics(self, metrics, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            util.makepath(dumpdir)

            # Save out the predictions for model i
            assert len(metrics[i]) == 1, "Bug. Expected only a single unique 'step' in the list of entries"
            groups = metrics[i].values()[0]
            save_metrics(dumpdir+"/metrics.txt", groups)

    def call_dumpviz(self, task_ids, sample_ids, foldid):
        if not self.dumpviz:
            return
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            call_dumpviz(dumpdir)

    def save_preprocessors(self, data, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            data.dump_preprocessors(dumpdir, slice(i,i+1))

    def add_hparam_metrics(self, hparams, metrics):
        groupkey = "validate" if "validate" in metrics[0].values()[0] else "train"
        hpsearch_result = {}
        for i in metrics:
            for step in metrics[i]:
                hparams_i = { key : val[i] for key,val in hparams.iteritems() }
                hparams_i["trainer:max_steps"] = step
                metrics_i = metrics[i][step][groupkey]
                hpsearch_result.setdefault(i,[]).append((hparams_i, metrics_i)) # Thus tuple is returned to hpsearch
        return hpsearch_result

        """
            if "vloss" in stats and stats["vloss"] is not None:
                loss.append(stats["vloss"])
                auc.append(stats["vauc"])
            else:
                loss.append(stats["tloss"])
                auc.append(stats["tauc"])

            if self.testfilter is not None:
                tidx = [i for i in range(len(vdata)) if vdata.foldids[i] in self.testfilter]
                tdata = vdata[tidx]
                tpred = gen_predictions(model, tdata)
                testauc,teststd = bootstrap_auc(tpred.ravel(), tdata.Y.ravel(), ntrial=20)
                flogfile = self.outdir + "/%s_%04d/fold%d.log" % (task_ids[0], sample_ids[0], foldid)
                with open(flogfile) as fh:
                    flog = fh.readlines()
                flog[-1] = flog[-1].rstrip() + "\ttestAUC=%.3f (%f)\n" % (testauc,teststd)
                with open(flogfile,"w") as fh:
                    fh.writelines(flog)
                testaucs.append((testauc, teststd))

            if report:
                reports.append(report)
                report.dump(want_html=True)
                #report.dump(want_html=self.want_html)

                # Dump each model to a separate file
                for inst in range(len(sample_ids)):
                    filename = self.outdir + ("/%s_%04d/fold%d.model.pkl" % (task_ids[inst], sample_ids[inst], foldid))
                    _save_model_inst(filename, inst, model, hparams)

            """
            #break
        """"
        if reports != []:
            # Dump the separate (individual) hyperparams that were used for each instance trained
            for inst in range(len(sample_ids)):
                dumpdir = self.outdir + ("/%s_%04d/" % (task_ids[inst], sample_ids[inst]))
                vloss = self.validation_performance[task_ids[inst]] if self.validation_performance else None
                _dump_hparams(dumpdir, _slice_hparams(hparams,inst), vloss)
                tdata.dump_preprocessors(dumpdir, slice(inst,inst+1))
            merged = self.report_class.merge_reports(self.outdir + "/%(task_id)s_%(sample_id)04d/final.log", task_ids, sample_ids, reports)
            #merged.dump(want_html=self.want_html)
            merged.dump()
            if testaucs:
                flogfile = self.outdir + "/%s_%04d/final.log" % (task_ids[0], sample_ids[0])
                with open(flogfile) as fh:
                    flog = fh.readlines()
                testauc = sum([_auc for _auc, _std in testaucs]) / len(testaucs)
                teststd = sum([_std for _auc, _std in testaucs]) / len(testaucs)
                flog[-1] = flog[-1].rstrip() + "\ttestAUC=%.3f (%f)\n" % (testauc,teststd)
                with open(flogfile,"w") as fh:
                    fh.writelines(flog)

        # Average the loss over each fold
        loss = np.mean(np.asarray(loss),axis=0)
        auc  = np.mean(np.asarray(auc),axis=0)

        # Dump each average loss and corresponding hyperparameters into a log file
        for inst in range(len(sample_ids)):
            util.makepath(self.outdir+"/hpsearch")
            with open(self.outdir+"/hpsearch/%s.log"%task_ids[inst],"a") as f:
                f.write("%.6f\t%.4f\t%s\n"%(loss[inst], auc[inst], hparams2str( _slice_hparams(hparams,inst) ).replace("\n",";")) )
                """
        sm.sync()

        # Return a list of objective values, one per search_id
        values = [float(x) for x in loss]
        return values

    def calc_metrics(self, entries):
        metrics = {}
        for taskidx in entries:
            for step in entries[taskidx]:
                for group in entries[taskidx][step]:
                    entry = entries[taskidx][step][group]
                    Z = entry["Z"]
                    Y = entry["Y"]

                    # Start computing stats
                    metric = metrics.setdefault(taskidx,{}).setdefault(step,{}).setdefault(group,{})
                    metric["loss"] = entry["L"]
                    if Z.shape[1] == 1:
                        metric.update(deepity.calc_metrics(Z.ravel(), Y.ravel(), self.aucrange))
        return metrics

def hparams2str(params):
    txt = ""
    for key in sorted(params.keys()):
        value = params[key]
        if isinstance(value, np.ndarray) and value.size > 10:
            value = "ndarray"
        txt += "  %s = %s\n" % (key + " "*max(0,20-len(key)),value)
    return txt

#######################################

def hypertrain(model, trainer, data,
               nfold=2, allfolds=True, outdir=None, nsample=20,
               devices=None, verbose=None, report_class=None,
               auxfilter=None):

    if report_class is None: report_class = training_report

    # Create the output directory if it doesn't already exist.
    if outdir is None:
        outdir = join(tempfile.gettempdir(),"hypertrain")

    # Define the search space
    space = _get_hypertrain_searchspace(model, trainer)

    # Perform the search, returning the best parameters in the search space.
    logging.info("calibrating...")
    samples = hp.search(space,
                        objective  = hypertrain_worker,
                        objective_initargs = (model,trainer,data,nfold,allfolds,outdir,report_class,devices,False,sm.get_default_dtype(),globals.flags,auxfilter,"calib",False),
                        task_ids = data.targetnames,
                        nsample  = nsample,
                        nprocess = len(devices),
                        nsample_per_process = 15,
                        print_progress = True)
    logging.info("...calibrating done")

    return samples

###########################################

def train(model, trainer, data, hparams=None, hparams_metric=None,
          nfold=1, outdir=None, nsample=1,
          devices=None, verbose=None, report_class=None,
          auxfilter=None, dumpviz=True):

    if report_class is None: report_class = training_report

    if hparams:
        for targetname in data.targetnames:
            for sample in range(nsample):
                for fold in range(nfold):
                    save_hparams_result(getinstdir(outdir, targetname, sample, fold)+"/calib.txt", hparams[targetname], hparams_metric)

    space = _get_fixed_searchspace(model, trainer, data.targetnames, hparams)

    #space = _get_hypertrain_searchspace(model, trainer)

    #if space and not hparams:
    #    raise ValueError("The given model has undetermined hyperparamters. Must call hypertrain first.")

    # Replace the randomly sampled hparams with fixed values specified by 'hparams'
    #for pname in space._pdefs.iterkeys():
    #    pbest = np.asarray([hparams[task_id].params[pname] for task_id in data.targetnames])
    #    space._pdefs[pname] = hp.fixed(pbest, pname)
        #print "assigning hparam",pname,"<-",pbest

    final_outdir = outdir
    logging.info("train...")
    hp.search(space,
              objective  = hypertrain_worker,
              objective_initargs = (model,trainer,data,nfold,True,final_outdir,report_class,devices,verbose,sm.get_default_dtype(),globals.flags,auxfilter,"train",dumpviz),
              task_ids = data.targetnames,
              nsample  = nsample,
              nsample_per_process = 2,#len(data.targetnames),  # Hack: only train numtargets models at a time, to ensure that when nsample>1 the next sample gets a different minibatch order
              nprocess = len(devices))
    logging.info("...train done")

#######################################################

def _get_fixed_searchspace(model, trainer, targetnames, hparams):
    pdefs = []
    if hparams:
        # Convert the hparams list-of-dictionaries (all dictionaries having same key)
        # into a single dictionary-of-lists
        hpvec = {}
        for targetname in targetnames:
            sample = hparams[targetname]
            for pkey in sample.params:
                hpvec.setdefault(pkey,[]).append(sample.params[pkey])
        for key in hpvec:
            pdefs.append(hp.fixed(np.array(hpvec[key]), key))
    space = hp.space(pdefs)
    return space

def _get_hypertrain_searchspace(model, trainer):
    # First, collect all hparams by visiting the model's dependency graph
    model_hparams = []
    def collect_hparam(path,attr):
        if isinstance(attr,hp.paramdef):
            attr.name = "model:" + path  # model:...path
            model_hparams.append(attr)
    model.visit(collect_hparam)

    # Next, ask the trainer for its hyperparams, and put a "trainer." prefix on the name of each one
    # so that they don't conflict with model_hparams
    trainer_hparams = []
    for name,attr in trainer.__dict__.iteritems():
        if isinstance(attr,hp.paramdef):
            attr.name = "trainer:" + name  # trainer:...path
            trainer_hparams.append(attr)

    # Return a search space built from model and trainer hyperparams
    return hp.space(trainer_hparams + model_hparams)


from copy import deepcopy
from flask import abort
from flask import Blueprint
from flask import request
from flask_api import status

from oneview_redfish_toolkit.api.event import Event
from oneview_redfish_toolkit.api.event_service import EventService
from oneview_redfish_toolkit.blueprints.util.response_builder import \
    ResponseBuilder
from oneview_redfish_toolkit import config
from oneview_redfish_toolkit import util

event_service = Blueprint("event_service", __name__)

ONEVIEW_TEST_ALERT = {
    "timestamp": "2018-02-12T20:12:03.231Z",
    "resource": {
        "category": "alerts",
        "associatedResource": {
            "resourceName": "0000A66101, bay 3",
            "resourceUri": "/rest/server-hardware/"
                           "30373737-3237-4D32-3230-313530314752"
        }
    }
}

ONEVIEW_TEST_TASK = {
    "timestamp": "2018-02-12T20:12:03.231Z",
    "resourceUri": "/rest/server-hardware/"
                   "30373737-3237-4D32-3230-313530314752",
    "changeType": None,
    "resource": {
        "category": "server-hardware",
        "name": "0000A66101, bay 3"
    }
}

REDFISH_TO_ONEVIEW_EVENTS = {
    "ResourceAdded": "Created",
    "ResourceUpdated": "Updated",
    "ResourceRemoved": "Deleted"
}

@event_service.route("/redfish/v1/EventService/", methods=["GET"])
def get_event_service():
    """Get the Redfish Event Service.

        Get method to return EventService JSON when
        /redfish/v1/EventService is requested.

        Returns:
            JSON: JSON with EventService.

    """
    evs = EventService(util.get_delivery_retry_attempts(),
                       util.get_delivery_retry_interval())

    return ResponseBuilder.success(evs)

@event_service.route(
    "/redfish/v1/EventService/Actions/EventService.SubmitTestEvent/",
    methods=["POST"])
def execute_test_event_action():
    """Executes the SubmitTestEvent Action

        Return a JSON containing the EventType received.
        Logs exception of any error and return abort.

        Returns:
            JSON: JSON containing the EventType.

        Exceptions:
            Exception: Missing EventType property.
            Return Bad Request status(400)
    """

    if not config.auth_mode_is_conf():
        abort(status.HTTP_404_NOT_FOUND,
              "EventService is not enabled.")

    event_type = None
    try:
        event_type = request.get_json()['EventType']
    except Exception:
        abort(status.HTTP_400_BAD_REQUEST,
              'Invalid JSON data. Missing EventType property.')

    if event_type not in util.get_subscriptions_by_type().keys():
        abort(status.HTTP_400_BAD_REQUEST,
              'Invalid EventType value: %s' % event_type)

    # Creates a sample OneView SCMB message according to
    # the value of 'event_type'
    if event_type == "Alert":
        message = deepcopy(ONEVIEW_TEST_ALERT)
    else:
        message = deepcopy(ONEVIEW_TEST_TASK)
        message['changeType'] = REDFISH_TO_ONEVIEW_EVENTS[event_type]

    event = Event(message)

    util.dispatch_event(event)

    return ResponseBuilder.response(event, status.HTTP_202_ACCEPTED)