c2t_cfg = C2TConfig(
    rsp_target = DebugClient(
        march = "4kc",
        new_rsp = get_new_rsp(
            regs = ["r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9",
                "r10", "r11", "r12", "r13", "r14", "r15", "r16", "r17", "r18",
                "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27",
                "r28", "r29", "r30", "r31", "status", "lo", "hi", "badvaddr",
                "cause", "pc", "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
                "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16",
                "f17", "f18", "f19", "f20", "f21", "f22", "f23", "f24", "f25",
                "f26", "f27", "f28", "f29", "f30", "f31", "fcsr", "fir",
                "restart"],
            pc = "pc",
            regsize = 32
        ),
        test_timeout = 20.,
        sp = "r29"
    ),
    qemu = DebugServer(Run(
        executable = "/usr/bin/qemu-system-mipsel",
        args = ("-m 256 -M mips -cpu 4Kc -kernel {bin} -S "
            "-gdb tcp:localhost:{port} -nographic"
        )
    )),
    gdbserver = DebugServer(Run(
        executable = "/usr/bin/gdbserver",
        args = "localhost:{port} {bin}"
    )),
    target_compiler = TestBuilder(
        Run( # compiler
            executable = "/usr/bin/mipsel-linux-gnu-gcc",
            args = "-g -O0 -march=4kc {src} -o {bin}"
        )
    ),
    oracle_compiler = TestBuilder(
        Run( # compiler
            executable = "/usr/bin/gcc",
            args = "-g -O0 {src} -o {bin} -no-pie"
        )
    )
)


# encoding=utf-8

from django.conf.urls import url
from views import (
    PublicJobsViewSet,
    OwnJobsViewSet,
    AreaViewSet,
    DetailedAreaViewSet,
    TechnologyViewSet,
    HierarchyViewSet,
    RoleViewSet,
    JobPostulationViewSet,
    OwnJobPostulationsViewSet,
    JobQuestionsFormView,
    NewJobFormView,
    KnowledgeRequirementFormView,
    QuestionFormView,
    OwnJobsInfoView,
    AdvanceSearchFiltersChoices,
    RelatedJobsViewSet,
    UserPostulationsJobs,
    EnterpriceJobs,
    SetFavoriteJob
)

list_only = {
    'get': 'list'
}

retrieve_only = {
    'get': 'retrieve'
}

get_post = {
    'get': 'list',
    'post': 'create'
}

get_update = {
    'get': 'retrieve',
    'put': 'update'
}

update_delete = {
    'get': 'retrieve',
    'put': 'update',
    'delete': 'destroy'
}

post_only = {
    'post': 'create'
}

urlpatterns = [

    url(
        r"^enterpricejobs$",
        EnterpriceJobs.as_view()
    ),
    url(
        r"^userpostulationjobs$",
        UserPostulationsJobs.as_view()
    ),
    url(
        r"^setfavoritejob$",
        SetFavoriteJob.as_view()
    ),
    url(
        r"^jobs/(?P<pk>[0-9]+)$",
        RelatedJobsViewSet.as_view({'get': 'mretrieve'})
    ),
    url(
        r"^jobs/?$",
        PublicJobsViewSet.as_view(list_only),
        name="public_jobs"
    ),
    url(
        r"^jobs/by_area/(?P<area_pk>[0-9]+)/?$",
        PublicJobsViewSet.as_view({
            "get": "by_area"
        })
    ),
    url(
        r"^jobs/with_roles/?$",
        PublicJobsViewSet.as_view({
            "get": "with_roles"
        })
    ),
    url(
        r"^jobs/(?P<job_pk>[0-9]+)/related?$",
        RelatedJobsViewSet.as_view(list_only),
        name="job_detail"
    ),
    url(
        r"^jobs/(?P<job_pk>[0-9]+)/postulate/?$",
        JobPostulationViewSet.as_view(post_only),
    ),
    url(
        r"^own_info$",
        OwnJobsInfoView.as_view()
    ),
    url(
        r"^own/?$",
        OwnJobsViewSet.as_view(get_post),
        name="own_jobs"
    ),
    url(
        r"^own/(?P<pk>[0-9]+)/?$",
        OwnJobsViewSet.as_view(update_delete),
        name="own_job_detail"
    ),
    url(
        r"^own/(?P<job_pk>[0-9]+)/postulations/?$",
        OwnJobPostulationsViewSet.as_view(list_only)
    ),
    url(
        r"^own/(?P<job_pk>[0-9]+)/postulations/(?P<pk>[0-9]+)/?$",
        OwnJobPostulationsViewSet.as_view({
            "get": "retrieve",
            "put": "set_status"
        })
    ),
    url(
        r"^own/(?P<job_pk>[0-9]+)/postulations/(?P<pk>[0-9]+)/favorite$",
        OwnJobPostulationsViewSet.as_view({
            "put": "set_favorite"
        })
    ),
    url(
        r"^own/lasts$",
        OwnJobsViewSet.as_view({"get": "get_lasts"}),
    ),
    url(
        r"^areas/?$",
        AreaViewSet.as_view(list_only)
    ),
    url(
        r"^areas/(?P<pk>[0-9]+)$",
        DetailedAreaViewSet.as_view(retrieve_only)
    ),
    url(
        r"^areas/(?P<slug>[a-z0-9_-]+)$",
        DetailedAreaViewSet.as_view({"get": "by_slug"})
    ),
    url(
        r"^technologies/?$",
        TechnologyViewSet.as_view(list_only)
    ),
    url(
        r"^technologies/(?P<pk>[0-9]+)/?$",
        TechnologyViewSet.as_view(retrieve_only)
    ),
    url(
        r"hierarchies/?$",
        HierarchyViewSet.as_view(list_only)
    ),
    url(
        r"^roles/?$",
        RoleViewSet.as_view(list_only)
    ),
    url(
        r"^as_choices/?$",
        AdvanceSearchFiltersChoices.as_view()
    ),
    # Forms
    url(
        r"^jobs/(?P<job_pk>[0-9]+)/postulate_form.html$",
        JobQuestionsFormView.as_view(),
    ),
    url(
        r"^templates/new_job.html$",
        NewJobFormView.as_view()
    ),
    url(
        r"^templates/new_knowledge_requirement.html$",
        KnowledgeRequirementFormView.as_view()
    ),
    url(
        r"^templates/new_question.html",
        QuestionFormView.as_view()
    )
]


__all__ = ["two_fluid_dispersion_solution"]

import astropy.units as u
import numpy as np

from astropy.constants.si import c, e, k_B, m_e, m_p, mu0

import plasmapy.formulary.parameters as pfp

from plasmapy.utils.decorators import validate_quantities

@validate_quantities(
    B={"can_be_negative": False},
    m_e={"can_be_negative": False},
    m_i={"can_be_negative": False},
    n={"can_be_negative": False},
    T_e={"can_be_negative": False, "equivalencies": u.temperature_energy()},
    T_i={"can_be_negative": False, "equivalencies": u.temperature_energy()},
    theta={"can_be_negative": True},
)
def two_fluid_dispersion_solution(
    B: u.T,
    k: u.m ** -1,
    n: u.m ** -3,
    T_e: u.K,
    T_i: u.K,
    gamma_e=1,
    gamma_i=3,
    ion="p+",
    m_e: u.kg = m_e,
    m_i: u.kg = m_p,
    theta: u.deg = 45 * u.deg,
    z=1,
):
    r"""
    Return a dictionary dictionary of frequencies corresponding to the
    solutions of dispersion relation in low frequency regime.

    **Aliases:** `tfds_`

    Parameters
    ----------
    B : ~astropy.units.Quantity
        Magnetic field
    gamma_e : float or int
        The adiabatic index for electrons, which defaults to 1.  This
        value assumes that the electrons are able to equalize their
        temperature rapidly enough that the electrons are effectively
        isothermal.
    gamma_i : float or int
        The adiabatic index for ions, which defaults to 3. This value
        assumes that ion motion has only one degree of freedom, namely
        along magnetic field lines.
    ion : string, optional
        Representation of the ion species (e.g., `'p'` for protons,
        `'D+'` for deuterium, or 'He-4 +1' for singly ionized
        helium-4), which defaults to protons.  If no charge state
        information is provided, then the ions are assumed to be
        singly charged.
    k : ~astropy.units.Quantity
        Wave number
    m_e : ~astropy.units.Quantity
        Mass of negative ion which defaults to electron
    m_i : ~astropy.units.Quantity
        Mass of positive ion which defaults to proton
    n : ~astropy.units.Quantity
        Number density.
    T_e : ~astropy.units.Quantity
        The electron temperature
    T_i : ~astropy.units.Quantity
        The ion temperature
    theta : ~astropy.units.Quantity
        Angle of propagation which defaults to 45 degrees.
    z : float or integer
        Average ionization number which defaults to 1 for protons.

    Returns
    -------
    omega : ~astropy.units.Quantity
        A dictionary of Wave frequencies corresponding to three modes namely
        a) Ion-acoustic mode, b) Alfven mode and c) Fast mode, in 1/s units

    Notes
    -----
    Computes the solution for wave dispersion relation based on equation 38 of
    Bellan2012JGR (doi:10.1029/2012JA017856)

    .. math::

        \frac{\omega}{\omega_{ci}} = \sqrt(2\Lambda \sqrt(-\frac{p}{3}) cos
        \left( \frac{1}{3} cos^{-1}\left( \frac{3q}{2p} \sqrt(-\frac{3}{p}
        \right) - \frac{2\pi}{3}j \right) + \frac{A\Lambda}{3})

    Where,
        j = 0 ==> fast mode
        j = 1 ==> Alfven mode
        j = 2 ==> Acoustic mode

    The above equation is derived from the general wave equation in the low
    frequency regime, where both electrons and ions play significant role (in
    the high frequency regime ions do not play any significant role and the
    process is dominated by electron dynamics).

    The complete dispersion equation is thus written as (from equation (1) of
    Bellan2012JGR):

    .. math::
        \left( cos^2\theta - Q\frac{\omega^2}{k^2 {v_A}^2} \right) \left[
        \left( cos^2\theta - \frac{\omega^2}{k^2 {c_s}^2 \right) -
        Q\frac{\omega^2}{k^2 {v_A}^2 \left( 1 - \frac{\omega^2}{k^2 {c_s}^2
        \right) \right] = \left(1 - \frac{\omega^2}{k^2 {c_s}^2 \right)
        \frac{\omega^2}{{\omega_{ci}}^2}cos^2\theta

    Here,
    .. math::
        Q = 1 + k^2 c^2/{\omega_{pe}}^2
    \omega_{ci} is the proton gyrofrequency

    References
    ----------
    .. [Bellan2012JGR] PM bellan, Improved basis set for low frequency plasma
    waves, 2012, JGR, 117, A12219, doi:10.1029/2012JA017856
    .. [Stringer1963JNE] TE Stringer, Low-frequency waves in an unbounded
    plasma, 1963, JNE, Part C, doi:10.1088/0368-3281/5/2/304
    .. [Rogers2001PRL] Rogers, B. N.; Denton, R. E.; Drake, J. F. & Shay, M. A.
    Role of Dispersive Waves in Collisionless Magnetic Reconnection, prl, 2001,
    87, 195004

    Examples
    --------
    >>> from astropy import units as u
    >>> from plasmapy.dispersion import two_fluid_dispersion_solution as tfds
    >>> k = 0.01 * u.m ** -1
    >>> theta = 30 * u.deg
    >>> B = 8.3E-9 * u.T
    >>> T_e = 1.6e6 * u.K
    >>> T_i = 4.e5 * u.K
    >>> z = 1
    >>> omega = tfds(B, k, n, T_e, T_i, theta, z)
    {'fast_mode': <Quantity [[1520.57692532]] rad / s>,
    'alfven_mode': <Quantity [[1260.01546096]] rad / s>,
    'acoustic_mode': <Quantity [[0.68815159]] rad / s>}

    >>> k = np.linspace(10**-7, 10**-2, 1E4) * u.m ** -1
    >>> theta = np.linspace(5, 85, 100) * u.deg
    >>> n = 5 * u.cm ** -3
    >>> B = 8.3E-9 * u.T
    >>> T_e = 1.6e6 * u.K
    >>> T_i = 4.e5 * u.K
    >>> z = 1
    >>> c = 3.e8 * u.m/u.s
    >>> c_s = pfp.ion_sound_speed(T_e=T_e, T_i=T_i, n_e=z * n)
    >>> v_A = pfp.Alfven_speed( B, n, ion='p+')
    >>> omega_ci = pfp.gyrofrequency(B=B, particle='p+', signed=False, Z=z)
    >>> omega = two_fluid_dispersion_solution(n=n, B=B, T_e=T_e, T_i=T_i,
        theta=theta, z=z, k=k)
    >>> omega['fast_mode'][:,40]
        [0.016117629, 0.17733531, 0.33868854, … , 1520.3016, 1520.4535,
        1520.6055] rad/s

    """
    # Required derived parameters
    # Compute the ion sound speed using the function from
    # plasmapy.formulary.parameters
    c_s = pfp.ion_sound_speed(
        T_e=T_e, T_i=T_i, n_e=z * n, gamma_e=gamma_e, gamma_i=gamma_i, ion=ion
    )

    # Compute the ion Alfven speed using the function from
    # plasmapy.formulary.parameters
    v_A = pfp.Alfven_speed(B, n, ion=ion)

    # Compute the ion gyro frequency using the function from
    # plasmapy.formulary.parameters
    omega_ci = pfp.gyrofrequency(B=B, particle="p+", signed=False, Z=z)

    # Compute the electron plasma frequency using the function from
    # plasmapy.formulary.parameters
    omega_pe = pfp.plasma_frequency(n=n, particle="e-", z_mean=z)

    # Compute the dimensionless parameters corresponding to equation 32 of
    # Bellan2012JGR
    alpha = (np.cos(theta.to("rad")) ** 2).value
    beta = (c_s ** 2 / v_A ** 2).value

    # Create a meshgrid of direction of propagation and the wavenumber
    alphav, kv = np.meshgrid(alpha, k)

    Lambda = (kv ** 2 * v_A ** 2 / omega_ci ** 2).value

    # Compute the dimensionless parameters corresponding to equation 2 of
    # Bellan2012JGR
    Q = 1 + (kv ** 2 * c ** 2 / omega_pe ** 2).value

    # Compute the dimensionless parameters corresponding to equation 35 of
    # Bellan2012JGR
    A = (Q + Q ** 2 * beta + Q * alphav + alphav * Lambda) / Q ** 2
    B = alphav * (1 + 2 * Q * beta + Lambda * beta) / Q ** 2
    C = alphav ** 2 * beta / Q ** 2

    # Compute the dimensionless parameters corresponding to equation 36 of
    # Bellan2012JGR
    p = (3 * B - A ** 2) / 3
    q = (9 * A * B - 2 * A ** 3 - 27 * C) / 27

    # List out the three wave modes for which this function gives the
    # frequencies
    keys = ["fast_mode", "alfven_mode", "acoustic_mode"]

    # Create a dictionary with the wave mode names as its keys
    omega = dict.fromkeys(keys)

    # Compute the value of  omega for each key and for each value of wavenumber
    # and direction of propagation
    for (j, key) in zip(range(3), keys):
        # The solution corresponding to equation 38
        omega[key] = omega_ci * np.sqrt(
            2
            * Lambda
            * np.sqrt(-p / 3)
            * np.cos(
                1 / 3 * np.arccos(3 * q / (2 * p) * np.sqrt(-3 / p)) - 2 * np.pi / 3 * j
            )
            + Lambda * A / 3
        )

    return omega

tfds_ = two_fluid_dispersion_solution
""" Alias to :func:`two_fluid_dispersion_solution`. """


from typing import List
from uuid import UUID, uuid4

import responses

from darwin.future.core.client import Client
from darwin.future.core.items.get import get_item_ids, get_item_ids_stage
from darwin.future.tests.core.fixtures import *
from darwin.future.tests.core.items.fixtures import *

def test_get_item_ids(UUIDs: List[UUID], UUIDs_str: List[str], base_client: Client) -> None:
    with responses.RequestsMock() as rsps:
        rsps.add(
            rsps.GET,
            base_client.config.api_endpoint + f"v2/teams/default-team/items/ids?not_statuses=archived,error&sort[id]=desc&dataset_ids=1337",
            json={"item_ids": UUIDs_str},
            status=200,
        )
        item_ids = get_item_ids(base_client, "default-team", "1337")
        assert item_ids == UUIDs

def test_get_item_ids_stage(UUIDs: List[UUID], UUIDs_str: List[str], base_client: Client) -> None:
    stage_id = str(uuid4())
    with responses.RequestsMock() as rsps:
        rsps.add(
            rsps.GET,
            base_client.config.api_endpoint + f"v2/teams/default-team/items/ids?workflow_stage_ids={stage_id}&dataset_ids=1337",
            json={"item_ids": UUIDs_str},
            status=200,
        )
        item_ids = get_item_ids_stage(base_client, "default-team", "1337", stage_id)
        assert item_ids == UUIDs


import sqlite3 as sql
import csv

con = sql.connect("movies.db")
cur = con.cursor()
cur.execute("CREATE TABLE MoviesTable (tconst, titleType, primaryTitle, originalTitle, isAdult, startYear, endYear, runtimeMinutes, genres);")

with open('movies.tsv', 'r', encoding ='utf8') as movies:
    dr = csv.DictReader(movies, delimiter="\t")
    to_db = to_db = [(i['tconst'], i['titleType'], i['primaryTitle'], i['originalTitle'], i['isAdult'], i['startYear'], i['endYear'], i['runtimeMinutes'], i['genres']) for i in dr]

cur.executemany("INSERT INTO  MoviesTable (tconst, titleType, primaryTitle, originalTitle, isAdult, startYear, endYear, runtimeMinutes, genres) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);", to_db)
con.commit()
con.close()


#Jonathan Balina
import math
import csv
import random

def normalize(df):
    for i in range(0, len(df)):
        for j in range(1, 7):
            df[i][j] = (df[i][j] - 1) / 5

    return df

def load(file):
    with open(file) as csvfile:
        csvReader = csv.reader(csvfile, delimiter=" ")
        data = []
        for row in csvReader:

            row.pop()
            row.pop(0)
            # print(x)
            d = []

            for r in row:
                d.append(int(r))
            data.append(d)
        return data

def read_csv(file):
    with open(file) as csvfile:
        csvReader = csv.reader(csvfile)
        data = []
        for row in csvReader:
            row.pop()
            row.pop(0)
            # print(x)
            d = []
            for r in row:
                d.append(int(r))
            data.append(d)
        return data

header = ["class", "a1", "a2", "a3", "a4", "a5", "a6", "a7"]

def unique_items(data, col):
    return set([item[col] for item in data])

def num_classes(data):
    classes = {}
    for item in data:
        class_item = item[0]
        if class_item not in classes:
            classes[class_item] = 0
        classes[class_item] += 1
    return classes

def majority_class(data):
    max_key = 0
    max_count = 0
    for key in num_classes(data):
        if num_classes(data)[key] > max_count:
            max_key = key
            max_count = num_classes(data)[key]
    return max_key

class Comparison:
    def __init__(self, col):
        self.col = col

def split(data, col):
    split_data = {}
    values = set([item[col] for item in data])
    for val in values:
        classcount = {}
        new_data = []
        class1 = 0
        class2 = 0
        for item in data:
            if item[col] == val:
                new_data.append(item)
                if item[0] == 0:
                    class1 += 1
                elif item[0] == 1:
                    class2 += 1
        classcount["data"] = new_data
        classcount["class1"] = class1
        classcount["class2"] = class2
        split_data[val] = classcount
    return split_data

"""
def gini(data):
    class_count = num_classes(data)
    impurity = 1
    for class_item in class_count:
        class_prob = class_count[class_item] / float(len(data))
        impurity -= class_prob**2
    return impurity
"""

def calc_entropy_data(dic, data):
    #class1_count = 0
    #class2_count = 0
    entropy = 0
    for key in dic:
        #class1_count += dic[key]["class1"]
        #class2_count += dic[key]["class2"]
        #print("class counts:")
        #print(dic[key]["class1"])
        #print(dic[key]["class2"])
        entropy += (dic[key]["class1"] + dic[key]["class2"]) / len(data) * (calc_entropy(dic[key]["class1"], dic[key]["class2"]))
    return entropy

def calc_entropy(class1_count, class2_count):
    p = 0
    if not class1_count == 0 and not class2_count == 0:
        p = -(class1_count/(class1_count+class2_count))*math.log(class1_count/(class1_count+class2_count), 2)\
            - (class2_count/(class1_count+class2_count))*math.log(class2_count/(class1_count+class2_count), 2)
    return p
"""
    p = float(len(left)) / (len(left) + len(right))
    return uncertainty - p * gini(left) - (1 - p) * gini(right)
"""

def best_split(data, no_pass=[]):
    #pass_col = False
    high_gain = 0
    best_col = None
    entropy = 0
    features = len(data[0])
    for col in range(1, features):
        if col in no_pass:
            continue
        #print(data)

        gain = 0
        classcount = split(data, col)
        #print(classcount)

        entropy = calc_entropy_data(classcount, data)
        #print(col)
        #print(entropy)
        class1_count, class2_count = 0, 0
        if 0 in num_classes(data):
            class1_count = num_classes(data)[0]
        if 1 in num_classes(data):
            class2_count = num_classes(data)[1]
            #print(entropy)
        gain = calc_entropy(class1_count, class2_count) - entropy
        #print(gain)
        if gain > high_gain:
            high_gain, best_col = gain, col
    #print(best_col)
    #print(high_gain)
    return high_gain, best_col

class Leaf:

    def __init__(self, dic):
        self.results = dic

    @classmethod
    def from_items(self, data):
        return Leaf(num_classes(data))

class DecisionNode:
    def __init__(self, col, data=[]):
        self.col = col
        self.children = {}
        dic = {}
        dic[majority_class(data)] = len(data)
        self.majority = dic

    @classmethod
    def from_dnode(self, node):
        self.col = node.col
        self.children = node.children
        self.majority = node.majority

def build_tree(data, orig_data=[], no_pass=[]):
    if len(orig_data) == 0:
        orig_data = data
    #print(data)
    gain, col = best_split(data, no_pass)

    if gain == 0:
        if len(num_classes(data)) != 1:
            dic = {}
            dic[majority_class(data)] = len(data)
            return Leaf(dic)
        else:
            return Leaf.from_items(data)
    node = DecisionNode(col, data)
    split_data = split(data, col)
    for key in unique_items(orig_data, col):
        if key not in split_data:#len(split_data[key]["data"]) == 0:
            dic = {}
            dic[majority_class(data)] = 0
            node.children[key] = Leaf(dic)
        else:
            node.children[key] = build_tree(split_data[key]["data"], orig_data, no_pass)
    #for key in split_data:
    #    node.children[key] = build_tree(split_data[key]["data"], no_pass)

    return node

def preprune_tree(data, valdata, top_node=None, orig_data=[], no_pass=[]):
    first_node = False
    if top_node == None:
        first_node = True
    if len(orig_data) == 0:
        orig_data = data
    #print(data)
    gain, col = best_split(data, no_pass)
    #print("hey")
    #print(gain)
    #no_pass.append(col)
    #print(col)
    if gain == 0:
        if len(num_classes(data)) != 1:
            dic = {}
            dic[majority_class(data)] = len(data)
            return Leaf(dic)
        else:
            return Leaf.from_items(data)
    node = DecisionNode(col, data)
    prune_acc = 0
    if first_node:
        top_node = node
        """
    else:
        dic = {}
        dic[majority_class(data)] = len(data)
        node = Leaf(dic)
        prune_acc = ret_accuracy(valdata, top_node)
        node = DecisionNode(col, data)
"""
    split_data = split(data, col)
    val_split = split(valdata, col)
    for key in unique_items(orig_data, col):
        if key not in split_data:#len(split_data[key]["data"]) == 0:
            dic = {}
            dic[majority_class(data)] = 0
            node.children[key] = Leaf(dic)
        else:
            if key not in val_split:
                dic = {"data": [], "class1": 0, "class2": 0}
                val_split[key] = dic
            node.children[key] = preprune_tree(split_data[key]["data"], val_split[key]["data"], top_node, orig_data, no_pass)
    if not first_node:
        if ret_accuracy(valdata, Leaf(dic={majority_class(data):len(data)})) < ret_accuracy(valdata, node):
            return Leaf(dic={majority_class(valdata):len(data)})
        """
    if not first_node:
        temp = node
        dec_acc = ret_accuracy(valdata, top_node)
        dic = {}
        dic[majority_class(data)] = len(data)
        node = Leaf(dic)
        prune_acc = ret_accuracy(valdata, top_node)

        if prune_acc < dec_acc:
            node = temp
        else:
            return node
            """
    #for key in split_data:
    #    node.children[key] = build_tree(split_data[key]["data"], no_pass)

    return node

def depth_prune_tree(data, depth, orig_data=[], no_pass=[]):
    if len(orig_data) == 0:
        orig_data = data
    #print(data)
    if depth <= 0:
        dic = {}
        dic[majority_class(data)] = len(data)
        return Leaf(dic)
    gain, col = best_split(data, no_pass)
    #print("hey")
    #print(gain)
    #no_pass.append(col)
    #print(col)
    if gain == 0:
        if len(num_classes(data)) != 1:
            dic = {}
            dic[majority_class(data)] = len(data)
            return Leaf(dic)
        else:
            return Leaf.from_items(data)
    node = DecisionNode(col, data)
    split_data = split(data, col)
    for key in unique_items(orig_data, col):
        if key not in split_data:#len(split_data[key]["data"]) == 0:
            dic = {}
            dic[majority_class(data)] = 0
            node.children[key] = Leaf(dic)
        else:
            node.children[key] = depth_prune_tree(split_data[key]["data"], depth-1, orig_data, no_pass)
    #for key in split_data:
    #    node.children[key] = build_tree(split_data[key]["data"], no_pass)

    return node

def print_tree(node, spacing=""):
    if isinstance(node, Leaf):
        print(spacing + "Predict", node.results)
        return

    print(spacing + "col:" + str(node.col))
    for key in node.children:

        print(spacing + str(key))
        print_tree(node.children[key], spacing + "  ")

def all_leaves(node):
    if isinstance(node, DecisionNode):
        for key in node.children:
            if not isinstance(node.children[key], Leaf):
                return False
        return True
    return False

def turn_leaf(node, data): #data is val set
    initial_wrong_count = len(data)
    for row in data:
        if (len(classify(row, node)) == 1) and (classify(row, node).get(row[0]) is not None):
            initial_wrong_count -= 1
    new_dict = {}
    turn_l = False
    for key in num_classes(data):
        eCount = len(data) - num_classes(data)[key]
        if eCount <= initial_wrong_count:
            initial_wrong_count = eCount
            turn_l = True
            new_dict.clear()
            new_dict[key] = len(data)
    return turn_l, new_dict

def prune(node, data):
    if all_leaves(node): #we know it is decision node with all leaves
        turn, dic = turn_leaf(node, data)
        if turn:
            return Leaf(dic)

    elif isinstance(node, DecisionNode):# and (not isinstance(node.true_path, Leaf) or not isinstance(node.false_path, Leaf)):
        dict = split(data, node.col)
        #count = 0
        for key in node.children:#unique_items(orig_data, node.col):
            #count += 1
            #print(dict[key]["data"])
            if key in dict:
                node.children[key] = prune(node.children[key], dict[key]["data"])
            else:
                arr = []
                node.children[key] = prune(node.children[key], arr)

    if all_leaves(node): #we know it is decision node with all leaves
        turn, dic = turn_leaf(node, data)
        if turn:
            return Leaf(dic)
    return node

def classify(row, node):
    if isinstance(node, Leaf):
        #print(node.results)
        return node.results
    else: # then is decision node
        #print(node.col)
        #print(row[node.col])
        #print("next")
        #if node.children[row[node.col]] is None:
        #    return
        return classify(row, node.children[row[node.col]])

def print_leaf(counts):
    total = sum(counts.values()) * 1.0
    probs = {}
    for lbl in counts.keys():
        probs[lbl] = str(int(counts[lbl] / total * 100)) + "%"
    return probs

def stratify(data):
    #trainSize = int(len(data)*.80)
    val_size = int(len(data)*.20)
    #print(trainSize)
    #print(val_size)
    data = random.sample(data, len(data))
    #count = 0
    while not stratified(data, val_size):
        data = random.sample(data, len(data))
        #count += 1
        #print(count)
    return data[val_size:], data[:val_size]

def stratified(data, val_size): #assuming only two classes
    if len(num_classes(data[val_size:])) <= 1 or len(num_classes(data[:val_size])) <= 1:
        return False
    #matching = False
    data1, data2 = [], []
    for key in num_classes(data[val_size:]):
        data1.append(num_classes(data[val_size:])[key])
        data2.append(num_classes(data[:val_size])[key])
    if data1[0]/data1[1] <= data2[0]/data2[1]+0.05 and data1[0]/data1[1] >= data2[0]/data2[1]-0.05:
        return True
    else:
        return False

def to_csv(data, outfile):
    for row in data:
        for item in row:
            outfile.write(str(item) + ",")
        outfile.write("\n")

def print_accuracy(test, tree):
    count = 0
    for row in test:
        if (len(classify(row, tree)) == 1) and (classify(row, tree).get(row[0]) is not None):
            count += 1
    print(count/len(test))

def ret_accuracy(test, tree):
    count = 0
    for row in test:
        if (len(classify(row, tree)) == 1) and (classify(row, tree).get(row[0]) is not None):
            count += 1
    if len(test) == 0:
        return 0
    return(count/len(test))

monks_train = load("monks-1.train")
monks_test = load("monks-1.test")
tree = build_tree(monks_train)
depthprune = depth_prune_tree(monks_train, 6)
print("Base Decision Tree:")
print_tree(tree)
print("Accuracy of Base Decision Tree:")
print_accuracy(monks_test, tree)
monks_train, monks_val = stratify(monks_train)
postprune = build_tree(monks_train)
#print_accuracy(monks_test, postprune)
postprune = prune(postprune, monks_val)
print("\n\nPost-Pruned Tree:")
print_tree(postprune)
print("Accuracy of Post-Pruned Tree:")
print_accuracy(monks_test, postprune)
preprune = preprune_tree(monks_train, monks_val)
print("\n\nPre-Pruned Tree:")
print_tree(preprune)
print("Accuracy of Pre-Pruned Tree:")
print_accuracy(monks_test, preprune)
print("\n\nPre-Pruned Tree:")
print_tree(depthprune)
print("Accuracy of Depth-Restricted Tree:")
print_accuracy(monks_test, depthprune)
#print_accuracy(monks_test, tree)


class Solution:
    def numDecodings(self, s):
        """
        :type s: str
        :rtype: int
        """
        letter = [str(i) for i in range(1,27)]
        l = len(s)
        ways = [1]
        for i in range(l):
            if s[i] == '0':
                ways.append(0)
            else:
                ways.append(ways[i])
            if i > 0 and s[i-1:i+1] in letter:
                ways[i+1] += ways[i-1]
        return ways[-1]

solution = Solution()
s = '12012226'
print(solution.numDecodings(s))


# -*- coding: utf-8 -*-
from django.conf.urls import patterns, include, url
from django.conf import settings
from django.conf.urls.static import static

from django.contrib import admin
from filebrowser.sites import site

admin.autodiscover() # функция автоматического обнаружения файлов admin.py в наших приложениях
urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'euProj.views.home', name='home'),
    # url(r'^blog/', include('blog.urls'))
    url(r'^master/filebrowser/', include(site.urls)),
    url(r'^master/', include(admin.site.urls)),     # URL админки http://site_name/admin
    url(r'^login/', include(admin.site.urls)),
    url(r'^$', include('euSite.urls')),
    url(r'^posts/', include('euSite.urls')),
    url(r'^os_moinkr_cal/', include('euSite.urls')),
    url(r'^news/', include('euSite.urls')),
    url(r'^calendar/', include('happenings.urls', namespace='calendar')),
    url(r'^forum/', include('pybb.urls', namespace='pybb')),
    url(r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.STATIC_ROOT}),    # Доступа к статическим файлам
)
if settings.DEBUG:
    urlpatterns += patterns('',
        (r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT}))


import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import os
import django

#os.environ.setdefault("DJANGO_SETTINGS_MODULE", "crawl.settings")
#django.setup()

from data.models import youtube

def makeadr(adr):
    k = adr.find('v')
    adr = adr[k+2:]
    return "https://www.youtube.com/embed/"+adr+"?rel=0;amp;autoplay=1&loop=1&playlist="+adr

def search(adr):
    driver = webdriver.Chrome()
    driver.get(adr)
    time.sleep(1)
    body = driver.find_element_by_tag_name("body")
    num=50
    while num:
        body.send_keys(Keys.PAGE_DOWN)
        time.sleep(0.3)
        num-=1
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    lis = soup.find_all('a', {'class': 'yt-simple-endpoint style-scope ytd-video-renderer'})
    for li in lis:
        tem = li['title']
        index = tem.find('-')
        if(index==-1):
            continue
        author = tem[:index-1]
        #print(author)
        title = tem[index+2:]
        #print(title)
        address = li['href']
        #print(address)
        youtube(title=title, author = author, link=address).save()
    driver.close()

singer = input()
sing = input()
b = youtube.objects.filter(author__icontains = singer).count()
if b==0:
    print('no object')
    driver = webdriver.Chrome()
    driver.get('https://www.youtube.com/')
    elem=driver.find_element_by_id('search')
    elem.send_keys(singer)
    but = driver.find_element_by_id('search-icon-legacy').click()
    time.sleep(1)
    adr = driver.current_url
    driver.close()
    search(adr)
    quit()
else:
    b = youtube.objects.filter(author__icontains = singer, title__icontains=sing)
    for ex in b:
        ex.save()
        print(ex.title)
        print(ex.author)
        print(ex.link)
    print(makeadr(b[0].link))


import turtle

t = turtle.Pen()
a=20
t.penup()
for i in range(1,21):
    t.pendown()
    t.circle(a*i)
    t.penup()
    t.goto(0,-i*a)

turtle.done()


# Problem 4:
# 	Read three integers from the keyboard a,b,c and those values in the following order.
#
#   max > mid > min
#
#   Sample Input
#   10 1 7
#
#   Sample Output
#   10 > 7 > 1

a,b,c=map(int ,input().split())
maxim = max(a,b,c)
mini = min(a,b,c)
mid = a+b+c-maxim-mini
print(maxim,mid,mini,sep=' > ')


from base64 import urlsafe_b64encode
from uuid import uuid1

from abei.interfaces import (
    IProcedure,
    IProcedureData,
    IProcedureFactory,
    IProcedureDetail,
)

from .data_basic import (
    ProcedureDataBasic,
    ProcedureDataBool,
)
from .joint_basic import (
    joint_validate,
    joint_run,
)

class ProcedureBasic(IProcedure):
    signature = 'do_nothing@py'
    docstring = ''
    input_signatures = []
    output_signatures = []

    def get_signature(self):
        return self.signature

    def get_input_signatures(self):
        return self.input_signatures

    def get_output_signatures(self):
        return self.output_signatures

    def get_docstring(self):
        return self.docstring

    def set_docstring(self, docstring):
        self.docstring = docstring

    def run(self, procedure_data_list, **kwargs):
        # assert isinstance(kwargs.setdefault('procedure_cache', {}), dict)
        return (
            self.run_normally(procedure_data_list, **kwargs) if
            self.run_input_check(
                procedure_data_list, self.input_signatures) else
            self.run_exceptionally(procedure_data_list, **kwargs)
        )

    @staticmethod
    def run_input_check(procedure_data_list, signatures):
        if len(procedure_data_list) != len(signatures):
            raise AssertionError('invalid data list')

        has_missing_params = False
        for d, sig in zip(procedure_data_list, signatures):
            if d is None:
                has_missing_params = True
                continue
            if not isinstance(d, IProcedureData):
                raise AssertionError('invalid data list')
            if d.get_signature() != sig:
                raise AssertionError('data signature miss match')

        return not has_missing_params

    def run_normally(self, procedure_data_list, **kwargs):
        return [None] * len(self.output_signatures)

    def run_exceptionally(self, procedure_data_list, **kwargs):
        return [None] * len(self.output_signatures)

class ProcedureComposite(IProcedureDetail, ProcedureBasic):
    name = 'composite@py'
    output_joints = []
    output_indices = []

    def __init__(
            self,
            signature=None,
            docstring=None,
            input_signatures=None,
            output_signatures=None,
    ):
        self.signature = signature or urlsafe_b64encode(
            uuid1().bytes).strip(b'=').decode('utf8')
        self.docstring = docstring or self.docstring
        self.input_signatures = input_signatures or self.input_signatures
        self.output_signatures = output_signatures or self.output_signatures

    def get_joints(self):
        return [(f, i) for f, i in zip(
            self.output_joints, self.output_indices)]

    def set_joints(self, joints, indices):
        joint_validate(
            joints,
            indices,
            self,
            self.output_signatures,
        )
        self.output_joints = joints
        self.output_indices = indices

    def run_normally(self, procedure_data_list, **kwargs):
        return [
            joint_run(joint, procedure_data_list, **kwargs)[i] if
            joint else procedure_data_list[i]
            for joint, i in self.get_joints()
        ]

class ProcedureBuiltin(ProcedureBasic):
    name = 'builtin_op@py'

    def __init__(self, data_signature):
        self.signature = '{}:{}'.format(data_signature, self.name)

class ProcedureUnaryOperator(ProcedureBuiltin):
    name = 'unary_op@py'
    native_function = staticmethod(lambda x: x)

    def __init__(self, data_signature=ProcedureDataBasic.signature):
        super().__init__(data_signature)
        self.input_signatures = [data_signature]
        self.output_signatures = [data_signature]

    def run_normally(self, procedure_data_list, **kwargs):
        ret = procedure_data_list[0].clone()
        ret.set_value(self.native_function(
            procedure_data_list[0].get_value()))
        return [ret]

class ProcedureBinaryOperator(ProcedureBuiltin):
    name = 'binary_op@py'
    native_function = staticmethod(lambda x, y: x)

    def __init__(self, data_signature=ProcedureDataBasic.signature):
        super().__init__(data_signature)
        self.input_signatures = [data_signature, data_signature]
        self.output_signatures = [data_signature]

    def run_normally(self, procedure_data_list, **kwargs):
        ret = procedure_data_list[0].clone()
        ret.set_value(self.native_function(
            procedure_data_list[0].get_value(),
            procedure_data_list[1].get_value(),
        ))
        return [ret]

class ProcedureComparator(ProcedureBuiltin):
    name = 'compare@py'
    native_function = staticmethod(lambda x, y: True)

    def __init__(self, data_signature=ProcedureDataBasic.signature):
        super().__init__(data_signature)
        self.input_signatures = [data_signature, data_signature]
        self.output_signatures = [data_signature]

    def run_normally(self, procedure_data_list, **kwargs):
        ret = ProcedureDataBool()
        ret.set_value(self.native_function(
            procedure_data_list[0].get_value(),
            procedure_data_list[1].get_value(),
        ))
        return [ret]

class ProcedureFilter(ProcedureBuiltin):
    name = 'filter@py'

    def __init__(
            self,
            data_signature=ProcedureDataBasic.signature,
            data_count=1,
    ):
        super().__init__(data_signature)
        self.data_signature = data_signature
        signatures = [data_signature] * data_count
        self.input_signatures = ['int@py', *signatures]
        self.output_signatures = signatures

    def run_normally(self, procedure_data_list, **kwargs):
        flag = procedure_data_list[0].get_value()
        assert isinstance(flag, int)

        return [
            (flag & (1 >> i)) and
            procedure_data_list[i + 1] or None
            for i in range(len(self.output_signatures))
        ]

    def run_exceptionally(self, procedure_data_list, **kwargs):
        return self.run_normally(procedure_data_list, **kwargs)

class ProcedureBranch(ProcedureBuiltin):
    name = 'branch@py'

    def __init__(
            self,
            data_signature=ProcedureDataBasic.signature,
            data_count=2,
    ):
        super().__init__(data_signature)
        signatures = [data_signature] * data_count
        self.input_signatures = ['int@py', data_signature]
        self.output_signatures = signatures

    def run_normally(self, procedure_data_list, **kwargs):
        flag = procedure_data_list[0].get_value()
        ret = procedure_data_list[1]
        return [
            (flag & (1 >> i)) and ret or None
            for i in range(len(self.output_signatures))
        ]

class ProcedureSelect(ProcedureBuiltin):
    name = 'select@py'

    def __init__(
            self,
            data_signature=ProcedureDataBasic.signature,
            data_count=2,
    ):
        super().__init__(data_signature)
        signatures = [data_signature] * data_count
        self.input_signatures = ['int@py', *signatures]
        self.output_signatures = [data_signature]

    def run_normally(self, procedure_data_list, **kwargs):
        flag = procedure_data_list[0].get_value()
        assert isinstance(flag, int)

        import math
        index = int(math.log2(flag)) + 1

        return [
            procedure_data_list[index] if
            index < len(procedure_data_list) else None
        ]

    def run_exceptionally(self, procedure_data_list, **kwargs):
        return self.run_normally(procedure_data_list, **kwargs)

class ProcedureNot(ProcedureUnaryOperator):
    name = 'not@py'
    native_function = staticmethod(lambda x: not x)

class ProcedureNegate(ProcedureUnaryOperator):
    name = 'neg@py'
    native_function = staticmethod(lambda x: -x)

class ProcedureSquare(ProcedureUnaryOperator):
    name = 'sq@py'
    native_function = staticmethod(lambda x: x * x)

class ProcedureAnd(ProcedureBinaryOperator):
    name = 'and@py'
    native_function = staticmethod(lambda x, y: x and y)

class ProcedureOr(ProcedureBinaryOperator):
    name = 'or@py'
    native_function = staticmethod(lambda x, y: x or y)

class ProcedureAdd(ProcedureBinaryOperator):
    name = 'add@py'
    native_function = staticmethod(lambda x, y: x + y)

class ProcedureSubtract(ProcedureBinaryOperator):
    name = 'sub@py'
    native_function = staticmethod(lambda x, y: x - y)

class ProcedureMultiply(ProcedureBinaryOperator):
    name = 'mul@py'
    native_function = staticmethod(lambda x, y: x * y)

class ProcedureDivide(ProcedureBinaryOperator):
    name = 'div@py'
    native_function = staticmethod(lambda x, y: x / y)

class ProcedureModulo(ProcedureBinaryOperator):
    name = 'mod@py'
    native_function = staticmethod(lambda x, y: x % y)

class ProcedureModDivide(ProcedureBinaryOperator):
    name = 'mod_div@py'
    native_function = staticmethod(lambda x, y: x // y)

class ProcedurePower(ProcedureBinaryOperator):
    name = 'pow@py'
    native_function = staticmethod(lambda x, y: x ** y)

class ProcedureEqual(ProcedureComparator):
    name = 'eq@py'
    native_function = staticmethod(lambda x, y: x == y)

class ProcedureNotEqual(ProcedureComparator):
    name = 'ne@py'
    native_function = staticmethod(lambda x, y: x != y)

class ProcedureLessThan(ProcedureComparator):
    name = 'lt@py'
    native_function = staticmethod(lambda x, y: x < y)

class ProcedureLessThanOrEqual(ProcedureComparator):
    name = 'lte@py'
    native_function = staticmethod(lambda x, y: x <= y)

class ProcedureGreaterThan(ProcedureComparator):
    name = 'gt@py'
    native_function = staticmethod(lambda x, y: x > y)

class ProcedureGreaterThanEqual(ProcedureComparator):
    name = 'gte@py'
    native_function = staticmethod(lambda x, y: x >= y)

class ProcedureFactoryBasic(IProcedureFactory):

    def __init__(self, service_site, **kwargs):
        self.procedure_classes = {p.name: p for p in [
            ProcedureComposite,
            ProcedureNot,
            ProcedureNegate,
            ProcedureSquare,
            ProcedureAnd,
            ProcedureOr,
            ProcedureAdd,
            ProcedureSubtract,
            ProcedureMultiply,
            ProcedureDivide,
            ProcedureModulo,
            ProcedureModDivide,
            ProcedurePower,
            ProcedureEqual,
            ProcedureNotEqual,
            ProcedureLessThan,
            ProcedureLessThanOrEqual,
            ProcedureGreaterThan,
            ProcedureGreaterThanEqual,
            ProcedureFilter,
            ProcedureBranch,
            ProcedureSelect,
        ]}

    def create(self, class_name, **kwargs):
        procedure_class = self.procedure_classes.get(class_name)
        return procedure_class and procedure_class(**kwargs)

    def register_class(self, class_name, procedure_class, **kwargs):
        assert class_name not in self.procedure_classes
        self.procedure_classes[class_name] = procedure_class

    def iterate_classes(self):
        return self.procedure_classes.keys()


# -*- coding: utf-8 -*-

import unittest
import requests
import json

from opengovpy import api

test_api = api()

class TestSignature(unittest.TestCase):
    def test_signature(self):
        self.assertRaises(Exception, callableObj=test_api, foo='bar')

class TestArgs(unittest.TestCase):
    """
    Check if objects returned by wrapper
    are equal to ones fetched directly from
    api
    """

    def setUp(self):
        self.basestring = test_api.base_url

    def test_base(self):
        self.assertEqual(test_api(), self.api_call(self.basestring + '/datasets'))
        self.assertEqual(test_api(id=658), self.api_call(self.basestring + '/datasets/658'))
        self.assertEqual(test_api(id=658, rows=True), self.api_call(self.basestring + '/datasets/658/rows'))
        self.assertEqual(test_api(id=658, count=True), self.api_call(self.basestring + '/datasets/658/count'))
        self.assertEqual(test_api(id=1786, features=True), self.api_call(self.basestring + '/datasets/1786/features'))
        self.assertEqual(test_api(id=1786, features=True,
                                  bbox=[37.49711036682129, 55.86543869723485, 37.5490379333496, 55.89110103788533]),
                         self.api_call(self.basestring + '/datasets/1786/features??bbox=37.49711036682129,'
                                                         '55.86543869723485,37.5490379333496,55.89110103788533'))

    def test_additional_params(self):
        self.assertEqual(test_api(id=658, rows=True, top=3, orderby='Number'),
                         self.api_call(self.basestring + '/datasets/658/rows?$top=3&$orderby=Number'))
        self.assertEqual(test_api(skip=10, inlinecount=True, top=5),
                         self.api_call(self.basestring + '/datasets?$skip=10&$top=5&$inlinecount=allpages'))

    def api_call(self, url):
        return json.loads(requests.get(url).text)

if __name__ == '__main__':
    unittest.main()


"""
Created on 21 Jan 2021

@author: Jade Page (jade.page@southcoastscience.com)
"""

from scs_core.aws.data.deployment import Deployment
from scs_core.aws.greengrass.aws_group import AWSGroup

# --------------------------------------------------------------------------------------------------------------------

class AWSDeploymentReporter(object):

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, client):
        self.__client = client

    # ----------------------------------------------------------------------------------------------------------------

    def get_all_group_ids(self):
        next_token = None
        group_ids = []

        while True:
            response = self.__get_groups(next_token)

            if "Groups" not in response:
                break

            for item in response["Groups"]:
                group_ids.append(item.get("Id"))

            if "NextToken" not in response:
                break

            next_token = response["NextToken"]

        return group_ids

    def get_group_id(self, group_name):
        aws_group_info = AWSGroup(group_name, self.__client)
        datum = aws_group_info.get_group_info_from_name()

        return None if datum is None else datum.node("GroupID")

    def get_group_names(self, group_ids, matching=None, currency=None):
        return tuple(deployment.group_name for deployment in
                     self.get_deployments(group_ids, matching=matching, currency=currency))

    def get_deployments(self, group_ids, matching=None, currency=None):
        deployments = []

        for id in group_ids:
            if id is None:
                continue

            response = self.__client.list_deployments(GroupId=id)
            group_name = self.__get_group_name(id)

            if matching and matching not in group_name:
                continue

            latest_deployment = response["Deployments"][0] if response["Deployments"] else None
            deployment = Deployment.construct_from_aws(group_name, latest_deployment)

            if currency is None or not deployment.is_current(currency):
                deployments.append(deployment)

        return sorted(deployments)

    # ----------------------------------------------------------------------------------------------------------------

    def __get_groups(self, next_token):
        if next_token is None:
            return self.__client.list_groups()

        return self.__client.list_groups(NextToken=next_token)

    def __get_group_name(self, group_id):
        response = self.__client.get_group(GroupId=group_id)

        if "Name" not in response:
            return None

        return response["Name"]

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "AWSDeploymentReporter:{client:%s}" % self.__client


# -*- coding: utf-8 -*-
"""
Created on Wed Nov 30 17:04:48 2016

@author: ssits
"""

def is_palindromic(sub_text):
    lst_str1=list(sub_text)
    lst_str2=reversed(list(sub_text))
    str_str1=''.join(lst_str1)
    str_str2=''.join(lst_str2)
    if (str_str1==str_str2):
        return True
    else:
        return False

def longest_palindromic(text):

    max_text=""

    for i in range(0,len(text)):
        for x in range(i+1,len(text)+1):
            substr_text=text[i:x]
            if ((len(substr_text)>len(max_text)) & (is_palindromic(substr_text))):
                max_text=substr_text

    return max_text

if __name__ == '__main__':
    assert longest_palindromic("artrartrt") == "rtrartr", "The Longest"
    assert longest_palindromic("abacada") == "aba", "The First"
    assert longest_palindromic("aaaa") == "aaaa", "The A"


#Mein Taschenrechner mit Python und Kivy
import kivy
kivy.require("1.9.2")

from kivy.app import App
from kivy.uix.gridlayout import GridLayout

class RechnerLayout(GridLayout):
    def calculate(self, rechnung):
        if rechnung:
            try:
                # Solve formula and display it in entry
                # which is pointed at by the display
                self.display.text = str(eval(rechnung))
            except Exception:
                self.display.text = "Fehler"

class RechnerKvApp(App):
    def build(self):
        return RechnerLayout()

rechApp = RechnerKvApp()
rechApp.run()


from mlaut.data import Data
import os
import random
from delgado_datasets import download_and_extract_datasets
from mlaut.experiments import Orchestrator
from estimators_500epochs_batch_1 import estimators

datasets, metadata = download_and_extract_datasets()

data = Data()
input_io = data.open_hdf5('data/delgado.h5', mode='a')

for item in zip(datasets, metadata):
    dts=item[0]
    meta=item[1]
    input_io.save_pandas_dataset(dataset=dts, save_loc='/delgado_datasets', metadata=meta)

out_io = data.open_hdf5('data/delgado-deep_epochs500_batch1.h5', mode='a')
dts_names_list, dts_names_list_full_path = data.list_datasets(hdf5_io=input_io, hdf5_group='delgado_datasets/')
random.seed(7)
split_dts_list = data.split_datasets(hdf5_in=input_io, hdf5_out=out_io, dataset_paths=dts_names_list_full_path)

orchest = Orchestrator(hdf5_input_io=input_io,
                            hdf5_output_io=out_io,
                            dts_names=dts_names_list,
                            original_datasets_group_h5_path='delgado_datasets/')
orchest.run(modelling_strategies=estimators)


"""
Created on Wed Sep 20 16:56:37 2017

TODO: Add npy, npz types for saving plain types
TODO: Use npy by default on numpy arrays

@author: alex
"""

import sys
import os
import os.path
from warnings import warn
import builtins
from pathlib import Path
import io
from collections import namedtuple, OrderedDict
import logging
import numpy as np
import dill
from inspect import isclass
from parameters import ParameterSet
logger = logging.getLogger('mackelab.iotools')

# TODO: Make dill an optional dependency, use pickle otherwise (with '.dill', '.pkl' extensions).
#       Do this by pulling out all the format-specific export code from `save`
#       into separate functions – then we can just define the fallback type as
#       either 'dill' or 'pkl' depending on whether `dill` could be loaded.
#       (c.f. comment in `save` starting with "Define the save functions below at top level of module […]")
# TODO: (Partly supersedes the one below) Remove any serialization code that
#       duplicates Pydantic functionality.

if sys.version_info.major >= 3 and sys.version_info.minor >= 6:
    PathTypes = (str, os.PathLike)
else:
    PathTypes = (str,)

Format = namedtuple("Format", ['ext', 'save', 'load', 'bytes'])
    # save: save function, `save(filename, data)`
    # load: load function, `load(filename)`
    # bytes: whether to use the 'b' flag when opening a file for loading/saving
    # TODO: Allow multiple extensions per format
    # TODO: Use namedtuple defaults statt None once Python3.7 is standard
defined_formats = OrderedDict([
    # List of formats known to the load/save functions
    # The order of these formats also defines a preference, for when two types might be used
    ('npr',  Format('npr', None, None, True)),
    ('repr', Format('repr', None, None, False)),
    ('brepr', Format('brepr', None, None, True)), # Binary version of 'repr'
    ('dill', Format('dill', None, None, True))
    ])

_load_types = {}
_format_types = {}

def register_datatype(type, typename=None, format=None, alias=None):
    """
    :param alias: alternative name, usually a shortened form of the full typename.
    """
    global _load_types, _format_types
    assert(isclass(type))
    if typename is None:
        typename = type.__module__ + '.' + type.__qualname__
    if alias is None:
        alias = type.__qualname__
        if alias in _load_types:  # Don't let default override existing alias
            alias = None
    assert(isinstance(typename, str))
    _load_types[typename] = type
    if alias is not None:
        _load_types[alias] = type
    if format is not None:
        assert isinstance(format, str)
        _format_types[typename] = format

def find_registered_typename(type):
    """
    If `type` is a registered datatype, return it's associated name.
    Otherwise, find the nearest parent of `type` which is registered and return its name.
    If no parents are registered, return `type.__name__`.
    """
    def get_name(type):
        # _load_types is always a superset of _format_types
        for registered_name, registered_type in _load_types.items():
            if registered_type is type:
                return registered_name
        return None

    typename = get_name(type)
    if typename is None:
        if not isinstance(type, builtins.type):
            type = builtins.type(type)
        for base in type.mro():
            typename = get_name(base)
            if typename is not None:
                break
    if typename is None:
        typename = type.__name__
            # No registered type; return something sensible (i.e. the type's name)
    return typename

def get_format_from_ext(ext):
    format = None
    ext = ext.strip('.')
    for name, info in defined_formats.items():
        exts = [info.ext] if isinstance(info.ext, str) else info.ext
        for e in exts:
            if e.strip('.') == ext:
                format = name
                break
        if format is not None:
            break
    if format is None:
        raise ValueError("No format matching extension {} was found."
                         .format(ext))
    return format

# Register basic data types
# Python structures can be arbitrarily nested, so we can't assume that they
# would compatible with any strict format
register_datatype(list, format='dill')
register_datatype(tuple, format='dill')
register_datatype(set, format='dill')
register_datatype(dict, format='dill')

def get_free_file(path, bytes=True, max_files=100, force_suffix=False, start_suffix=None):
    """
    Return a file handle to an unused filename. If 'path' is free, return a handle
    to that. Otherwise, append a number to it until a free filename is found or the
    number exceeds 'max_files'. In the latter case, raise 'IOError'.

    Returning a file handle, rather than just a file name, avoids the possibility of a
    race condition (a new file of the same name could be created between the time
    where one finds a free filename and then opens the file).

    Parameters
    ----------
    path: str
        Path name. Can be absolute or relative to the current directory.
    bytes: bool (Default: True)
        (Optional) Specify whether to open the file for byte (True) or plain
        text (False) output. Default is to open for byte output, which
        is suitable for passing to `numpy.save`.
    max_files: int
        (Optional) Maximum allowed number of files with the same name. If this
        number is exceeded, IOError is raised.
    force_suffix: bool (default False)
        (Optional) If True, a suffix '_#', where # is a number, is always added
        to the file name. Forcing suffixes also changes the default value
        of 'start_suffix' to 1.
    start_suffix: int (default 2)
        If creating a file with 'path' is unsuccessful (or 'force_suffix is
        set to True), this is the first number to try appending to the file name.

    Returns
    -------
    filehandle
        Write-only filehandle, as obtained from a call to
        `open(pathname, 'mode='xb')`.
    pathname: str
        Pathname (including the possibly appended number) of the opened file.
    """

    # Get a full path
    # TODO: is cwd always what we want here ?
    if isinstance(path, Path):
        pathname = str(path.absolute())
    elif path[0] == '/':
        #path is already a full path name
        pathname = path
    else:
        #Make a full path from path
        pathname = os.path.abspath(path)

    # Set the default value for start_suffix
    if start_suffix is None:
        start_suffix = 1 if force_suffix else 2

    # Set the mode
    if bytes:
        mode = 'xb'
    else:
        mode = 'x'

    # Make sure the directory exists
    os.makedirs(os.path.dirname(pathname), exist_ok=True)

    try:
        if force_suffix:
            raise IOError
        else:
            f = open(pathname, mode=mode)
            return f, pathname
    except IOError:
        name, ext = os.path.splitext(pathname)
        for i in range(start_suffix, max_files+start_suffix):
            appendedname = name + "_" + str(i) + ext
            try:
                f = open(appendedname, mode=mode)
                return f, appendedname
            except IOError:
                continue

        raise IOError("Number of files with the name '{}' has exceeded limit."
                      .format(path))

def save(file, data, format=None, overwrite=False):
    """Save `data`.
    First the function checks if :param:data defines a `save()` method; if so,
    the method is called as `save(output_path)`. If this is successful, the
    function terminates.
    If the call is not successful, or :param:data does not define a `save()`
    method, then the function attempts to save to the formats defined by
    `format`. By default, only the 'numpy_repr' representation is saved,
    if `data` defines a numpy representation.
    Not only is the numpy representation format more future-proof, it can be an
    order of magnitude more compact.
    If the numpy_repr save is unsuccessful (possibly because `data` does not provide a
    `numpy_repr` method), then `save()` falls back to saving a plain (dill) pickle of 'data'.

    Parameters
    ----------
    file: str
        Path name or file object. Note that the file extension is mostly
        ignored and will be replaced by the one associated with the format.
        This is to allow saving to multiple formats.
    data: Python object
        Data to save
    format: str
        The format in which to save the data. Possible values are:
          - 'npr' (default) Save with the numpy_repr format. This is obtained by calling the
            method 'nprepr' on the `data`. If this call fails, a warning is issued
            and the 'dill' format is used.
            Output file have the extension 'npr'.
            Objects using this format should implement the `from_nprepr` method.
          - 'repr' Call `repr` on the data and save the resulting string to file. The save will
            fail (and fall back to 'dill' format) if the `repr` is simply inherited from object,
            as simply saving the object address is not useful for reconstructing it. Still, there
            is no way of ensuring that the `repr` is sufficiently informative to reconstruct the
            object, so make sure it is before using this format.
            Output file have the extension 'repr'.
            Objects using this format should implement the `from_repr` method.
          - 'dill' A dill pickle.
            Output file has the extension 'dill'
        Formats can also be combined as e.g. 'npr+dill'.
    overwrite: bool
        If True, allow overwriting previously saved files. Default is false, in
        which case a number is appended to the filename to make it unique.

    Returns
    -------
    List of output paths.
        List because many formats may be specified, leading to multiple outputs.
    """
    if isinstance(format, str):
        selected_formats = format
    else:
        if format is None:
            typename = find_registered_typename(type(data))
        else:
            if not isinstance(format, type):
                logger.error("The `format` argument should be either a string "
                             "or type. Provided value: {}"
                             "Attempting to infer type from data".format(format))
                typename = find_registered_typename(type(data))
            typename = find_registered_typename(format)
        if typename in _format_types:
            format = _format_types[typename]
        else:
            logger.error("Type '{}' has no associated format".format(typename))
            format = 'npr'

    selected_formats = set(format.split('+'))

    # Check argument - format
    bad_formats = [f for f in selected_formats if f not in defined_formats]
    selected_formats = selected_formats.difference(bad_formats)
    if len(bad_formats) > 0:
        format_names = ["'" + f + "'" for f in defined_formats]
        bad_format_names = ["'" + f + "'" for f in bad_formats]
        formatstr = "format"
        if len(format_names) > 1:
            format_names = ", ".join(format_names[:-1]) + " and " + format_names[-1]
        if len(bad_format_names) > 1:
            formatstr = "formats"
            bad_format_names = ", ".join(bad_format_names[:-1]) + " and " + bad_format_names[-1]
        logger.warning("Unrecognized save {} {}.".format(formatstr, bad_format_names)
                       + "Recognized formats are " + format_names)
        if len(selected_formats) == 0:
            logger.warning("Setting the format to {}.".format_names)
            # We don't want to throw away the result of a long calculation because of a
            # flag error, so instead we will try to save into every format and let the user
            # sort out the files later.
            format = '+'.join(format_names)

    get_output = None
    def set_str_file(filename):
        nonlocal get_output
        def _get_output(filename, ext, bytes, overwrite):
            return output(filename, ext, bytes, overwrite)
        get_output = _get_output

    # Check argument - file
    if isinstance(file, io.IOBase):
        thisfilename = os.path.realpath(file.name)
        if 'luigi' in os.path.basename(thisfilename):
            # 'file' is actually a Luigi temporary file
            luigi = True
        else:
            luigi = False
        filename = thisfilename  # thisfilename used to avoid name clashes
        if not any(c in file.mode for c in ['w', 'x', 'a', '+']):
            logger.warning("File {} not open for writing; closing and reopening.")
            file.close()
            set_str_file(thisfilename)
        else:
            def _get_output(filename, ext, bytes, overwrite):
                # Check that the file object is compatible with the arguments,
                # and if succesful, just return the file object unmodified.
                # If it is not successful, revert to opening a file as though
                # a filename was passed to `save`.
                # TODO: Put checks in `dummy_file_context`
                fail = False
                if (os.path.splitext(os.path.realpath(filename))[0]
                    != os.path.splitext(os.path.realpath(thisfilename))[0]):
                    logger.warning("[iotools.save] Given filename and file object differ.")
                    fail = True
                thisext = os.path.splitext(thisfilename)[1].strip('.')
                if not luigi and thisext != ext.strip('.'):
                    # Luigi adds 'luigi' to extensions of temporary files; we
                    # don't want that to trigger closing the file
                    logger.warning("[iotools.save] File object has wrong extension.")
                    fail = True
                if (bytes and 'b' not in file.mode
                    or not bytes and 'b' in file.mode):
                    if luigi:
                        # Luigi's LocalTarget always saves to bytes, and it's
                        # the Format class that takes care of converting data
                        # (possibly text) to and back from bytes.
                        logger.warning("\n"
                            "WARNING [iotools]: Attempted to save a 'luigi' target with the wrong "
                            "mode (binary or text). Note that Luigi targets "
                            "always use the same mode internally; use the "
                            "`format` argument to convert to/from in your code. "
                            "In particular, LocalTarget writes in binary. "
                            "Consequently, the file will not be saved as {}, "
                            "but as {}; specify the correct value to `bytes` "
                            "to avoid this message.\n"
                            .format("bytes" if bytes else "text",
                                    "text" if bytes else "bytes"))
                    else:
                        logger.warning("[iotools.save] File object has incorrect byte mode.")
                        fail = True
                if (overwrite and 'a' in file.mode):
                    # Don't check for `not overwrite`: in that case the damage is already done
                    logger.warning("[iotools.save] File object unable to overwrite.")
                    fail = True
                if fail:
                    logger.warning("[iotools.save] Closing and reopening file object.")
                    file.close()
                    set_str_file(thisfilename)
                    return output(filename, ext, bytes, overwrite)
                else:
                    return dummy_file_context(file)
            get_output = _get_output
    else:
        assert isinstance(file, PathTypes)
        filename = file
        set_str_file(file)

    # Ensure target directory exists
    dirname = os.path.dirname(filename)
    if dirname != "":
        os.makedirs(dirname, exist_ok=True)

    output_paths = []

    # If data provides a "save" method, use that
    # This overrides the "format" argument – only exception is if save fails,
    # then we reset it to what it was and try the other formats
    if isinstance(data, ParameterSet):
        # Special case of data with `save` attribute
        _selected_formats_back = selected_formats
        selected_formats = []  # Don't save to another format if successful
        with get_output(filename, ext="", bytes=False, overwrite=overwrite) as (f, output_path):
            # Close the file since Parameters only accepts urls as filenames
            # FIXME: This introduces a race condition; should use `f` to save
            #        This would require fixing the parameters package to
            #        accept file objects in `save()`
            pass
        try:
            logger.info("Saving ParameterSet using its own `save` method...")
            data.save(output_path, expand_urls=True)
        except (AttributeError, PermissionError) as e:
            logger.warning("Calling the data's `save` method failed with '{}'."
                           .format(str(e)))
            selected_formats = _selected_formats_back
        else:
            output_paths.append(output_path)
    elif hasattr(data, 'save'):
        _selected_formats_back = selected_formats
        selected_formats = []  # Don't save to another format if successful
        # See if this type is in the registered formats, so we can get the
        # expected extension
        typename = find_registered_typename(data)
            # Always returns a type name: if none is found, returns that of data
        format = _format_types.get(typename, None)
        if format is None or format not in defined_formats:
            ext = ""
        else:
            ext = defined_formats[format].ext
        with get_output(filename, ext=ext, bytes=False, overwrite=overwrite) as (f, output_path):
            # TODO: Use `f` if possible, and only `output_path` if it fails.
            pass
        try:
            logger.info("Saving data using its own `save` method...")
            data.save(output_path)
        except (AttributeError, PermissionError) as e:
            logger.warning("Calling the data's `save` method failed with '{}'."
                           .format(str(e)))
            selected_formats = _selected_formats_back
        else:
            output_paths.append(output_path)

    # Save to all specified formats
    for name, formatinfo in defined_formats.items():
        if name in ('npr', 'repr', 'brepr', 'dill'):
            # TODO: Define the save functions below at top level of module
            # and treat these formats as any other
            #       Make sure 'dill' is still used as backup
            continue
        if name in selected_formats:
            if formatinfo.save is None:
                logger.error("Format '{}' does not define a save function"
                             .format(name))
                fail = True
            else:
                fail = False
                ext = formatinfo.ext
                try:
                    with get_output(filename, ext, formatinfo.bytes, overwrite) as (f, output_path):
                        formatinfo.save(f, data)
                except IOError:
                    fail = True
                except Exception as e:
                    logger.error("Silenced uncaught exception during saving process to attempt another format.")
                    logger.error("Silenced exception was: " + str(e))
                    fail = True
                else:
                    output_paths.append(output_path)
            if fail:
                try: os.remove(output_path)  # Ensure there are no leftover files
                except: pass
                logger.warning("Unable to save to {} format."
                               .format(name))
                if 'dill' not in selected_formats:
                    # Warn the user that we will use another format
                    logger.warning("Will try a plain (dill) pickle dump.")
                    selected_formats.add('dill')
    # Save data as numpy representation
    if 'npr' in selected_formats:
        fail = False
        ext = defined_formats['npr'].ext
        try:
            with get_output(filename, ext, True, overwrite) as (f, output_path):
                try:
                    logger.info("Saving data to 'npr' format...")
                    np.savez(f, **data.repr_np)
                except AttributeError:
                    fail = True
                else:
                    output_paths.append(output_path)
        except IOError:
            fail = True
        if fail:
            # TODO: Use custom error type
            try: os.remove(output_path)  # Ensure there are no leftover files
            except: pass
            logger.warning("Unable to save to numpy representation ('npr') format.")
            if 'dill' not in selected_formats:
                # Warn the user that we will use another format
                logger.warning("Will try a plain (dill) pickle dump.")
                selected_formats.add('dill')

    # Save data as representation string ('repr' or 'brepr')
    for format in [format
                   for format in selected_formats
                   if format in ('repr', 'brepr')]:
        bytes = (format == 'brepr')
        fail = False
        if data.__repr__ is object.__repr__:
            # Non-informative repr -- abort
            fail = True
        else:
            ext = defined_formats['repr'].ext
            try:
                with get_output(filename, ext=ext, bytes=bytes, overwrite=overwrite) as (f, output_path):
                    try:
                        logger.info("Saving data to plain-text 'repr' format'")
                        f.write(repr(data))
                    except:
                        fail = True
                    else:
                        output_paths.append(output_path)
            except IOError:
                fail = True
        if fail:
            try: os.remove(output_path)  # Ensure there are no leftover files
            except: pass
            logger.warning("Unable to save to numpy representation ('npr') format.")
            if 'dill' not in selected_formats:
                # Warn the user that we will use another format
                logger.warning("Will try a plain (dill) pickle dump.")
                selected_formats.add('dill')

    # Save data in dill format
    if 'dill' in selected_formats:
        ext = defined_formats['dill'].ext
        try:
            with get_output(filename, ext, True, overwrite) as (f, output_path):
                logger.info("Saving data as a dill pickle.")
                dill.dump(data, f)
                output_paths.append(output_path)
        except IOError:
            # There might be other things to save, so don't terminate
            # execution because this save failed
            try: os.remove(output_path)  # Ensure there are no leftover files
            except: pass
            logger.warning("Unable to save picke at location {}."
                           .format(output_path))

    # Return the list of output paths
    return [Path(path) for path in output_paths]

def find_file(file, format=None):
    """
    If `file` has no extension and `format` is None:
    Returns the list of paths matching the file. If the list has length one,
    a simple path is returned
    Otherwise, returns the file path with the specified format.
    In both cases, if no file is found, raises FileNotFoundError.

    Parameters
    ----------
    file: str | path-like
    format: str | type
        Should correspond to an entry in `defined_formats`.
    """
    if isinstance(file, PathTypes):
        basepath, ext = os.path.splitext(file)
        dirname, basename = os.path.split(basepath)
        if dirname == '':
            dirname = '.'

        # Normalize the format and see if it is recognized
        if isinstance(format, str):
            formatext = defined_formats.get(format, None).ext
        elif isinstance(format, type):
            def_format = defined_formats.get(find_registered_typename(format), None)
            if def_format is not None:
                formatext = def_format.ext
            else:
                warn("The type {} does not correspond to one of the types "
                     "registered with `mackelab_toolbox.iotools.register_datatype`"
                     .format(format))
                formatext = None
        else:
            formatext = None

        # Find the file
        if len(ext) > 0 and formatext is None:
            format = ext.strip('.')
        if len(ext) == 0 and formatext is None:
            # Try every file whose name without extension matches `file`
            match = lambda fname: os.path.splitext(fname)[0] == basename
            fnames = [name for name in os.listdir(dirname) if match(name)]
            # Order the file names so we try most likely formats first (i.e. npr, repr, dill)
            # We do not attempt to load other extensions, since we don't know the format
            ordered_fnames = []
            for formatinfo in defined_formats.values():
                name = basename + '.' + formatinfo.ext
                if name in fnames:
                    ordered_fnames.append(os.path.join(dirname, name))
            if len(ordered_fnames) == 0:
                # No file was found
                raise FileNotFoundError(
                    "No file with base name '{}' and a recognized extension was "
                    "found.\nDirectory searched: {}"
                    .format(basename, dirname))
            elif len(ordered_fnames) == 1:
                ordered_fnames = ordered_fnames[0]
            return ordered_fnames

        elif os.path.exists(file):
            return file
        else:
            return basepath + "." + formatext.strip('.')
    else:
        raise ValueError("File finding is only implemented for path-like "
                         "arguments.")

def load(file, types=None, load_function=None, format=None, input_format=None):
    """
    Load file at `file`. How the data is loaded is determined by the input format,
    which is inferred from the filename extension. It can also be given by `format`
    explicitly.
    If `load_function` is provided, it is be applied to the loaded data. Otherwise,
    we try to infer type from the loaded data.
      - For 'npr' data, we look for a 'type' entry.
      - For 'repr' data, we look at the first part of the returned string, before
        any whitespace or special characters.
    Type inference can be completely disabled by passing `load_function=False`.
    If a type is found, we then try to match it to one of the entries in `types`. Thus
    `types` should be a dictionary of str:Type pairs, where the keys match the type name
    stored in files, and the Type is a loaded Python class which is associated to that
    name; typically, the type name will be the class name, although that is not required.
    It is also possible to add to the list of default types by calling this module's
    `add_load_type` method.

    Parameters
    ----------
    file: str | file object (TODO)

    types: dict
        (Optional)
    load_function: function
        (Optional) Post-processing function, called on the result of the loaded
        data. I.e. does not override `type`, but provides a handle to process
        the result.
    format: str
        Specify the format of the data; overrides any deductions from the
        filename. Effectively this specifies the loading function, and thus
        should correspond to a key in `types`.
    input_format: str (DEPRECATED)
        Deprecated synonym for `format`.
    """
    global _load_types
    if types is not None:
        types = {**_load_types, **dict(types)}
    else:
        types = _load_types

    if format is None:
        format = input_format

    if isinstance(file, PathTypes):
        # _, ext = os.path.splitext(file)
        # if len(ext) > 0 and format is None:
        #     format = ext.strip('.')
        ordered_fnames = find_file(file, format)
        if isinstance(ordered_fnames, list):
            # Try to load every file name in sequence. Terminate after the first success.
            for fname in ordered_fnames:  # Empty list -> FileNotFound below
                try:
                    # Recursively call `load`
                    data = load(fname, types, load_function)
                except (FileNotFoundError):
                    # Basically only possible to reach here with a race condition, where
                    # file is deleted after having been listed
                    # TODO: Also catch loading errors ?
                    continue
                else:
                    return data
            # No file was found
            raise FileNotFoundError(
                "No file with base name '{}' and a recognized extension was "
                "found.\nDirectory searched: {}"
                .format(basename, dirname))

        elif os.path.exists(ordered_fnames):
            # Normal exit from `load` recursion
            openfilename = ordered_fnames
            basepath, ext = os.path.splitext(openfilename)
        else:
            # No file was found
            raise FileNotFoundError("No file '{} was found. (Specified "
                                    " format: '{}')".format(file, format))

    elif isinstance(file, io.IOBase):
        ext = os.path.splitext(file.name)[1]
    elif hasattr(file, 'fn'):
        # Treats Luigi LocalTarget
        ext = os.path.splitext(file.fn)[1]
    else:
        raise TypeError("[iotools.load] File '{}' is of unrecognized type '{}'."
                        .format(file, type(file)))

    if format is None:
        format = get_format_from_ext(ext[1:])
    if format not in defined_formats:
        warn("mackelab_toolbox.iotools.load: Unrecognized format `{}`. "
             "Ignoring `format` argument and using file extension instead."
             .format(format))
        format = get_format_from_ext(ext[1:])

    if format not in ('npr', 'repr', 'dill'):
        # TODO: Define following load functions separately, add them to
        # to defined_formats, and then treat these as any other format
        formatinfo = defined_formats[format]
        data = defined_formats[format].load(basepath+ext)
    elif format == 'npr':
        with wrapped_open(file, 'rb') as f:
            data = np.load(f)
                # np.load provides dict access to the file object;
                # but doesn't load it to memory. We must make sure it's in
                # memory before we exit `with` block.
            in_memory = False
            if load_function is False:
                pass
            elif load_function is not None:
                data = load_function(data)
                # Is it safe to assume that `load_function` always puts data in memory ?
                in_memory = True
            elif 'type' in data:
                # 'type' stored as a 0D array
                if (data['type'].ndim == 0
                    and data['type'].dtype.kind in {'S', 'U'}
                    and str(data['type']) in types) :
                    # make sure it's really 0D
                    cls = types[str(data['type'])]
                    if hasattr(cls, 'from_repr_np'):
                        data = cls.from_repr_np(data)
                        in_memory = True
                else:
                    # TODO: Error message
                    pass
            if not in_memory:
                # Since it's still not in memory, just load it as a dictionary.
                data = dict(data)

    elif format == 'repr':
        with wrapped_open(openfilename, 'r') as f:
            data = f.read()
        if load_function is False:
            pass
        elif load_function is not None:
            data = load_function(data)
        else:
            # Search for the type name in the initial data
            if data[0] == '<':
                i, j = data.index('>'), data.index('<')
                if i == -1 or (j > -1 and j < i):
                    # Misformed type specification
                    pass
                else:
                    test_clsname = data[1:i]
                    if test_clsname in types:
                        cls = types[test_clsname]
                        if hasattr(cls, 'from_repr'):
                            data = cls.from_repr(data)
    elif format == 'dill':
        with wrapped_open(openfilename, 'rb') as f:
            try:
                data = dill.load(f)
            except EOFError:
                logger.warning("File {} is corrupted or empty. You should "
                               "delete it to prevent confusion.".format(file))
                raise FileNotFoundError

    return data

class output():
    def __init__(self, path, ext, bytes, overwrite=False):

        # Add extension
        basepath, _ = os.path.splitext(path)
            # Remove possible extension from path
        if len(ext) > 0 and ext[0] != ".":
            ext = "." + ext
        self.output_path = basepath + ext
        self.orig_output_path = self.output_path
        self.overwrite = overwrite
        self.bytes = bytes

    def __enter__(self):
        # Open file
        try:
            if not self.overwrite:
                self.f, self.actual_path = get_free_file(self.output_path, bytes=self.bytes)
            else:
                mode = 'wb' if self.bytes else 'w'
                self.f = open(self.orig_output_path, mode)
                self.actual_path = self.orig_output_path
        except IOError:
            logger.error("Could not create a file at '{}'."
                         .format(self.orig_output_path))
            raise

        return self.f, self.actual_path

    def __exit__(self, type, value, traceback):
        self.f.close()

class dummy_file_context:
    def __init__(self, file):
        self.f = file

    def __enter__(self):
        return self.f, os.path.realpath(self.f.name)

    def __exit__(self, type, value, traceback):
        # Since the file was not created in this context, don't close it
        pass

class wrapped_open:
    # Acts like `open`, but if given a file object, just returns the object.
    def __init__(self, file, mode):
        self.file = file
        self.mode = mode
        self.exit = None

    def __enter__(self):
        if isinstance(self.file, io.IOBase):
            if self.file.mode != mode:
                logger.warning("Already open file object has mode '{}', but "
                               " '{}' is expected."
                               .format(self.file.mode, self.mode))
            self.f = self.file
            self.exit = False  # Don't close a file we didn't open
        elif hasattr(self.file, 'open'):
            # Treats Luigi LocalTarget
            self.f = self.file.open(self.mode)
            self.exit = True
        else:
            self.f = open(self.file, self.mode)
            self.exit = True
        return self .f

    def __exit__(self, type, value, traceback):
        if self.exit is None:
            logger.warning("ERROR [iotools.wrapped_open]: `self.exit()` was "
                           "never set.")
        if self.exit:
            self.f.__exit__()


'''arashi_imageファイルにある画像をOpenCVで顔を検出し、切り出すプログラム。
顔の検出にはOpenCVのデフォルトの分類器(https://github.com/opencv/opencv/blob/master/data/haarcascades/haarcascade_frontalface_default.xml)を使用する。
'''
import cv2, os, re,requests, time, bs4
from urllib.request import urlretrieve
from urllib import request as req
from urllib import error,parse
from PIL import Image
import numpy as np
import glob

members = ["二宮和也","櫻井翔","相葉雅紀","大野智","松本潤"]
img_dir = "./arashi_image/"
cascade_file = "./haarcascade_frontalface_alt.xml"#OpenCVのデフォルトの分類器のpath
cascade = cv2.CascadeClassifier(cascade_file)
for member in members:
	files = glob.glob(img_dir+member+"/*.jpg")
	print("{}の写真は{}枚です。顔認識を始めます。".format(member, len(files)))
	for index, file in enumerate(files,1):
		img = cv2.imread(file)
		img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
		#minSizeを5の倍数で5から100まで変更して検出することで、顔の検出率を高める。
		for i in range(1,20):
			minValue = i * 5
			facerect = cascade.detectMultiScale(img_gray, minSize=(minValue,minValue))
			if len(facerect) == 1:
				break
		if len(facerect) != 1:
				continue
		for x,y,w,h in facerect:
			img = img[y:y+h, x:x+w]
		face_path = img_dir+member+"/face"
		if not os.path.exists(face_path):
			os.makedirs(face_path)
		cv2.imwrite(face_path  + "/" + str(index)+".jpg", img)


def substring(string,start,end):
    """
    Returns the substring from start'th character
    to end'th character.  Truncates if start or
    end out of bounds.
    """

    if string == "":
        return ""

    start = max(0,start)

    end = min(len(string)-1,end)

    return string[start:end+1]


import pymorphy2
from russian_tagsets import converters
from rupo.main.tokenizer import Tokenizer, Token

class PyMoprhyRussianMorphology:
    @staticmethod
    def pymorphy_process(input_filename, output_filename):
        """
        Сделать морфоразметку на вход генератору с помощью pymorphy2 и russian-tagsets.

        :param input_filename: входной файл - raw текст.
        :param output_filename: выходной файл - разметка.
        """
        morph = pymorphy2.MorphAnalyzer()
        to_ud = converters.converter('opencorpora-int', 'ud14')
        with open(input_filename, "r", encoding="utf-8") as inp:
            with open(output_filename, "w", encoding="utf-8") as out:
                for line in inp:
                    tokens = Tokenizer.tokenize(line)
                    accepted_types = [Token.TokenType.WORD]
                    tokens = [token for token in tokens if token.token_type in accepted_types]
                    for token in tokens:
                        text = token.text.lower()
                        parse = morph.parse(text)[0]
                        lemma = parse.normal_form
                        ud_tag = to_ud(str(parse.tag), text)
                        pos = ud_tag.split()[0]
                        gram = ud_tag.split()[1]
                        out.write("%s\t%s\t%s\t%s\n" % (text, lemma, pos, gram))
                    out.write("\n")

    @staticmethod
    def vocab_process(input_filename, output_filename, word_form_vocabulary, grammeme_vectorizer):
        """
        Сделать морфоразметку по словарю словоформ.

        :param input_filename: входной файл - raw текст.
        :param output_filename: выходной файл - разметка.
        :param word_form_vocabulary: слоаврь словоформ.
        :param grammeme_vectorizer: векторы граммем.
        :return:
        """
        with open(input_filename, "r", encoding="utf-8") as r:
            with open(output_filename, "w", encoding="utf-8") as w:
                for line in r:
                    tokens = Tokenizer.tokenize(line)
                    accepted_types = [Token.TokenType.WORD]
                    tokens = [token for token in tokens if token.token_type in accepted_types]
                    for token in tokens:
                        text = token.text.lower()
                        word_forms = word_form_vocabulary.get_word_forms_by_text(text)
                        if len(word_forms) == 0:
                            print(text)
                        else:
                            indices = [word_form_vocabulary.get_word_form_index(form) for form in word_forms]
                            index = min(indices)
                            word_form = word_form_vocabulary.get_word_form_by_index(index)
                            name = grammeme_vectorizer.get_name_by_index(word_form.gram_vector_index)
                            pos = name.split("#")[0]
                            gram = name.split("#")[1]
                            w.write("%s\t%s\t%s\t%s\n" % (text, word_form.lemma, pos, gram))
                    w.write("\n")


import json
import codecs
import os

def load_data(file_path):
    if os.path.exists(file_path):
        with codecs.open(file_path, 'r', 'utf-8') as file_handler:
            return json.load(file_handler)
    else:
        raise SystemExit('Wrong file path')

def get_biggest_bar(bars_list):
    return [max(bars_list, key=lambda x:
            x['SeatsCount'])[x] for x in ['Name', 'Address']]

def get_smallest_bar(bars_list):
    return [min(bars_list, key=lambda x:
            x['SeatsCount'])[x] for x in ['Name', 'Address']]

def get_closest_bar(bars_list, usr_longitude, usr_latitude):
    return [min(bars_list, key=lambda x:
            abs(float(x['Latitude_WGS84']) - usr_latitude) +
            abs(float(x['Longitude_WGS84']) - usr_longitude))[x]
            for x in ['Name', 'Address']]

def get_user_coordinate(coordinate_name):
    try:
        return float(input('Enter your'
                           ' current {}\n'.format(coordinate_name.lower())))
    except ValueError:
        raise SystemExit('{} must be a number'.format(coordinate_name))

if __name__ == '__main__':
    list_of_bars = load_data(input('Enter file path\n'))
    user_longitude = get_user_coordinate('Longitude')
    user_latitude = get_user_coordinate('Latitude')
    print('The biggest bar is {},'
          ' located at {}'.format(*get_biggest_bar(list_of_bars)))
    print('The smallest bar is {},'
          ' located at {}'.format(*get_smallest_bar(list_of_bars)))
    print('The closest bar is {},'
          ' located at {}'.format(*get_closest_bar(list_of_bars,
                                                   user_longitude,
                                                   user_latitude)))