# Copyright (c) James Percent and Unlock contributors.
# All rights reserved.
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#    3. Neither the name of Unlock nor the names of its contributors may be used
#       to endorse or promote products derived from this software without
#       specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import numpy as np
import os
import re

__author__ = 'jpercent'

class DataTransformer(object):
    def apply(self, datum):
        return datum

class MobilabMilliVoltsDataTransformer(DataTransformer):
    def __init__(self, channel_sensitivity=500):
        self.channel_sensitivity = channel_sensitivity
        self.resolution_factor = (10 /2**18) * self.channel_sensitivity
        self.micro_to_millis_factor = 1000

    def apply(self, datum):
        return datum * self.resolution_factor * self.micro_to_millis_factor

class PersistentDataTransformer(DataTransformer):
    def __init__(self, transformer, file_name, row_size):
        self.transformer = transformer
        self.file_name = file_name
        self.file_open = False
        self.file_desc = None
        self.row = np.zero(row_size)
        self.row_size = row_size
        self.cursor = 0

    def open(self):
        self.file_open = True
        self.file_desc = open(self.file_name, 'w+')

    def close(self):
        self.file_open = False
        self.file_desc.close()
        self.file_desc = None

    def apply(self, datum):
        assert self.file_open and self.file_desc
        transformed_datum = self.transformer.apply(datum)
        self.row[self.cursor] = datum
        self.cursor += 1
        if self.cursor == self.row_size:
            self.row.tofile(self.file_desc, sep='\t', format='%s')
            self.file_desc.write('\n')
            cursor = 0

        return transformed_datum

class DataLoader(object):
    def __init__(self):
        super(DataLoader, self).__init__()
    def load(self):
        return []
    def store(self, data, file_name):
        pass

class NumpyFileSystemDataLoader(object):
    def __init__(self, file_path, separator='\t', transformer=DataTransformer()):
        super(NumpyFileSystemDataLoader, self).__init__()
        self.file_path = file_path
        self.separator = separator
        self.transformer = transformer

    def load(self):
        np_array = np.array([[self.transformer.apply(float(datum)) for datum in line.split(self.separator)] for line in open(self.file_path)])
        return np_array

    def store(self, file_name, np_array):
        np.savetxt(file_name, np_array, fmt='%s', delimiter='\t')

class DirectoryScanner(DataLoader):
    def __init__(self, directory, file_filter_regex=r'.*', transformer=DataTransformer()):
        self.directory = os.path.join(*directory)
        self.regex = file_filter_regex
        self.transformer = transformer

    def file_generator(self):
        for root, dirs, files in os.walk(self.directory):
            for file_name in files:
                if re.match(self.regex, file_name):
                    loader = NumpyFileSystemDataLoader(os.path.join(root,file_name), transformer=self.transformer)
                    loader.file_name = file_name
                    loader.directory = os.path.basename(os.path.dirname(loader.file_path))
                    yield loader

class Schema(object):
    def __init__(self, data_channels, timestamps, triggers, sampling_rate_hz, start=None, end=None):
        super(Schema, self).__init__()
        self.data_channel_values = data_channels
        self.sample_timestamp_dict = timestamps
        self.trigger_dict = triggers
        self.sampling_rate_hz = sampling_rate_hz
        self.start = start
        self.end = end
        if not start:
            self.start = 0
        if not end:
            self.end = len(self.data_channel_values)
        assert self.start <= self.end

    def get_channel_name(self, channel):
        # this is a hack
        for key, value in self.data_channel_values.items():
            if value == channel[0]:
                return key

    def data_channels(self):
        values = []
        for key, value in self.data_channel_values.items():
            if value >= self.start and value <= self.end:
                values.append(value)
        return sorted(values)

    def all_data_channel_combinations_generator(self):
        values = self.data_channels()
        # we compute the power set here, minus the empty set;  create a set with the first element.  loop through the
        # rest of the elements and create a new set for the element and a new set with the element for each existing
        # subset.
        subsets = None
        for value in values:
            val_set = [value]
            if not subsets:
                subsets = [val_set]
                yield val_set
            else:
                new_subsets = [val_set + subset for subset in subsets]
                subsets.append(val_set)
                for new_subset in new_subsets:
                    yield new_subset
                subsets.extend(new_subsets)

    def single_channel_generator(self):
        values = self.data_channels()
        for value in values:
            yield [value]

    def timestamps(self):
        return sorted([value for key,value in self.timestamp_dict.items()])

    def triggers(self):
        return sorted([value for key,value in self.trigger_dict.items()])

class NumpyDataTable(object):
    def __init__(self, schema, raw_data):
        super(NumpyDataTable, self).__init__()
        self.schema = schema
        self.raw_data = raw_data

    def signal_rows(self):
        rows = self.raw_data.shape[0]
        mod = rows % self.schema.sampling_rate_hz
        if mod:
            rows -= mod
        return rows

    def signal_data(self, columns=None):

        rows = self.signal_rows()
        if not columns:
            columns = self.schema.data_channels()

        return self.raw_data[ :rows , columns]

class OtherMobilabInfo(DataTransformer):
    def __init__(self, sampling_hz=256, high_pass_filter_bound=0.5, low_pass_filter_bound=100):
        self.sampling_hz = sampling_hz
        self.high_pass = high_pass_filter_bound
        self.low_pass = low_pass_filter_bound

# notes from andres; he sent a follow up email that says the channel sensitivity should be 500 not 5/10 (as written below)
#
#                 Fs               = 256;                                % Sampling frequency
#chSensit      = 500/1000;                        % channel sensitivity is 500 microvolts, passing from micro to millivolts
#filtHigh          = 0.5;                                % high pass bound for filter
#filtLow           = 100;                               % low pass bound for filter
#resolutFactor = (2*5/(2^16*4))*(chSensit); % resolution factor to multiply raw data to get it in millivolts
#The data read by unlock is then multiplied by the resolution factor (resolutFactor):
#unlockData.data     = rawData*resolutFactor*1000;         % This gives data in milliVolts, how it is usually

#I dug deeper into the matter of the format of the recorded data to be output in millivolt values and realized I was
# dividing by an extra 1000 value in the formula. The channel sensitivity I gave you yesterday was 500/1000 but it
# should be in reality 500. This means the 1000 should be removed.


#sequencia de Fibonacci
i = int(input('Index a partir de 1: '))
fibonacci = [0,1,1]
count = 1

while count < i:
	fibonacci.append(fibonacci[count] + fibonacci[count + 1])
	count = count + 1

print('index: %d \nNumero da sequencia: %d' %(i, fibonacci[i]))


#!/usr/bin/env python3

# Ref: https://techbase.kde.org/Development/Tutorials/Sensors#Protocol

import subprocess
import xml.etree.ElementTree as ET
import collections

CmdInfo = collections.namedtuple('CmdInfo', ['current', 'min', 'max'])

zero = lambda _: 0

supported_query = {
    'Temperature(C)': CmdInfo(lambda gpu: gpu.find('./temperature/gpu_temp').text.split(' ')[0], zero, zero),
    'Utilization(%)': CmdInfo(lambda gpu: gpu.find('./utilization/gpu_util').text.split(' ')[0], zero, lambda _: 100),
    'Memory_Used(MiB)': CmdInfo(lambda gpu: gpu.find('./fb_memory_usage/used').text.split(' ')[0], zero, lambda gpu: gpu.find('./fb_memory_usage/total').text.split(' ')[0])}

def make_cmd(gpu_id, key):
    return f'{gpu_id}_{key}'

def break_cmd(cmd):
    return cmd.split('_', maxsplit=1)

class GPURenamer:
    def __init__(self):
        self.id2name = {}
        self.name2id = {}

    def name(self, identifier):
        name = self.id2name.get(identifier)
        if name is None:
            name = "GPU%d" % len(self.id2name)
            self.id2name[identifier] = name
            self.name2id[name] = identifier
        return name

    def identifier(self, name):
        return self.name2id[name]

if __name__ == '__main__':
    print('ksysguardd 1.2.0')
    keys = supported_query.keys()
    names_mapped = False
    renamer = GPURenamer()
    while True:
        cmd = input('ksysguardd> ')
        info = ET.fromstring(subprocess.check_output(['nvidia-smi', '-q', '-x']))
        if not names_mapped:
            names_mapped = True
            for gpu in info.findall('gpu'):
                renamer.name(gpu.attrib['id'])
        if cmd == 'monitors':
            for gpu in info.findall('gpu'):
                gpu_id = gpu.attrib['id']
                for key in keys:
                    print(f'{make_cmd(renamer.name(gpu_id), key)}\tfloat')
        else:
            gpu_name, key = break_cmd(cmd)
            gpu_id = renamer.identifier(gpu_name)
            gpu = info.find(f'.//gpu/[@id="{gpu_id}"]')
            if key.endswith('?'):
                cmd_info = supported_query[key[:-1]]
                print(f'{make_cmd(gpu_name, key[:-1])}\t{cmd_info.min(gpu)}\t{cmd_info.max(gpu)}')
            else:
                result = supported_query[key].current(gpu)
                print(result)


from __future__ import absolute_import
from __future__ import unicode_literals

import flask

from git_code_debt.server import logic
from git_code_debt.server import metric_config
from git_code_debt.server.presentation.commit_delta import CommitDeltaPresenter
from git_code_debt.server.presentation.delta import DeltaPresenter
from git_code_debt.server.render_mako import render_template

commit = flask.Blueprint('commit', __name__)

@commit.route('/commit/<sha>')
def show(sha):
    changes = logic.get_metric_changes(flask.g.db, sha)

    commit_deltas = sorted([
        CommitDeltaPresenter.from_data(
            metric_name, DeltaPresenter('javascript:;', change),
        )
        for metric_name, change in changes
    ])

    links = [
        (link_name, link.format(sha=sha))
        for link_name, link in metric_config.commit_links
    ]

    return render_template(
        'commit.mako',
        sha=sha,
        short_sha=sha[:8],
        commit_deltas=commit_deltas,
        links=links,
    )


from pdb import set_trace as bb
import numpy as np
import torch

from torch.autograd import Variable

import pyro
from pyro.distributions import DiagNormal
from pyro.distributions import Bernoulli
import visdom

# from pyro.infer.abstract_infer import LikelihoodWeighting, lw_expectation
# from pyro.infer.importance_sampling import ImportanceSampling
from pyro.infer.kl_qp import KL_QP

import torchvision.datasets as dset
import torchvision.transforms as transforms
mnist = dset.MNIST(
    root='./data',
    train=True,
    transform=None,
    target_transform=None,
    download=True)
print('dataset loaded')

def local_model(i, datum):
    beta = Variable(torch.ones(1)) * 0.5
    c = pyro.sample("class_of_datum_" + str(i), Bernoulli(beta))
    mean_param = Variable(torch.zeros(784), requires_grad=True)
    # do MLE for class means
    m = pyro.param("mean_of_class_" + str(c[0]), mean_param)

    sigma = Variable(torch.ones(m.size()))
    pyro.observe("obs_" + str(i), DiagNormal(m, sigma), datum)
    return c

def local_guide(i, datum):
    alpha = torch.ones(1) * 0.1
    beta_q = Variable(alpha, requires_grad=True)
    guide_params = pyro.param("class_posterior_", beta_q)
    c = pyro.sample("class_of_datum_" + str(i), Bernoulli(guide_params))
    return c

def inspect_posterior_samples(i):
    c = local_guide(i, None)
    mean_param = Variable(torch.zeros(784), requires_grad=True)
    # do MLE for class means
    m = pyro.param("mean_of_class_" + str(c[0]), mean_param)
    sigma = Variable(torch.ones(m.size()))
    dat = pyro.sample("obs_" + str(i), DiagNormal(m, sigma))
    return dat

#grad_step = ELBo(local_model, local_guide, model_ML=true, optimizer="adam")
optim_fct = pyro.optim(torch.optim.Adam, {'lr': .0001})

data = Variable(mnist.train_data).float() / 255.
nr_samples = data.size(0)
grad_step = KL_QP(local_model, local_guide, optim_fct)

d0 = inspect_posterior_samples(0)
d1 = inspect_posterior_samples(1)

vis = visdom.Visdom()

nr_epochs = 50
# apply it to minibatches of data by hand:
for epoch in range(nr_epochs):
    total_loss = 0.
    for i in range(nr_samples):
        # print('starting datum '+str(i))
        # mod_forward=local_model(i,data[i])
        # mod_inv=local_guide(i,data[i])
        loss_sample = grad_step(i, data[i])
        total_loss += loss_sample
        dat = inspect_posterior_samples(i)
        d2 = dat.data.numpy().reshape((28, 28))
        do2 = data[i].view(28, 28).data.numpy()
        # grad_step(i,d)
        #print("loss per sample {}".format(loss_sample))
    vis.image(d2)
    vis.image(do2)
    print("loss per epoch {}".format(total_loss / nr_samples))


# actions: 0 for left, 1 for right, 2 for up, 3 for down

class state:
    Array_States={}
    def __init__(self,value,i,j):
        self.value=value
        self.i=i
        self.j=j
        self.optimal_action=0

# implements state transition function
    def state_transition(self,action,i,j,n):
        if (action==0):
            if(j==0):
                return (i,j)
            if(j!=0):
                return (i,j-1)
        if (action==1):
            if(j==n-1):
                return (i,j)
            if(j!=n-1):
                return (i,j+1)
        if (action==2):
            if(i==0):
                return (i,j)
            if(i!=0):
                return (i-1,j)
        if (action==3):
            if(i==n-1):
                return (i,j)
            if(i!=n-1):
                return (i+1,j)


# -*- coding: utf-8 -*-

from odoo import models, fields, api
import base64
from lxml import etree

import logging
_logger = logging.getLogger(__name__)

from .special_dict import CaselessDictionary

class IrAttachment(models.Model):
    _inherit = 'ir.attachment'

    @api.depends('invoice_ids')
    def _compute_account_invoice_count(self):
        for attach in self:
            try:
                attach.invoice_count = len(attach.invoice_ids)
            except Exception:
                pass
    @api.depends('payment_ids')
    def _compute_account_payment_count(self):
        for attach in self:
            try:
                attach.payment_count = len(attach.payment_ids)
            except Exception:
                pass

    cfdi_uuid = fields.Char("CFDI UUID", copy=False)
    #cfdi_type = fields.Selection([('E','Emisor'),('R','Receptor')],"CFDI Invoice Type", copy=False)
    cfdi_type = fields.Selection([
    ('I', 'Facturas de clientes'), #customer invoice, Emisor.RFC=myself.VAT, Customer invoice
    ('SI', 'Facturas de proveedor'), #Emisor.RFC!=myself.VAT, Supplier bill
    ('E', 'Notas de crédito clientes'), #customer credit note, Emisor.RFC=myself.VAT, Customer credit note
    ('SE', 'Notas de crédito proveedor'), #Emisor.RFC!=myself.VAT, Supplier credit note
    ('P', 'REP de clientes'), #Emisor.RFC=myself.VAT, Customer payment receipt
    ('SP', 'REP de proveedores'), #Emisor.RFC!=myself.VAT, Supplier payment receipt
    ('N', 'Nominas de empleados'), #currently we shall not do anythong with this type of cfdi, Customer Payslip
    ('SN', 'Nómina propia'), #currently we shall not do anythong with this type of cfdi, Supplier Payslip
    ('T', 'Factura de traslado cliente'), #currently we shall not do anythong with this type of cfdi, WayBill Customer
    ('ST', 'Factura de traslado proveedor'),], #currently we shall not do anythong with this type of cfdi, WayBill Supplier
    "Tipo de comprobante",
    copy=False)

    date_cfdi = fields.Date('Fecha')
    rfc_tercero = fields.Char("RFC tercero")
    nombre_tercero = fields.Char("Nombre tercero")
    cfdi_total = fields.Float("Importe")
    creado_en_odoo = fields.Boolean("Creado en odoo", copy=False)
    invoice_ids = fields.One2many("account.invoice",'attachment_id',"Facturas")
    invoice_count = fields.Integer(compute='_compute_account_invoice_count', string='# de facturas', store=True)

    payment_ids = fields.One2many("account.payment",'attachment_id',"Pagos")
    payment_count = fields.Integer(compute='_compute_account_payment_count', string='# de pagos', store=True)

    serie_folio = fields.Char("Folio")

    @api.model
    def create(self, vals):
        ctx = self._context.copy()
        if ctx.get('is_fiel_attachment'):
            datas = vals.get('datas')
            if datas:
                xml_content = base64.b64decode(datas)
                if b'xmlns:schemaLocation' in xml_content:
                    xml_content = xml_content.replace(b'xmlns:schemaLocation', b'xsi:schemaLocation')
                try:
                    tree = etree.fromstring(xml_content)
                except Exception as e:
                    _logger.error('error : '+str(e))
                    raise
                try:
                    ns = tree.nsmap
                    ns.update({'re': 'http://exslt.org/regular-expressions'})
                except Exception:
                    ns = {'re': 'http://exslt.org/regular-expressions'}

                tfd_namespace = {'tfd': 'http://www.sat.gob.mx/TimbreFiscalDigital'}
                tfd_elements = tree.xpath("//tfd:TimbreFiscalDigital", namespaces=tfd_namespace)
                tfd_uuid = tfd_elements and tfd_elements[0].get('UUID')
                cfdi_type = vals.get('cfdi_type','I')

                if cfdi_type in ['I','E','P','N','T']:
                    element_tag = 'Receptor'
                else:
                    element_tag = 'Emisor'
                try:
                    elements = tree.xpath("//*[re:test(local-name(), '%s','i')]"%(element_tag), namespaces=ns)
                except Exception:
                    _logger.info("No encontró al Emisor/Receptor")
                    elements = None
                client_rfc, client_name = '', ''
                if elements:
                    attrib_dict = CaselessDictionary(dict(elements[0].attrib))
                    client_rfc = attrib_dict.get('rfc')
                    client_name = attrib_dict.get('nombre')

                vals.update({
                        'cfdi_uuid' :tfd_uuid,
                        'rfc_tercero' : client_rfc,
                        'nombre_tercero' : client_name,
                        'cfdi_total' : tree.get('Total', tree.get('total')),
                        'date_cfdi' : tree.get('Fecha',tree.get('fecha')),
                        'serie_folio' : tree.get('Folio',tree.get('folio'))
                    })
        return super(IrAttachment, self).create(vals)

    @api.multi
    def action_view_payments(self):
        payments = self.mapped('payment_ids')
        if payments and payments[0].payment_type=='outbound':
            action = self.env.ref('account.action_account_payments_payable').read()[0]
        else:
            action = self.env.ref('account.action_account_payments').read()[0]

        if len(payments) > 1:
            action['domain'] = [('id', 'in', payments.ids)]
        elif len(payments) == 1:
            action['views'] = [(self.env.ref('account.view_account_payment_form').id, 'form')]
            action['res_id'] = payments.ids[0]
        else:
            action = {'type': 'ir.actions.act_window_close'}
        return action

    @api.multi
    def action_view_invoice(self):
        invoices = self.mapped('invoice_ids')
        action = self.env.ref('account.action_invoice_tree1').read()[0]
        if len(invoices) > 1:
            action['domain'] = [('id', 'in', invoices.ids)]
        elif len(invoices) == 1:
            action['views'] = [(self.env.ref('account.invoice_form').id, 'form')]
            action['res_id'] = invoices.ids[0]
        else:
            action = {'type': 'ir.actions.act_window_close'}
        return action

    @api.multi
    def action_renmove_invoice_link(self):
        for attach in self:
            if attach.invoice_ids:
                attach.invoice_ids.write({'attachment_id' : False})
            if attach.payment_ids:
                attach.payment_ids.write({'attachment_id' : False})
            vals = {'res_id':False, 'res_model':False, 'l10n_mx_edi_cfdi_name':False}
            if attach.creado_en_odoo:
                vals.update({'creado_en_odoo':False})
                #attach.creado_en_odoo=False
            attach.write(vals)
        return True


from math import sqrt

if __name__ == '__main__':
    a = float(input())
    b = float(input())
    c = float(input())
    if a == 0:
        raise SystemExit
    if b == 0 and c == 0:
        print(0)
        raise SystemExit
    d = b ** 2 - 4 * a * c
    if d > 0:
        x1 = (-b - sqrt(d)) / (2 * a)
        x2 = (-b + sqrt(d)) / (2 * a)
        if isinstance(x1, int):
            x1 = int(x1)
        else:
            x1 = '{0:.6f}'.format((-b - sqrt(d)) / (2 * a))
        if isinstance(x2, int):
            x2 = int(x2)
        else:
            x2 = '{0:.5f}'.format((-b + sqrt(d)) / (2 * a))
        if x1 > x2:
            print(x2, x1)
        elif x1 < x2:
            print(x1, x2)
    elif d == 0:
        x = -b / (2 * a)
        if isinstance(x, int):
            x = int(x)
        else:
            x = '{0:.6f}'.format(-b / (2 * a))
        print(x)

# решение через создание функции
# def solver(a, b, c):
#     if b == 0 and c == 0:
#         return [0]
#     d = b ** 2 - 4 * a * c
#     if d > 0:
#         x1 = (-b - sqrt(d)) / (2 * a)
#         x2 = (-b + sqrt(d)) / (2 * a)
#         return [x1, x2]
#     elif d == 0:
#         x = -b / (2 * a)
#         return [x]

# if __name__ == '__main__':
#     #unittest.main()
#     a = float(input())
#     b = float(input())
#     c = float(input())
#
#     solution = solver(a, b, c)
#     if len(solution) == 0:
#         raise SystemExit
#     elif len(solution) == 1:
#         x = solution[0]
#         if x.is_integer():
#             x = int(x)
#         else:
#             x = '{0:.6f}'.format(-b / (2 * a))
#         print(x)
#     elif len(solution) == 2:
#         x1 = solution[0]
#         x2 = solution[1]
#         if x1.is_integer():
#             x1 = int(x1)
#         else:
#             x1 = '{0:.6f}'.format(x1)
#         if x2.is_integer():
#             x2 = int(x2)
#         else:
#             x2 = '{0:.5f}'.format(x2)
#         if x1 > x2:
#             print(x2, x1)
#         elif x1 < x2:
#             print(x1, x2)
#     else:
#         raise SystemExit


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import datetime
import logging
import requests

from bs4 import BeautifulSoup
from gnucash import Session, GncNumeric
from gnucash_patch import GncPrice

def get_quote_onvista_bond(isin):
    url = 'http://www.onvista.de/anleihen/snapshot.html?ISIN={}'.format(isin)
    r = requests.get(url)
    soup = BeautifulSoup(r.text)
    price = soup.select('.INHALT #KURSINFORMATIONEN ~ .t span:nth-of-type(2)')[0].get_text()
    currency = 'EUR'
    logging.info('Got quote for %s: %s%%', isin, price)
    return GncNumeric(int(price.replace(',', '')), 100 * 100), currency

def get_quote_onvista_stock(isin):
    url = 'http://www.onvista.de/suche/?searchValue={}'.format(isin)
    r = requests.get(url)
    soup = BeautifulSoup(r.text)
    spans = soup.select('.INHALT ul.KURSDATEN li:nth-of-type(1) span')
    price = spans[0].get_text()
    currency = str(spans[1].get_text())
    logging.info('Got quote for %s: %s %s', isin, price, currency)
    return GncNumeric(int(price.replace(',', '')), 1000), currency

def update_quote(commodity, book):
    fullname = commodity.get_fullname()
    mnemonic = commodity.get_mnemonic()
    isin = commodity.get_cusip()
    if len(isin) != 12:
        return
    logging.info('Processing %s (%s, %s)..', fullname, mnemonic, isin)
    value, currency = None, None
    try:
        if name == 'BOND':
            value, currency = get_quote_onvista_bond(isin)
        else:
            value, currency = get_quote_onvista_stock(isin)
    except:
        logging.exception('Failed to get quote for %s', isin)
    if value and currency:
        table = book.get_table()
        gnc_currency = table.lookup('ISO4217', currency)
        p = GncPrice(book)
        p.set_time(datetime.datetime.now())
        p.set_commodity(commodity)
        p.set_currency(gnc_currency)
        p.set_value(value)
        book.get_price_db().add_price(p)

parser = argparse.ArgumentParser()
parser.add_argument('gnucash_file')
parser.add_argument('--dry-run', action='store_true', help='Do not write anything, noop-mode')

args = parser.parse_args()

logging.basicConfig(level=logging.INFO)
logging.getLogger('urllib3').setLevel(logging.WARN)

s = Session(args.gnucash_file)
try:
    book = s.book
    table = book.get_table()
    for namespace in table.get_namespaces_list():
        name = namespace.get_name()
        if name != 'CURRENCY':
            for commodity in namespace.get_commodity_list():
                update_quote(commodity, book)
    if not args.dry_run:
        s.save()
finally:
    s.end()


#!/usr/bin/env python
import sys
import struct
import re
import argparse

def hex2bytes(text):
    m_iter = re.finditer(r'(?P<hex>((?<=[\s])[0-9a-fA-F]{2}|^[0-9a-fA-F]{2}))(?:$|\s)',
                        text,
                        re.MULTILINE)
    bytes = [int(m.group('hex'), 16) for m in m_iter]
    return bytes

def bytes2bin(bytes):
    bin = "".join([struct.pack('B', x) for x in bytes])
    return bin

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Convert hex in text representation into real binary.")
    args = parser.parse_args()

    input = sys.stdin.read()
    bytes = hex2bytes(input)
    bin = bytes2bin(bytes)
    sys.stdout.write(bin)


# -*- coding: utf-8 -*-
# License, author and contributors information in:
# __openerp__.py file at the root folder of this module.
from openerp import api, models, fields
import logging
_log = logging.getLogger(__name__)

class ResPartner(models.Model):
    _inherit = 'res.partner'

    root_partner_id = fields.Many2one(
        comodel_name='res.partner',
        compute='_compute_root_partner_id',
        store=True,
        string='Root partner')

    @api.one
    @api.depends('parent_id')
    def _compute_root_partner_id(self):
        if self.is_company:
            self.root_partner_id = self.id
        else:
            partner = self
            if partner.parent_id:
                while partner.parent_id:
                    partner = partner.parent_id
                    if partner.is_company:
                        self.root_partner_id = partner.id
            else:
                self.root_partner_id = partner.id


from JumpScale import j
from .LoaderBase import LoaderBase, LoaderBaseObject

from JumpScale.core.extensions.PMExtensionsGroup import PMExtensionsGroup
from JumpScale.core.extensions.PMExtensions import PMExtensions
import JumpScale.baselib.tags

class ActorExtensionsGroup(PMExtensionsGroup):

    """
    ActorExtensionsGroup
    """

    def __init__(self, path):
        """
        ActorExtensionsGroup constructor
        """

        PMExtensionsGroup.__init__(self)

        self.pm_name = "a"
        self.pm_location = "a"

        extensionLoader = PMExtensions(self, 'a.')

        extensionLoader.load(path, True)

        self._activate()

    def _activate(self):
        """
        Activates all extensions in the extention group
        """
        for extensionName in list(self.pm_extensions.keys()):
            extension = self.pm_extensions[extensionName]
            extension.activate()

class Class():
    pass

class GroupAppsClass(object):
    def __init__(self, actorsloader):
        self.actorsloader = actorsloader

    def __getattr__(self, appname):
        app = AppClass(appname)
        setattr(self, appname, app)
        return app

class AppClass(object):
    def __init__(self, appname):
        self._appname = appname

    def __getattr__(self, actorname):
        if actorname in ('__members__', '__methods__', 'trait_names', '_getAttributeNames'):
            return object.__getattr__(self, actorname)
        actor = j.apps.actorsloader.getActor(self._appname, actorname)
        setattr(self, actorname, actor)
        return actor

class ActorsLoader(LoaderBase):
    """
    loader for all actors
    """

    def __init__(self):
        """
        """
        import gevent.coros
        LoaderBase.__init__(self, "actor", ActorLoader)
        self.actorIdToActorLoader = self.id2object
        self.getActorLoaderFromId = self.getLoaderFromId
        self.osiscl = None
        self.lock = gevent.coros.RLock()

    def reset(self):
        j.apps = GroupAppsClass(self)
        j.core.specparser.app_actornames = {}
        j.core.specparser.actornames = []
        j.core.specparser.appnames = []
        j.core.specparser.modelnames = {}
        j.core.specparser.roles = {}
        j.core.specparser.specs = {}
        j.core.codegenerator.classes = {}
        # j.core.portal.active._init()

    def getApps(self):
        result = {}
        for item in list(self.id2object.keys()):
            if item.find("__") != -1:
                app = item.split("__")[0]
                result[app] = 1
        return list(result.keys())

    def getAppActors(self):
        result = []
        for item in list(self.id2object.keys()):
            if item.find("__") != -1:
                app, actor = item.split("__", 2)
                result.append([app, actor])
        return result

    def getActor(self, appname, actorname):

        key = "%s__%s" % (appname.lower(), actorname.lower())

        if key in j.core.portal.active.actors:
            return j.core.portal.active.actors[key]

        print("get actor cache miss for %s %s" % (appname, actorname))

        if key in self.actorIdToActorLoader:
            loader = self.actorIdToActorLoader[key]
            with self.lock:
                aobj = loader.activate()
            j.core.portal.active.actors[key] = aobj
            return j.core.portal.active.actors[key]
        else:
            raise RuntimeError("Cannot find actor from app %s with name %s" % (appname, actorname))

    def existsActorLoader(self, appname, actorname):
        key = "%s__%s" % (appname.lower(), actorname.lower())
        return key in self.id2object

    def existsActor(self, appname, actorname):
        try:
            key = "%s__%s" % (appname.lower(), actorname.lower())
            app = getattr(j.apps, appname, False)
            return app and getattr(app, actorname, False)
        except RuntimeError:
            return False

    def scan(self, path, reset=False):
        paths = path
        if isinstance(paths, basestring):
            paths = [paths]

        for path in paths:
            jsonfiles = j.system.fs.listFilesInDir(path, filter='*.json')
            for jsonfile in jsonfiles:
                j.tools.swaggerGen.loadSpecFromFile(jsonfile)
                j.tools.swaggerGen.generateActors(path)

        return super(ActorsLoader, self).scan(paths, reset)

    def loadOsisTasklets(self, actorobject, actorpath, modelname):
        path = j.system.fs.joinPaths(actorpath, "osis", modelname)
        if j.system.fs.exists(path):
            for method in ["set", "get", "delete", "list", "find", "datatables"]:
                path2 = j.system.fs.joinPaths(path, "method_%s" % method)
                actorobject._te["model_%s_%s" % (modelname, method)] = j.core.taskletengine.get(path2)

class ActorLoader(LoaderBaseObject):

    def __init__(self):
        LoaderBaseObject.__init__(self, "actor")
        self.osiscl = None

    def createDefaults(self, path):
        base = j.system.fs.joinPaths(j.core.portalloader.getTemplatesPath(), "%s" % self.type)
        j.system.fs.copyDirTree(base, path, overwriteFiles=False)

    def raiseError(self, msg, path=None):
        raise RuntimeError("%s\npath was:%s"%(msg,path))

    def loadFromDisk(self, path, reset=False):
        # the name is $appname__actorname all in lowercase
        name = j.system.fs.getDirName(path, True)
        # print "load actor dir:%s"%path
        if name.find("__") != -1:
            app, actor = name.split("__", 1)
        else:
            return self.raiseError("Cannot create actor, name of directory needs to be $appname__$actorname", path=path)
        self._loadFromDisk(path, reset=False)
        self.model.application = app
        self.model.actor = actor

    def _removeFromMem(self):
        print("remove actor %s from memory" % self.model.id)
        j.core.specparser.removeSpecsForActor(self.model.application, self.model.actor)
        j.core.codegenerator.removeFromMem(self.model.application, self.model.actor)
        j.core.portal.active.unloadActorFromRoutes(self.model.application, self.model.actor)
        key = "%s_%s" % (self.model.application.lower(), self.model.actor.lower())
        if key in j.core.portal.active.actors:
            j.core.portal.active.actors.pop(key)

    def reset(self):
        self._removeFromMem()
        self.loadFromDisk(self.model.path, reset=True)
        j.core.portal.active.actorsloader.getActor(self.model.application, self.model.actor)

    def _descrTo1Line(self, descr):
        if descr == "":
            return descr
        descr = descr.strip()
        descr = descr.replace("\n", "\\n")
        # descr=descr.replace("'n","")
        return descr

    def activate(self):
        print("activate actor: %s %s" % (self.model.application, self.model.actor))

        appname=self.model.application
        actorname=self.model.actor
        actorpath=self.model.path
        # parse the specs
        j.core.specparser.parseSpecs("%s/specs" % actorpath, appname=appname, actorname=actorname)

        # retrieve the just parsed spec
        spec = j.core.specparser.getActorSpec(appname, actorname, raiseError=False)
        if spec == None:
            return None

        if spec.tags == None:
            spec.tags = ""
        tags = j.core.tags.getObject(spec.tags)

        spec.hasTasklets = tags.labelExists("tasklets")

        # generate the tasklets for the methods of the actor
        # j.core.codegenerator.generate(spec,"tasklet",codepath=actorpath)

        # generate the class for the methods of the actor
        args = {}
        args["tags"] = tags
        classpath = j.system.fs.joinPaths(actorpath, "methodclass", "%s_%s.py" % (spec.appname, spec.actorname))
        modelNames = j.core.specparser.getModelNames(appname, actorname)

        actorobject = j.core.codegenerator.generate(spec, "actorclass", codepath=actorpath, classpath=classpath,
                                                    args=args, makeCopy=True)()

        if len(modelNames) > 0:
            actorobject.models = Class()

            for modelName in modelNames:
                modelspec = j.core.specparser.getModelSpec(appname, actorname, modelName)
                modeltags = j.core.tags.getObject(modelspec.tags)

                # will generate the tasklets
                modelHasTasklets = modeltags.labelExists("tasklets")
                if modelHasTasklets:
                    j.core.codegenerator.generate(modelspec, "osis", codepath=actorpath, returnClass=False, args=args)

                if spec.hasTasklets:
                    self.loadOsisTasklets(actorobject, actorpath, modelname=modelspec.name)

                classs = j.core.codegenerator.getClassJSModel(appname, actorname, modelName, codepath=actorpath)
                if modelspec.tags == None:
                    modelspec.tags = ""
                index = j.core.tags.getObject(modelspec.tags).labelExists("index")
                tags = j.core.tags.getObject(modelspec.tags)

                db = j.db.keyvaluestore.getMemoryStore()
                osis = False
                if tags.tagExists("dbtype"):
                    dbtypes = [item.lower() for item in tags.tagGet("dbtype").split(",")]
                    if "arakoon" in dbtypes:
                        if dbtypes.index("arakoon") == 0:
                            db = j.db.keyvaluestore.getArakoonStore(modelName)
                    if "fs" in dbtypes:
                        if dbtypes.index("fs") == 0:
                            db = j.db.keyvaluestore.getFileSystemStore(namespace=modelName, serializers=[j.db.serializers.getSerializerType('j')])
                    if "redis" in dbtypes:
                        if dbtypes.index("redis") == 0:
                            db = j.db.keyvaluestore.getRedisStore(namespace=modelName, serializers=[j.db.serializers.getSerializerType('j')])
                    if "osis" in dbtypes:
                        osis = True

                if osis:
                    # We need to check if the correct namespace is existing and
                    # the namespace maps to the actor name, every object is a
                    # category
                    namespacename = actorname
                    if not self.osiscl:
                        import JumpScale.grid.osis
                        self.osiscl = j.clients.osis.getByInstance('main')
                    if actorname not in self.osiscl.listNamespaces():
                        template = tags.tagGet('osis_template', 'modelobjects')
                        self.osiscl.createNamespace(actorname, template=template)
                    if modelName not in self.osiscl.listNamespaceCategories(namespacename):
                        self.osiscl.createNamespaceCategory(namespacename, modelName)
                try:
                    if not osis:
                        actorobject.models.__dict__[modelName] = j.core.osismodel.get(appname, actorname, modelName, classs, db, index)
                    else:
                        actorobject.models.__dict__[modelName] = j.core.osismodel.getRemoteOsisDB(appname, actorname, modelName, classs)
                except Exception as e:
                    raise
                    msg = "Could not get osis model for %s_%s_%s.Error was %s." % (appname, actorname, modelName, e)
                    raise RuntimeError(msg)
        # add routes to webserver
        for methodspec in spec.methods:
            # make sure tasklets are loaded

            methodtags = j.core.tags.getObject(methodspec.tags)
            methodspec.hasTasklets = methodtags.labelExists("tasklets")

            if methodspec.hasTasklets or spec.hasTasklets:
                taskletpath = j.system.fs.joinPaths(actorpath, "methodtasklets", "method_%s" % methodspec.name)
                if not j.system.fs.exists(taskletpath):
                    j.system.fs.createDir(taskletpath)
                    taskletContent = """
def main(j, args, params, actor, tags, tasklet):
    return params

def match(j, args, params, actor, tags, tasklet):
    return True
                    """
                    methodtasklet = j.system.fs.joinPaths(taskletpath, "5_%s.py" % methodspec.name)
                    j.system.fs.writeFile(methodtasklet, taskletContent)
                actorobject._te[methodspec.name] = j.core.taskletengine.get(taskletpath)

            if j.core.portal.active != None:

                params = {}
                for var in methodspec.vars:
                    param = {'optional': False, 'description': '', 'default': None, 'type': None, 'tags': None}
                    tags = j.core.tags.getObject(var.tags)
                    if tags.labelExists("optional"):
                        param['optional'] = True
                        descr = var.description + " (optional)"
                    else:
                        descr = var.description
                    param['description'] = descr
                    param['type'] = var.ttype
                    if var.defaultvalue:
                        param['default'] = var.defaultvalue
                    params[var.name] = param
                    param['tags'] = tags

                tags = j.core.tags.getObject(methodspec.tags)
                if tags.tagExists("returnformat"):
                    returnformat = tags.tagGet("returnformat")
                else:
                    returnformat = None

                auth = not tags.labelExists("noauth")
                methodcall = getattr(actorobject, methodspec.name)
                methodtags = j.core.tags.getObject(methodspec.tags)

                methodtypes = ('post', )
                if 'method' in methodtags.tags:
                    methodtypes = [tag for tag in methodtags.tags['method'].split(',') if tag]
                for methodtype in methodtypes:
                    j.core.portal.active.addRoute(methodcall, appname, actorname, methodspec.name, params=params,
                                                  description=methodspec.description, auth=auth, returnformat=returnformat,
                                                  httpmethod=methodtype)

        # load taskletengines if they do exist
        tepath = j.system.fs.joinPaths(actorpath, "taskletengines")
        if j.system.fs.exists(tepath):
            if "taskletengines" not in actorobject.__dict__:
                actorobject.taskletengines = Class()
            tepaths = j.system.fs.listDirsInDir(tepath)
            for tepath in tepaths:
                actorobject.taskletengines.__dict__[j.system.fs.getBaseName(tepath)] = j.core.taskletengine.get(tepath)

        # LOAD actorobject to qbase tree
        if appname not in j.apps.__dict__:
            j.apps.__dict__[appname] = AppClass(appname)

        if actorname not in j.apps.__dict__[appname].__dict__:
            j.apps.__dict__[appname].__dict__[actorname] = actorobject

        if "runningAppserver" in j.core.portal.__dict__:
            key = "%s_%s" % (spec.appname.lower(), spec.actorname.lower())
            j.core.portal.active.actors[key] = actorobject

        # load extensions
        actorobject.__dict__['extensions'] = ActorExtensionsGroup(j.system.fs.joinPaths(actorpath, "extensions"))

        return actorobject


from data_handling.errors import SeriesError, optional_series_error, optional_mp_error
from data_handling.parse_manifest import parse_manifest
from data_handling.transform_acl import transform_acl
from input.get_dummy_series_dc import get_dummy_series_dc
from input_output.read_file import read_file
from rest_requests.series_requests import series_exists, create_series
from rest_requests.ingest_media_package import create_media_package, add_track, add_attachment, add_catalog, ingest
from rest_requests.request_error import RequestError

def __filter_assets(catalogs, attachments):
    """
    Separate series from episode assets.

    :param catalogs: All catalogs
    :type: list
    :param attachments: All attachments
    :type attachments: list
    :return: Episode catalogs, episode attachments, series catalogs, series attachments
    :rtype: list, list, list, list
    """
    series_catalogs = [catalog for catalog in catalogs if "series" in catalog.flavor]
    episode_catalogs = [catalog for catalog in catalogs if catalog not in series_catalogs]

    series_attachments = [attachment for attachment in attachments if "series" in attachment.flavor]
    episode_attachments = [attachment for attachment in attachments if attachment not in series_attachments]

    return episode_catalogs, episode_attachments, series_catalogs, series_attachments

def recover_mp(mp, base_url, digest_login, workflow_id, ignore_errors):
    """
    Recover a media package by first creating a new media package, then adding tracks, catalogs and attachments to it
    and finally ingesting it. Recover the series as well if necessary.

    :param mp: The media package to be recovered.
    :type mp: MediaPackage
    :param base_url: The base url for the rest requests.
    :type base_url: str
    :param digest_login: User and password for digest authentication.
    :type digest_login: DigestLogin
    :param workflow_id: The workflow to be run on ingest.
    :type workflow_id: str
    :param ignore_errors: Whether to ignore errors and recover the media package anyway
    :type ignore_errors: bool
    :raise MediaPackageError:
    """

    # create empty media package
    new_mp = create_media_package(base_url, digest_login)

    # parse manifest
    series_id, tracks, catalogs, attachments = parse_manifest(mp, ignore_errors)

    if series_id:

        catalogs, attachments, series_catalogs, series_attachments = \
            __filter_assets(catalogs, attachments)  # series can't have tracks, so don't filter those

        if not series_exists(base_url, digest_login, series_id):

            try:
                recover_series(series_id, base_url, digest_login, ignore_errors, series_catalogs, series_attachments)
                print("Recovered series {}.".format(series_id))

            except SeriesError as e:
                print("Series {} could not be recovered: {}".format(series_id, str(e)))
            except RequestError as e:
                print("Series {} could not be recovered: {}".format(series_id, e.error))
            except Exception as e:
                print("Series {} could not be recovered: {}".format(series_id, str(e)))

    for track in tracks:
        try:
            new_mp = add_track(base_url, digest_login, new_mp, track)
        except RequestError as e:
            optional_mp_error("Track {} could not be added.".format(track.id), ignore_errors, e)

    for attachment in attachments:
        try:
            new_mp = add_attachment(base_url, digest_login, new_mp, attachment)
        except RequestError as e:
            optional_mp_error("Attachment {} could not be added.".format(attachment.id), ignore_errors, e)

    for catalog in catalogs:
        try:
            new_mp = add_catalog(base_url, digest_login, new_mp, catalog)
        except RequestError as e:
            optional_mp_error("Catalog {} could not be added.".format(catalog.id), ignore_errors, e)

    workflow = ingest(base_url, digest_login, new_mp, workflow_id)
    return workflow

def recover_series(series_id, base_url, digest_login, ignore_errors, series_catalogs, series_attachments=None):
    """
    Recover a series by recreating it with the series Dublin Core catalog and optionally a series ACL.

    :param series_id: The ID of the series
    :type series_id: str
    :param base_url: The base url for the rest requests.
    :type base_url: str
    :param digest_login: User and password for digest authentication.
    :type digest_login: DigestLogin
    :param ignore_errors: Whether to ignore errors and recover the media package anyway
    :type ignore_errors: bool
    :param series_catalogs: The series catalogs
    :type series_catalogs: list
    :param series_attachments: The series attachments (optional)
    :type series_attachments: list or None
    :raise SeriesError:
    """

    series_dcs = [catalog for catalog in series_catalogs if catalog.flavor == "dublincore/series"]
    series_acls = [attachment for attachment in series_attachments if attachment.flavor == "security/xacml+series"]

    if len(series_dcs) > 1 or len(series_acls) > 1:
        optional_series_error("More than one series Dublin Core catalog or ACL in series {}.".format(series_id),
                              ignore_errors)

    if not series_dcs:
        optional_series_error("Series Dublin Core catalog of series {} missing.".format(series_id), ignore_errors)

    series_dc_contents = []
    series_acl_contents = []

    for series_dc in series_dcs:
        try:
            series_dc_content = read_file(series_dc.path)
            series_dc_contents.append(series_dc_content)
        except Exception as e:
            optional_series_error("Series Dublin Core catalog of series {} could not be read.".format(series_id),
                                  ignore_errors, e)

    for series_acl in series_acls:
        try:
            series_acl_content = read_file(series_acl.path)
        except Exception as e:
            optional_series_error("Series ACL of series {} could not be read.".format(series_id), ignore_errors, e)
            continue

        try:
            series_acl_content = transform_acl(series_acl_content)
            series_acl_contents.append(series_acl_content)
        except Exception as e:
            optional_series_error("Series ACL of series {} could not be transformed.".format(series_id), ignore_errors,
                                  e)

    series_dc_content = series_dc_contents[0] if series_dc_contents else get_dummy_series_dc(series_id)
    series_acl_content = series_acl_contents[0] if series_acl_contents else None

    create_series(base_url, digest_login, series_dc_content, series_acl_content)


number = "9,223,372,036,854,775,807"
cleanedNumber = ''

for i in range(0 , len(number)):
    if number[i] in "0123456789":
        cleanedNumber += number[i]

newNumber = int(cleanedNumber)
print("The number is {} ".format(newNumber))
# Augumentatioin Assessement
x = 23
#adding x
x += 1
print(x)
# Now x = 24

# Subtraction
x -=4
print(x)
# Now x = 20

# Multiplicatioin
x *= 5
print(x)

# Now x = 100

x /= 4
print(x)

# Now x 25

x **= 2
print(x)

# Now x is 625

x %= 60
print(x)

# Now x is 25

greeting = input("Good Morning : ")
greeting += " How was you Day !"
print("hey {}".format(greeting))
greeting *= 5
print(greeting)


# Convolutional Neural Network

# Installing Theano
# pip install --upgrade --no-deps git+git://github.com/Theano/Theano.git

# Installing Tensorflow
# pip install tensorflow

# Installing Keras
# pip install --upgrade keras

# Part 1 - Building the CNN

# Importing the Keras libraries and packages
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense

# Initialising the CNN
classifier = Sequential()

# Step 1 - Convolution
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))

# Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size = (2, 2)))

# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))

# Step 3 - Flattening
classifier.add(Flatten())

# Step 4 - Full connection
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 1, activation = 'sigmoid'))

# Compiling the CNN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

# Part 2 - Fitting the CNN to the images

from keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(rescale = 1./255,
                                   shear_range = 0.2,
                                   zoom_range = 0.2,
                                   horizontal_flip = True)

test_datagen = ImageDataGenerator(rescale = 1./255)

training_set = train_datagen.flow_from_directory('dataset/training_set',
                                                 target_size = (64, 64),
                                                 batch_size = 32,
                                                 class_mode = 'binary')

test_set = test_datagen.flow_from_directory('dataset/test_set',
                                            target_size = (64, 64),
                                            batch_size = 32,
                                            class_mode = 'binary')

# CheckPoints
from keras.callbacks import ModelCheckpoint
#checkpointer = ModelCheckpoint(filepath="tmp/weights.hdf5", verbose=1, save_best_only=True)

# checkpoint
#filepath="tmp/weights-improvement-{epoch:02d}-{val_acc:.2f}.hdf5"
#checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
#callbacks_list = [checkpoint]

# best checkpoint
filepath="tmp/weights.best.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
callbacks_list = [checkpoint]

classifier.fit_generator(training_set,
                         steps_per_epoch = 8000,
                         epochs = 25,
                         validation_data = test_set,
                         validation_steps = 2000,
                         callbacks=callbacks_list)

# --------------------------
## Try to save the Model weights which can be called later
# serialize model to JSON

model_json = classifier.to_json()
with open("cnn_model.json", "w") as json_file:
    json_file.write(model_json)

# serialize weights to HDF5

classifier.save_weights("cnn_model.h5")
print("Saved model to disk")
# --------------------------
#classifer.save('One_convolutional_layer.h5')

# Part 3 - Making new predictions
#==============================================================================
#
# import numpy as np
# from keras.preprocessing import image
# test_image = image.load_img('dataset/single_prediction/cat_or_dog_1.jpg', target_size = (64, 64))
# test_image = image.img_to_array(test_image)
# test_image = np.expand_dims(test_image, axis = 0)  # 4th dim becoz it predict method requires a batch
# result = classifier.predict(test_image)
# training_set.class_indices
# if result[0][0] == 1:
#     prediction = 'dog'
# else:
#     prediction = 'cat'
#==============================================================================


"""Tests for pycounter.sushi"""

from __future__ import absolute_import

import unittest
import os
from httmock import urlmatch, HTTMock
import datetime

from pycounter import sushi
import pycounter.exceptions

@urlmatch(netloc=r'(.*\.)?example\.com$')
def sushi_mock(url, request):
    path = os.path.join(os.path.dirname(__file__),
                        'data', 'sushi_simple.xml')
    with open(path, 'rb') as datafile:
        return datafile.read().decode('utf-8')

@urlmatch(netloc=r'(.*\.)?example\.com$')
def bogus_mock(url, request):
    return "Bogus response with no XML"

class TestHelpers(unittest.TestCase):
    def test_ns(self):
        self.assertEqual(
            sushi._ns("sushi", "name"),
            "{http://www.niso.org/schemas/sushi}name")

class TestConvertRawSimple(unittest.TestCase):
    def setUp(self):
        path = os.path.join(os.path.dirname(__file__),
                            'data', 'sushi_simple.xml')
        with open(path, 'rb') as datafile:
            self.report = sushi._raw_to_full(datafile.read())

    def test_report(self):
        self.assertEqual(self.report.report_type, u'JR1')
        self.assertEqual(self.report.report_version, 4)

class TestSushiRequest(unittest.TestCase):
    def setUp(self):
        with HTTMock(sushi_mock):
            self.report = sushi.get_report('http://www.example.com/Sushi',
                                           datetime.date(2015, 1, 1),
                                           datetime.date(2015, 1, 31)
                                           )

    def test_report(self):
        self.assertEqual(self.report.report_type, u'JR1')
        self.assertEqual(self.report.report_version, 4)

class TestBogusXML(unittest.TestCase):
    def test_request(self):
        with HTTMock(bogus_mock):
            self.assertRaises(pycounter.exceptions.SushiException,
                              sushi.get_report,
                              'http://www.example.com/bogus',
                              datetime.date(2015, 1, 1),
                              datetime.date(2015, 1, 31)
                              )


from datetime import date
from datetime import datetime
from ftw.builder import Builder
from ftw.builder import create
from ftw.testbrowser import browsing
from ftw.testbrowser.pages import editbar
from ftw.testbrowser.pages.statusmessages import info_messages
from opengever.meeting.period import Period
from opengever.meeting.reopen import ReopenMeeting
from opengever.testing import IntegrationTestCase
import pytz

class TestReopenMeeting(IntegrationTestCase):

    features = ('meeting',)
    REOPEN_MEETING_ACTION = 'Reopen meeting'

    def test_cannot_reopen_meeting_if_feature_is_deactivated(self):
        self.login(self.committee_responsible)

        self.schedule_ad_hoc(self.meeting, 'Foo')
        self.schedule_paragraph(self.meeting, u'Para')
        agenda_item = self.schedule_proposal(
            self.meeting, self.submitted_proposal
        )
        agenda_item.decide()

        self.deactivate_feature('meeting')
        self.assertEqual(
            ["Can't reopen meeting when feature is disabled."],
            ReopenMeeting(self.meeting.model).get_errors())

    def test_reopen_held_meeting(self):
        self.login(self.committee_responsible)

        self.schedule_ad_hoc(self.meeting, 'Foo')
        self.schedule_paragraph(self.meeting, u'Para')
        agenda_item = self.schedule_proposal(
            self.meeting, self.submitted_proposal
        )
        agenda_item.decide()
        model = self.meeting.model
        period = Period.get_current(
            model.committee.resolve_committee(),
            model.start.date()
        )

        self.assertEqual(3, period.decision_sequence_number)
        self.assertEqual(2, period.meeting_sequence_number)
        self.assertEqual('held', model.workflow_state)
        self.assertEqual(
            ['pending', 'pending', 'decided'],
            [each.workflow_state for each in model.agenda_items]
        )
        self.assertEqual(
            [2, None, 3],
            [each.decision_number for each in model.agenda_items]
        )
        self.assertEqual(2, model.meeting_number)

        ReopenMeeting(model).reopen_meeting()

        self.assertEqual('pending', model.workflow_state)
        self.assertEqual(
            ['pending', 'pending', 'pending'],
            [each.workflow_state for each in model.agenda_items]
        )
        self.assertEqual(1, period.decision_sequence_number)
        self.assertEqual(1, period.meeting_sequence_number)

    def test_reopen_closed_meeting(self):
        self.login(self.committee_responsible)

        agenda_item = self.schedule_ad_hoc(self.meeting, 'Foo')
        self.schedule_paragraph(self.meeting, u'Para')
        agenda_item.decide()
        model = self.meeting.model
        model.close()
        period = Period.get_current(
            model.committee.resolve_committee(),
            model.start.date()
        )

        self.assertEqual(2, period.decision_sequence_number)
        self.assertEqual(2, period.meeting_sequence_number)
        self.assertEqual('closed', model.workflow_state)
        self.assertEqual(
            ['decided', 'pending'],  # the paragraph is always pending
            [each.workflow_state for each in model.agenda_items]
        )
        self.assertEqual(
            [2, None],
            [each.decision_number for each in model.agenda_items]
        )
        self.assertEqual(2, model.meeting_number)

        ReopenMeeting(model).reopen_meeting()

        self.assertEqual('pending', model.workflow_state)
        self.assertEqual(
            ['pending', 'pending'],
            [each.workflow_state for each in model.agenda_items]
        )
        self.assertEqual(1, period.decision_sequence_number)
        self.assertEqual(1, period.meeting_sequence_number)

    def test_error_invalid_meeting_state(self):
        self.login(self.committee_responsible)

        reopener = ReopenMeeting(self.meeting.model)
        self.assertEqual(
            [u"Can't reopen meeting in state 'pending'.",
             u"Unexpected state, can't reset meeting number."],
            reopener.get_errors()
        )

    def test_error_invalid_agenda_item_state(self):
        self.login(self.committee_responsible)

        agenda_item = self.schedule_ad_hoc(self.meeting, 'Foo')
        agenda_item.decide()
        agenda_item.decision_number = None

        reopener = ReopenMeeting(self.meeting.model)
        self.assertEqual(
            [u"Unexpected state, have agenda items but "
             u"can't reset decision number."],
            reopener.get_errors()
        )

    def test_error_newer_meetings_with_meeting_number_exist(self):
        self.login(self.committee_responsible)

        agenda_item = self.schedule_ad_hoc(self.meeting, 'Foo')
        self.schedule_paragraph(self.meeting, u'Para')
        agenda_item.decide()
        model = self.meeting.model
        model.close()

        meeting_dossier = create(
            Builder('meeting_dossier')
            .within(self.leaf_repofolder)
        )
        newer_meeting = create(
            Builder('meeting')
            .having(
                title=u'8. Sitzung der Rechnungspr\xfcfungskommission',
                committee=self.committee.load_model(),
                start=datetime(2016, 9, 12, 19, 30, tzinfo=pytz.UTC),
            )
            .link_with(meeting_dossier)
        )
        agenda_item = self.schedule_ad_hoc(newer_meeting, 'Foo')
        agenda_item.decide()

        reopener = ReopenMeeting(self.meeting.model)
        self.assertEqual(
            [u"The meetings '8. Sitzung der Rechnungspr\xfcfungskommission' "
             u"need to be reopened first."],
            reopener.get_errors()
        )

    def test_only_considers_newer_meetings_from_same_period(self):
        self.login(self.committee_responsible)

        agenda_item = self.schedule_ad_hoc(self.meeting, 'Foo')
        self.schedule_paragraph(self.meeting, u'Para')
        agenda_item.decide()
        model = self.meeting.model
        model.close()

        # Create meeting with higher meeting number but in different period
        period = create(Builder('period').having(
                title=u'2015',
                start=date(2015, 1, 1),
                end=date(2015, 12, 31)).within(self.committee))
        period.meeting_sequence_number = 20

        meeting_dossier = create(
            Builder('meeting_dossier')
            .within(self.leaf_repofolder)
        )
        old_meeting = create(
            Builder('meeting')
            .having(
                title=u'8. Sitzung der Rechnungspr\xfcfungskommission',
                committee=self.committee.load_model(),
                start=datetime(2015, 9, 12, 19, 30, tzinfo=pytz.UTC),
            )
            .link_with(meeting_dossier)
        )
        agenda_item = self.schedule_ad_hoc(old_meeting, 'Foo')
        agenda_item.decide()

        self.assertTrue(old_meeting.meeting_number > self.meeting.model.meeting_number)
        reopener = ReopenMeeting(self.meeting.model)
        self.assertEqual([], reopener.get_errors())

    @browsing
    def test_browser_reopen_meeting(self, browser):
        self.login(self.committee_responsible)

        self.schedule_ad_hoc(self.meeting, 'Foo')
        agenda_item = self.schedule_proposal(
            self.meeting, self.submitted_proposal
        )
        agenda_item.decide()
        model = self.meeting.model
        period = Period.get_current(
            model.committee.resolve_committee(),
            model.start.date()
        )

        self.assertEqual(3, period.decision_sequence_number)
        self.assertEqual(2, period.meeting_sequence_number)
        self.assertEqual('held', model.workflow_state)
        self.assertEqual(
            ['pending', 'decided'],
            [each.workflow_state for each in model.agenda_items]
        )
        self.assertEqual(
            [2, 3],
            [each.decision_number for each in model.agenda_items]
        )
        self.assertEqual(2, model.meeting_number)

        self.login(self.manager, browser)
        browser.open(model.get_url())
        editbar.menu_option('Actions', self.REOPEN_MEETING_ACTION).click()
        browser.find('Confirm').click()

        self.assertEqual([u"The meeting has been reopened."], info_messages())

        self.assertEqual('pending', model.workflow_state)
        self.assertEqual(
            ['pending', 'pending'],
            [each.workflow_state for each in model.agenda_items]
        )
        self.assertEqual(1, period.decision_sequence_number)
        self.assertEqual(1, period.meeting_sequence_number)

    @browsing
    def test_non_manager_cannot_reopen(self, browser):
        self.login(self.administrator, browser)
        browser.open(self.meeting.model.get_url())
        self.assertNotIn(
            self.REOPEN_MEETING_ACTION, editbar.menu_options('Actions')
        )
        with browser.expect_unauthorized():
            browser.open(self.meeting.model.get_url(view='reopen_meeting'))


from scipy.optimize import curve_fit
import numpy as np
import matplotlib.pyplot as plt
import csv
exp=lambda x,a,mu,sigma,y0:y0+a*np.exp(-1/2/sigma**2*(x-mu)**2)
def max_finder(y):
    z=[]
    for i in range(10,len(y)-10):
        if(y[i]>=y[i+1] and y[i]>0.01 and y[i]>=y[i-1] and y[i]>=y[i+2] and y[i]>=y[i-2] and y[i]>=y[i+3] and y[i]>=y[i-3] and y[i]>=y[i+4] and y[i]>=y[i-4]): z.append(i)
    return z
f=open("NewFile1.csv")

data=csv.reader(f)
t=[]
U=[]
for row in data:
    t.append(row[0])
    U.append(row[1])
t=t[2::1]
U=U[2::1]
for i in range(len(U)):
    U[i]=float(U[i])
    t[i]=float(t[i])
plt.xlim(-0.05,0.04)
z=max_finder(U)
Uneu=[]
tneu=[]
for i in range(len(z)):
    Uneu.append(U[z[i]])
    tneu.append(t[z[i]])
werte,fehler=curve_fit(exp,tneu,Uneu)

plt.plot(t,U,label="Messwerte")
plt.plot(t,exp(t,*werte),color="red",label="Gauß-Fit \n$y_0+A*e^\\frac{-(x-\mu)^2}{2*\sigma^2}$"+"\n$\mu ={:.3f}$\n$\sigma ={:.3f}$\n$y_0={:.3f}$\n$A={:.3f}$".format(werte[1],werte[2],werte[3],werte[0]))
plt.legend()
plt.plot(0,0.15)
plt.title("Diagramm 2: Signal bei Interferenz")
plt.xlabel("Zeit $t [s]$")
plt.ylabel("CH 1 $[Volt]$")
plt.savefig("liebsteDiagramm 2.jpg")
print(werte[2])
print(fehler[2][2])

#plt.scatter(x_werte,y_werte)

plt.show()


"""HassOS Cli docker object."""
import logging

import docker

from ..coresys import CoreSysAttributes
from .interface import DockerInterface

_LOGGER: logging.Logger = logging.getLogger(__name__)

class DockerHassOSCli(DockerInterface, CoreSysAttributes):
    """Docker Supervisor wrapper for HassOS Cli."""

    @property
    def image(self):
        """Return name of HassOS CLI image."""
        return f"homeassistant/{self.sys_arch.supervisor}-hassio-cli"

    def _stop(self, remove_container=True):
        """Don't need stop."""
        return True

    def _attach(self, tag: str):
        """Attach to running Docker container.
        Need run inside executor.
        """
        try:
            image = self.sys_docker.images.get(f"{self.image}:{tag}")

        except docker.errors.DockerException:
            _LOGGER.warning("Can't find a HassOS CLI %s", self.image)

        else:
            self._meta = image.attrs
            _LOGGER.info(
                "Found HassOS CLI %s with version %s", self.image, self.version
            )


import jinja2
import openpyxl
import os
import string
import sys

from ntmap import *

# Get the path of this file
dir_path = os.path.dirname(os.path.realpath(__file__))

# Load the Jinja2 template
loader = jinja2.FileSystemLoader(dir_path)
environment = jinja2.Environment(loader=loader)
controls_template = environment.get_template('security_control.j2')
crm_template = environment.get_template('crm.j2')

# Open the SCTM Excel workbook rw so we can look at row dimensions.
wb = openpyxl.load_workbook(sys.argv[1], read_only=False)
ws = wb['Matrix']

# Build the column index
column_index = {}
for row in ws.iter_rows(min_row=1, max_row=1):
    for i, cell in enumerate(row):
        column_index[i] = cell.value

# Init the sctm dict, to be keyed to control title
sctm = {}

# Iterate over every row in the sheet, starting at the 2nd row
for i, row in enumerate(ws.iter_rows(min_row=2), start=2):
    # Skip hidden rows
    if ws.row_dimensions[i].hidden is True:
        continue

    # Temporary dict for this iteration
    r = {}

    # Iterate over the cells and insert the values into the temporary dict r
    # keyed with the column name.
    for j, cell in enumerate(row):
        # If the cell is non-blank, insert the cell's contents into the temp
        # dict r keyed by column name.
        if cell.value and cell.value != '':
            r[column_index[j]] = cell.value
        else:
            r[column_index[j]] = 'undefined'

    # Key for the temp dict
    ref_num = r['NIST Ref #']

    # Build the control title with nice capitalization
    r["Title"] = " ".join(w.capitalize() for w in str(nt_map[ref_num]).split())

    # First we check to see if this control has been processed before
    # Remove all those pesky new lines
    r['Requirements'] = r['Requirements'].replace(u"\n", u" ")
    if sctm.get(ref_num):
        # If it has, we concat the control language
        sctm[ref_num][0]['Original Requirements'] += (u" " + r['Requirements'])
    else:
        # Otherwise we start fresh
        r['Original Requirements'] = r['Requirements']

    # Build an alphabetic part number to jive with the FedRAMP template
    r['Part'] = string.lowercase[int(r['Org Ref #'].split("_")[-1])]

    # Now we build a list of control parts and store it under the same key
    if sctm.get(ref_num):
        sctm[ref_num].append(r)
    else:
        sctm[ref_num] = [r]

# Process the master sctm data structure and look for Tenant responsible
# controls
crm = {}
for key in sctm.keys():
    for part in sctm[key]:
        if u"OpenShift Tenant" in part['Role']:
            crm[key] = sctm[key]

output = controls_template.render(sctm=sctm)
with open('controls.rst', 'w') as controls_file:
    controls_file.write(output.encode('utf-8'))

output = crm_template.render(sctm=crm)
with open('crm.rst', 'w') as controls_file:
    controls_file.write(output.encode('utf-8'))


#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import warnings
import itertools
import numpy as np
from scipy.spatial.distance import pdist, squareform
from pyannote.generators.batch import BaseBatchGenerator
from pyannote.audio.generators.labels import FixedDurationSequences
from pyannote.audio.embedding.callbacks import UpdateGeneratorEmbedding

class TripletGenerator(object):
    """Triplet generator for triplet loss sequence embedding

    Generates ([Xa, Xp, Xn], 1) tuples where
      * Xa is the anchor sequence (e.g. by speaker S)
      * Xp is the positive sequence (also uttered by speaker S)
      * Xn is the negative sequence (uttered by a different speaker)

    and such that d(f(Xa), f(Xn)) < d(f(Xa), f(Xp)) + margin where
      * f is the current state of the embedding network (being optimized)
      * d is the (euclidean or cosine) distance
      * margin is the triplet loss margin (e.g. 0.2, typically)

    Parameters
    ----------
    extractor: YaafeFeatureExtractor
        Yaafe feature extraction (e.g. YaafeMFCC instance)
    file_generator: iterable
        File generator (the training set, typically)
    distance: {'sqeuclidean', 'cosine'}
        Distance for which the embedding is optimized. Defaults to 'sqeuclidean'.
    margin : float, optional
        Defaults to 0.2.
    duration: float, optional
    step: float, optional
        Duration and step of sliding window (in seconds).
        Default to 3s and half duration.
    min_duration: float, optional
        Sequence minimum duration. When provided, generates sequences with
        random duration in range [min_duration, duration]. Defaults to
        fixed-duration sequences.
    per_label: int, optional
        Number of samples per label. Defaults to 40.
    per_fold: int, optional
        When provided, randomly split the training set into
        fold of `per_fold` labels (e.g. 40) after each epoch.
        Defaults to using the whole training set.
    batch_size: int, optional
        Batch size. Defaults to 32.
    """

    def __init__(self, extractor, file_generator,
                 distance='sqeuclidean', margin=0.2,
                 duration=3.0, min_duration=None, step=None,
                 per_fold=0, per_label=40, batch_size=32):

        super(TripletGenerator, self).__init__()

        self.extractor = extractor
        self.file_generator = file_generator
        self.distance = distance
        self.margin = margin
        self.duration = duration
        self.min_duration = min_duration
        self.step = step
        self.per_fold = per_fold
        self.per_label = per_label
        self.batch_size = batch_size

        self.generator_ = FixedDurationSequences(
            self.extractor,
            duration=self.duration,
            min_duration=self.min_duration,
            step=self.step,
            batch_size=-1)

        self.triplet_generator_ = self.iter_triplets()

        # consume first element of generator
        # this is meant to pre-generate all labeled sequences once and for all
        # and get the number of unique labels into self.n_labels
        next(self.triplet_generator_)

    def iter_triplets(self):

        # pre-generate all labeled sequences (from the whole training set)
        # this might be huge in memory
        X, y = [], []
        for batch_sequences, batch_labels in self.generator_(self.file_generator):
            X.append(batch_sequences)
            y.append(batch_labels)
        X = np.vstack(X)
        y = np.hstack(y)

        # unique labels
        unique, y, counts = np.unique(y, return_inverse=True, return_counts=True)
        self.n_labels = len(unique)

        # warn that some labels have very few training samples
        too_few_samples = np.sum(counts < self.per_label)
        if too_few_samples > 0:
            msg = '{n} labels (out of {N}) have less than {per_label} training samples.'
            warnings.warn(msg.format(n=too_few_samples,
                                     N=self.n_labels,
                                     per_label=self.per_label))

        # HACK (see __init__ for details on why this is done)
        yield

        # infinite loop
        while True:

            # shuffle labels
            shuffled_labels = np.random.choice(self.n_labels,
                                               size=self.n_labels,
                                               replace=False)

            if self.per_fold < 1:
                self.per_fold = self.n_labels

            # take them per_fold per per_fold
            for k in range(self.n_labels / self.per_fold):
                from_label = k * self.per_fold
                to_label = (k+1) * self.per_fold
                labels = shuffled_labels[from_label: to_label]

                # select min(per_label, count) sequences
                # at random for each label

                # per_label[i] contains the actual number of examples
                # available for ith label -- as it may actually be smaller
                # than the requested self.per_label for small classes (e.g.
                # for speakers without only a few seconds of speech)
                per_label = []

                # indices contains the list of indices of all sequences
                # to be used for later triplet selection
                indices = []

                for label in labels:

                    # number of available sequences for current label
                    # per_label.append(min(self.per_label, counts[label]))
                    per_label.append(self.per_label)

                    # NOTE the impact of choosing per_label instead of
                    # min(per_label, counts[label]) should be evaluated.
                    # indeed, for labels with a counts[label] smaller than
                    # per_label, the following 'choice' will repeat sequences.
                    # is this really an issue?

                    # randomly choose this many sequences
                    # from the set of available sequences
                    i = np.random.choice(
                        np.where(y == label)[0],
                        size=per_label[-1],
                        replace=True)

                    # append indices of selected sequences
                    indices.append(i)

                # after this line, per_label[i] will contain the position of
                # the first sequence of ith label so that the range
                # per_label[i]: per_label[i+1] points to the indices
                # corresponding to all sequences from ith label
                per_label = np.hstack([[0], np.cumsum(per_label)])

                # turn indices into a 1-dimensional numpy array.
                # combined with (above) per_label, it can be used
                # to get all indices of sequences from a given label
                indices = np.hstack(indices)

                # pre-compute pairwise distances d(f(X), f(X')) between every
                # pair (X, X') of selected sequences, where f is the current
                # state of the embedding being optimized, and d is the
                # euclidean distance

                # selected sequences
                sequences = X[indices]
                # their embeddings (using current state of embedding network)
                embeddings = self.embedding.transform(
                    sequences, batch_size=self.batch_size)

                # pairwise squared euclidean distances
                distances = squareform(pdist(embeddings, metric=self.distance))

                for i in range(self.per_fold):

                    positives = list(range(per_label[i], per_label[i+1]))
                    negatives = list(range(per_label[i])) + list(range(per_label[i+1], per_label[-1]))

                    # loop over all (anchor, positive) pairs for current label
                    for anchor, positive in itertools.combinations(positives, 2):

                        # find all negatives within the margin
                        d = distances[anchor, positive]
                        within_margin = np.where(
                            distances[anchor, negatives] < d + self.margin)[0]

                        # choose one at random (if at least one exists)
                        if len(within_margin) < 1:
                            continue
                        # TODO / add an option to choose the most difficult one
                        negative = negatives[np.random.choice(within_margin)]

                        yield [sequences[anchor], sequences[positive], sequences[negative]], 1

                        # FIXME -- exit this loop when an epoch has ended
                    # FIXME -- exit this loop when an epoch has ended
                # FIXME -- exit this loop when an epoch has ended

    def __iter__(self):
        return self

    def next(self):
        return self.__next__()

    def __next__(self):
        return next(self.triplet_generator_)

    @property
    def shape(self):
        return self.generator_.shape

    def signature(self):
        shape = self.shape
        return (
            [
                {'type': 'ndarray', 'shape': shape},
                {'type': 'ndarray', 'shape': shape},
                {'type': 'ndarray', 'shape': shape}
            ],
            {'type': 'scalar'}
        )

    def callbacks(self, extract_embedding=None):
        callback = UpdateGeneratorEmbedding(
            self, extract_embedding=extract_embedding, name='embedding')
        return [callback]

class TripletBatchGenerator(BaseBatchGenerator):
    """Triplet generator for triplet loss sequence embedding

    Generates ([Xa, Xp, Xn], 1) batch tuples where
      * Xa are anchor sequences (e.g. by speaker S)
      * Xp are positive sequences (also uttered by speaker S)
      * Xn are negative sequences (uttered by a different speaker)

    and such that d(f(Xa), f(Xn)) < d(f(Xa), f(Xp)) + margin where
      * f is the current state of the embedding network (being optimized)
      * d is the euclidean distance
      * margin is the triplet loss margin (e.g. 0.2, typically)

    Parameters
    ----------
    feature_extractor: YaafeFeatureExtractor
        Yaafe feature extraction (e.g. YaafeMFCC instance)
    file_generator: iterable
        File generator (the training set, typically)
    distance: {'sqeuclidean', 'cosine'}
        Distance for which the embedding is optimized. Defaults to 'sqeuclidean'.
    margin : float, optional
        Defaults to 0.2.
    duration: float, optional
    step: float, optional
        Duration and step of sliding window (in seconds).
        Default to 3s and half step.
    min_duration: float, optional
        Sequence minimum duration. When provided, generates sequences with
        random duration in range [min_duration, duration]. Defaults to
        fixed-duration sequences.
    per_label: int, optional
        Number of samples per label. Defaults to 40.
    per_fold: int, optional
        Randomly split the training set into disjoint folds of `per_fold`
        labels. Defaults to using one big fold.
    batch_size: int, optional
        Batch size. Defaults to 32.
    """
    def __init__(self, feature_extractor, file_generator,
                 distance='sqeuclidean', margin=0.2,
                 duration=3.0, min_duration=None, step=None,
                 per_fold=0, per_label=40, batch_size=32):

        self.triplet_generator_ = TripletGenerator(
            feature_extractor, file_generator,
            margin=margin, distance=distance,
            duration=duration, min_duration=min_duration, step=step,
            per_fold=per_fold, per_label=per_label, batch_size=batch_size)

        super(TripletBatchGenerator, self).__init__(
            self.triplet_generator_, batch_size=batch_size)

    def signature(self):
        return self.triplet_generator_.signature()

    @property
    def shape(self):
        return self.triplet_generator_.shape

    def get_steps_per_epoch(self, protocol, subset='train'):
        """
        Parameters
        ----------
        protocol : pyannote.database.protocol.protocol.Protocol
        subset : {'train', 'development', 'test'}, optional

        Returns
        -------
        steps_per_epoch : int
            Number of batches per epoch.
        """
        n_labels = len(protocol.stats(subset)['labels'])
        per_label = self.triplet_generator_.per_label
        samples_per_epoch = per_label * (per_label - 1) * n_labels
        return samples_per_epoch // self.batch_size

    def callbacks(self, extract_embedding=None):
        return self.triplet_generator_.callbacks(
            extract_embedding=extract_embedding)


__author__ = 'Kalyan'

notes = '''
    Sets are  unordered collection of elements without duplicates. They are designed to be very efficient with respect to
    set operations like membership, union, intersection. Doing similar operations on lists will be very inefficient as lists
    are optimized for scenarios which require ordering.

    So set will never follow the order in which you have inserted the values into it.
    And here is the important conclusion for this, we cannot use the index values to get the
    set members.

    Duplicate values are removed if entered into a set.
'''

from placeholders import *

def test_set_type():
    test_set = {"one", "two", "three"}   # note the new syntax
    assert 'set' == type(test_set).__name__

def test_set_empty():
    empty_set_wrong = {}  #curly braces are used for both sets and dicts, so how do you disambiguate?
    assert False == isinstance(empty_set_wrong, set)
    # If it was 'dict' in place of 'set' then it would be True
    empty_set = set()
    assert True == isinstance(empty_set, set)
    assert 0 == len(empty_set)

def test_set_length():
    fruits = {"apple", "apple", "apple"}
    assert 1 == len(fruits)  #are duplicates removed? Yes, duplicates are removed

    veggies = {"beetroot", "potato", "spinach"}
    assert 3 == len(veggies)

def test_set_creation():
    """
    sets can be created from any sequence like string, list or a tuple.
    """
    test_list = [1, 2, 1, 3]
    set1 = set(test_list) # Converting it from list(contains duplicates) to set(duplicates are removed)
    assert {1, 2, 3} == set1

    test_string = "applepie"
    set2 = set(test_string)
    assert {'a', 'p', 'l', 'e', 'i'} == set2    # As sets don't follow any order of arrangement you can hae these random

    test_dict = {1: "one", 2: "two"}
    set3 = set(test_dict)   # When you convert a dict to set it'll only take key values
    assert {1, 2} == set3

    set4 = set(test_dict.values())
    assert {'one', 'two'} == set4
# Important
    set5 = set(test_dict.items())
    assert {(1, 'one'), (2, 'two')} == set5

def test_set_membership():
    fruits = {"apple", "mango", "kiwi"}
    assert True == ("apple" in fruits)
    assert False == ("dog" in fruits)

def test_set_operations():
    set1 = {"one", "two", "three"}
    set2 = {"three", "four"}

    all = set1 | set2 # union
    assert {'one', 'three', 'two', 'four'} == all

    common = set1 & set2 # intersection
    assert {'three'} == common

    diff1 = set1 - set2 # set difference
    assert {'one', 'two'} == diff1

    diff2 = set2 - set1
    assert {'four'} == diff2

    diff3 = set1 - set1
    assert set() == diff3

    diff4 = set1.symmetric_difference(set2)
    assert {'two', 'four', 'one'} == diff4

    #read up help on other methods using the help method in the python console.

def test_set_valid_members():
    # When you add items like below they get added from index 0 and push the old to items forward
    test_set = set()
    test_set.add("hello")
    test_set.add(1)
    test_set.add((1, 2))

    try:
        test_set.add([])
    except TypeError as te:
        print(te)
        assert True # "unhashable type: 'list'"

    try:
        test_set.add((1, []))
    except TypeError as te:
        print(te)
        assert True

    assert {(1, 2), 1, 'hello'} == test_set

#apply what you have learnt here.
def common(input1, input2):
    """
    Given two lists, return the common elements in both the lists as a set
    """
    return set(list(set(input1).intersection(input2))) # write a single line of code

def test_common():
    assert {10} == common(range(11), range(10, 100))
    assert set() == common(range(10), range(10, 20))
    assert set(range(5, 10)) == common(range(10), range(5, 15))

notes2='''
Be sure to read and understand this before you call this unit done :-)!

When should you use which data structure is an important consideration. Read this thread completely to understand the
considerations involved: http://stackoverflow.com/questions/3489071/in-python-when-to-use-a-dictionary-list-or-set
'''

three_things_i_learnt = """
-
-
-
"""


#!/usr/bin/env python3
import json
import os
import sys
from timeit import default_timer

from dataReadWrite.NativeReadWrite import NativeReadWrite
from dataReadWrite.ReadWriteAll import ALGORITHMS
from graphHDS.prune_cluster import ALGORITHMS as SHAVING_ALGORITHMS
from graphHDS.pseudo_stability import get_algo_cluster_stabilities
from labeling.LabelingManager import LabelingManager
from util import convert_connections_to_edges
from util import shave_clustering, SHAVING_METHOD

def main():
    from argparse import ArgumentParser

    parser = ArgumentParser(description="generates labeling file")

    parser.add_argument("--staging-dir", "-s", required=True,
                        help="root staging dir of auto-hds g")
    parser.add_argument("--experiment-name", "-e", required=True,
                        help="Experiment name usually yyyy-mm-dd")
    parser.add_argument("--algo", "-a", required=True,
                        choices=ALGORITHMS.keys(),
                        help="clustering algorithm to use")
    parser.add_argument("--top-k", "-k", type=int, required=True,
                        help="top k clusters from HMETIS")
    parser.add_argument("--k-dithered", "-kd", type=int, required=True,
                        help="k dithered")
    parser.add_argument("--shaving-algo", "-sa", choices=SHAVING_ALGORITHMS,
                        help="shaving algorithm to use (1=sorted by flow to "
                             "node, 2=incremental shaving by flow to node)")
    parser.add_argument("--max-sample", "-ms", type=int, required=True,
                        help="Max sample per cluster")
    parser.add_argument("--seed", "-sd", type=int, required=True, help="seed")
    parser.add_argument("--shave-type", "-t", required=True,
                        choices={"rand_shave", "flow_shave"},
                        help="shave type to do on non autohds-g algos")
    parser.add_argument("--params-suffix", "-p", required=False, default=None, help="the params file location. the"
                                                                                " program will look for a file with"
                                                                                " judge_params.[suffix].json.")
    parser.add_argument('--debug', action="store_true", default=False,
                        help="Debug allows individual loaders and algo to sample for testing",)

    args = parser.parse_args()

    staging_dir = os.path.expanduser(args.staging_dir)
    experiment_name = args.experiment_name
    algo = args.algo
    top_k = args.top_k
    k_dithered = args.k_dithered
    shaving_algo = args.shaving_algo
    max_sample = args.max_sample
    seed = args.seed
    shave_type = args.shave_type
    params_suffix = args.params_suffix
    debug = args.debug

    shaving_method = SHAVING_METHOD.RAND if shave_type == "rand_shave" else SHAVING_METHOD.FLOW

    if shaving_method == SHAVING_METHOD.FLOW and shaving_algo is None:
        parser.error("--shaving-algo/-sa is required if shaving method is flow_shave")

    if not os.path.isdir(staging_dir):
        print("Could not find staging dir: {}".format(staging_dir))
        sys.exit(1)

    if params_suffix is None:
        judge_params_file = os.path.join(staging_dir, "judge_params.json")
    else:
        judge_params_file = os.path.join(staging_dir, "judge_params.{}.json".format(params_suffix))

    if not os.path.exists(judge_params_file):
        print("You need to run generate_genediver_judging_file.py before running this!")
        sys.exit(1)
    with open(judge_params_file, "r") as f:
        judge_params = json.load(f)
        for _var in ["k_deduped", "fraction_kept", "num_points_in_graph", "jaccard_thresh"]:
            if _var not in judge_params:
                print("Required param {} not found in judge_param file at: {}".format(_var, judge_params_file))
    k_deduped = judge_params["k_deduped"]
    fraction_kept = judge_params["fraction_kept"]
    num_points_in_graph = judge_params["num_points_in_graph"]
    jaccard_thresh = judge_params["jaccard_thresh"]
    keep_disconnected_nodes = judge_params["keep_disconnected_nodes"]

    if shaving_algo not in SHAVING_ALGORITHMS:
        raise ValueError("shaving algorithm '{}' does not exist".format(shaving_algo))
    if k_deduped < top_k:
        raise Exception("k_deduped is {} but the number of algo clusters is only {}".format(top_k, k_deduped))

    native_read_writer = NativeReadWrite(staging_dir, experiment_name)

    # ==LOAD INPUT GRAPH to get the number of points for clustering_edges==
    print("Loading graph from file...", end="", flush=True)
    start_time = default_timer()
    connection_graph, num_points, num_edges = native_read_writer.read_graph(jaccard_threshold=jaccard_thresh)

    input_graph = convert_connections_to_edges(connection_graph)
    del connection_graph

    if num_points != num_points_in_graph:
        print("Graph has changed since last genediver judge file generation.")
        print("You need to run generate_genediver_judging_file.py before running this!")
        sys.exit(1)
    print(" done. (time={:.3f} s)".format(default_timer() - start_time))
    print()

    if params_suffix is None:
        algo_staging_dir = os.path.join(staging_dir, algo)
    else:
        algo_staging_dir = os.path.join(staging_dir, algo + "." + params_suffix)

    if not os.path.isdir(algo_staging_dir):
        print("Could not find algo staging dir: {}, please run graph algo first!".format(algo_staging_dir))
        sys.exit(1)

    hdsg_to_external_id_mapping = native_read_writer.read_mapping()

    # ==READ CLUSTERING FILE OUTPUT==
    print("Loading clustering from file...", end="", flush=True)
    start_time = default_timer()
    # TODO: warn when num_algo_clusters does not match the filename
    reader = ALGORITHMS[algo](algo_staging_dir)
    cluster_points = reader.read_clustering()
    if not keep_disconnected_nodes:

        reverse_node_id_mapping = dict()
        with open(os.path.join(algo_staging_dir, "id_mapping.tsv")) as f:
            for line in map(str.strip, f):
                old_node_id, new_node_id = line.split("\t")
                reverse_node_id_mapping[int(new_node_id)] = int(old_node_id)

        for cluster_id, cluster in cluster_points.items():
            cluster_points[cluster_id] = set(map(reverse_node_id_mapping.get, cluster))

    print(" done. (time={:.3f} s)".format(default_timer() - start_time))

    # ==SHAVE CLUSTERS (if filtering didn't already shave enough)==
    print("Sample proportion: {}".format(fraction_kept))

    print("Shaving clusters using algorithm {}...".format(shaving_algo))
    start_time = default_timer()
    # shave clusters down to the same number of nodes clustered as autoHDS-G
    shaved_cluster_points = shave_clustering(
        num_points_in_graph=num_points_in_graph,
        clustering=cluster_points,
        method=shaving_method,
        fraction_kept=fraction_kept,
        seed=seed,
        input_graph=input_graph,
        shaving_algo=shaving_algo,
        debug=debug
    )

    num_shaved_cluster_points = LabelingManager.count_unique_points_in_clusters(shaved_cluster_points)
    print("num shaved cluster points: {}".format(num_shaved_cluster_points))
    print("Finished shaving clusters in (time={:.3f} s)".format(default_timer() - start_time))
    print()

    # ==FILTER CLUSTERS==
    # This will work even if top_k = num_algo_clusters
    print("Getting the top {} clusters from the output of algo...".format(top_k), end="", flush=True)
    start_time = default_timer()
    # get cluster_stabilities
    cluster_stabilities = get_algo_cluster_stabilities(
        graph=input_graph,
        cluster_points=shaved_cluster_points,
    )

    filtered_cluster_points, top_k_cluster_ids, full_dithered_cluster_ids = LabelingManager.filter_clustering(
        clustering=shaved_cluster_points,
        cluster_stabilities=cluster_stabilities,
        top_k=top_k,
        k_dithered=k_dithered
    )
    num_filtered_points = LabelingManager.count_unique_points_in_clusters(filtered_cluster_points)
    print(" done. (time={:.3f} s)".format(default_timer() - start_time))
    print("Number of nodes in judged clusters before shaving: {}".format(num_filtered_points))

    if params_suffix:
        algo += "." + params_suffix

    LabelingManager.generate_judging_files(
        clustering=shaved_cluster_points,
        filtered_clustering=filtered_cluster_points,
        top_k_cluster_ids=top_k_cluster_ids,
        k_dithered_cluster_ids=full_dithered_cluster_ids,
        algorithm_name=algo,
        max_sample=max_sample,
        seed=seed,
        cluster_stabilities=cluster_stabilities,
        staging_dir=staging_dir,
        hdsg_to_external_id_mapping=hdsg_to_external_id_mapping
    )

if __name__ == "__main__":
    main()