import numpy as np
import torch
import torch.nn as nn

class fixedDilatedConv(nn.Module):
    def __init__(self, in_channel=4, dil_start=5,
                 dil_end=24,
                 trainable=False, bulge_size=2,
                 binarize_fd=False):
        super(fixedDilatedConv, self).__init__()
        self.binarize_fd = binarize_fd
        self.in_channel = in_channel
        self.out_channel = int(
            (dil_end - dil_start + 1) * 6 * bulge_size * 3)
        self.out_channel = self.out_channel - (
            (dil_end - dil_start + 1) * 6)
        self.dil_start = dil_start
        self.dil_end = dil_end
        self.bulge_size = bulge_size
        self.kernel_size = int(
            dil_end - dil_start + 1) * 2
        self.mid = int(self.kernel_size / 2)
        weights = torch.Tensor(
            torch.zeros(
                self.out_channel,
                self.in_channel,
                self.kernel_size))
        weights = self.inject_weights(weights)
        self.weights = torch.nn.Parameter(weights)
        if not trainable:
            self.weights.requires_grad = False

    def inject_weights(self, weights):
        A, C, G, U = 0, 1, 2, 3
        weight_codes = [
            (A, U, 1), (U, A, 1.25), (C, G, 1.5),
            (G, C, 1.75), (G, U, .5), (U, G, .75)]
        if self.binarize_fd:
            weight_codes = [
                (A, U, 1), (U, A, 1), (C, G, 1),
                (G, C, 1), (G, U, 1), (U, G, 1)]
        self.weight_names = []
        kernel_idx = 0
        for nuc_tuple in weight_codes:
            nuc1, nuc2, adweight = nuc_tuple
            for i in range(self.dil_start, self.dil_end + 1):
                part_1 = round(i / 2)
                part_2 = i - part_1 + 1
                for each_bulge in range(1, self.bulge_size + 1):
                    if each_bulge == 1:
                        curbulges = [-each_bulge, 0, each_bulge]
                    else:
                        curbulges = [-each_bulge, each_bulge]
                    for bulge in curbulges:
                        idx_1 = np.array(
                            [int(self.kernel_size / 2) - part_1,
                             int(self.kernel_size / 2) + part_2])
                        if bulge > 0:
                            idx_1[idx_1 > self.mid] += bulge
                        elif bulge < 0:
                            idx_1[idx_1 < self.mid] -= bulge
                        idx_1 = idx_1[idx_1 < self.kernel_size]
                        idx_1 = idx_1[idx_1 > 0]
                        idx_0 = np.setdiff1d(
                            np.arange(self.kernel_size), idx_1)
                        weights[kernel_idx, :, idx_0] = 0
                        weights[kernel_idx, nuc1, idx_1[0]] = adweight
                        weights[kernel_idx, nuc2, idx_1[-1]] = adweight
                        adname = "{}_{} dil {} bulge {}".format(
                            nuc1, nuc2, i, bulge)
                        self.weight_names.append(adname)
                        kernel_idx += 1
        return weights[:kernel_idx, :, :]

    def forward(self, x):
        out = torch.nn.functional.conv1d(
            x, self.weights)
        if self.binarize_fd:
            out[out < torch.max(out)] = 0
            out[out > 0] = 1
        return out

class stackedDilatedConv(nn.Module):
    def __init__(self, dil_start=5, dil_end=24,
                 kernel_size=16, width=30,
                 inputsize=50):
        super(stackedDilatedConv, self).__init__()
        # self.padding = padding
        self.dil_start = dil_start
        self.dil_end = dil_end
        self.kernel_size = kernel_size
        self.width = width
        self.inputsize = inputsize
        self.padding = 0
        self.padding = self.get_lout(
            dil_end)
        if self.padding < 0:
            self.padding = int(abs(self.padding) + (inputsize * 2))
        self.pad1 = int(self.padding / 2)
        self.pad2 = self.padding - self.pad1
        conv_id = 0
        for i in range(dil_start, dil_end + 1):
            adlayer = nn.Conv1d(
                4, self.width,
                kernel_size=self.kernel_size,
                dilation=i)
            setattr(self, "Dilation.{}".format(conv_id), adlayer)
            conv_id += 1
        self.num_layers = conv_id
        self.width_out = int(self.num_layers * self.width)
        self.kernel_out = self.get_lout(self.dil_start)

    def get_lout(self, dil):
        lout = self.inputsize + 2 * self.padding -\
            dil * (self.kernel_size - 1)
        return lout

    def forward(self, x):
        x_padded = nn.functional.pad(
            x, (self.pad1, self.pad2))
        out = torch.zeros(
            x.shape[0],
            self.width_out,
            self.kernel_out).to(x.device)
        for i in range(self.num_layers):
            width_st = int(i * self.width)
            width_end = width_st + self.width
            adname = "Dilation.{}".format(i)
            temp = getattr(self, adname)(x_padded)
            dim_diff = int(
                (self.kernel_out - temp.shape[-1]) / 2)
            dim_diff_2 = int(self.kernel_out - dim_diff - temp.shape[-1])
            out[:, width_st:width_end, ] = nn.functional.pad(
                temp, (dim_diff, dim_diff_2))
        return out

class ConvArm1(nn.Module):
    def __init__(self, inputsize=50, kernels=[16, 10, 5],
                 widths=[64, 64, 64],
                 pools=[2, 2, 4], dropout=0.5,
                 disable_conv_dp=False):
        super(ConvArm1, self).__init__()
        self.inputsize = inputsize
        self.kernels = kernels
        self.pools = pools
        self.widths = widths
        self.disable_conv_dp = disable_conv_dp
        list1 = [
            nn.Conv1d(
                self.inputsize,
                widths[0], kernel_size=self.kernels[0]),
            nn.BatchNorm1d(widths[0]),
            nn.ReLU()]
        list2 = [
            nn.Conv1d(
                widths[0],
                widths[1], kernel_size=self.kernels[1]),
            nn.BatchNorm1d(widths[1]),
            nn.ReLU()]
        list3 = [
            nn.Conv1d(
                widths[1],
                widths[2], kernel_size=self.kernels[2]),
            nn.BatchNorm1d(widths[2]),
            nn.ReLU()]
        if self.disable_conv_dp:
            list1.append(nn.Dropout(p=dropout))
            list2.append(nn.Dropout(p=dropout))
            list3.append(nn.Dropout(p=dropout))
        list1.append(nn.MaxPool1d(pools[0]))
        list2.append(nn.MaxPool1d(pools[1]))
        list3.append(nn.MaxPool1d(pools[2]))
        self.a1conv1 = nn.Sequential(*list1)
        self.a1conv2 = nn.Sequential(*list2)
        self.a1conv3 = nn.Sequential(*list3)

    def forward(self, x):
        out = self.a1conv1(x)
        out = self.a1conv2(out)
        out = self.a1conv3(out)
        return out

def get_n_params(model):
    pp = 0
    pp2 = 0
    for p in list(model.parameters()):
        nn = 1
        for s in list(p.size()):
            nn = nn*s
        if p.requires_grad:
            pp += nn
        else:
            pp2 += nn
    print("{} trainable and {} non-trainable".format(pp, pp2))
    return pp, pp2

class DeepBind(nn.Module):
    def __init__(self, num_motifs=16, motif_len=24,
                 lin_dim=32, dp=0.5):
        super(DeepBind, self).__init__()
        self.channel_out = num_motifs
        self.kernel_size = motif_len
        self.lin_dim = lin_dim
        self.dp = dp
        self.conv = nn.Sequential(
            nn.Conv1d(
                4, self.channel_out,
                kernel_size=self.kernel_size),
            nn.ReLU(),
            nn.AdaptiveMaxPool1d(self.lin_dim))
        self.feedForward = nn.Sequential(
            nn.Linear(int(self.lin_dim * self.channel_out),
                      self.lin_dim),
            nn.ReLU(),
            nn.Dropout(self.dp),
            nn.Linear(self.lin_dim, 2))

        for m in self.modules():
            if isinstance(m, nn.Conv1d):
                nn.init.kaiming_normal_(
                    m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.Linear):
                nn.init.xavier_uniform_(m.weight)
                m.bias.data.fill_(0.01)

    def forward(self, x):
        x_padded = nn.functional.pad(
            x, (self.kernel_size, self.kernel_size))
        out = self.conv(x_padded)
        out = out.reshape(out.shape[0], -1)
        out = self.feedForward(out)
        return out

class PythiaModel(nn.Module):
    def __init__(self, init_channels=64, kernel_size=16,
                 inputsize=50,
                 use_fixedConv=True, dp=0.25, trainable=False,
                 dil_start=5, dil_end=24, bulge_size=2,
                 binarize_fd=False, disable_conv_dp=False,
                 num_outputs=2):
        super(PythiaModel, self).__init__()
        self.disable_conv_dp = disable_conv_dp
        self.use_fixedConv = use_fixedConv
        self.init_channels = init_channels
        self.kernel_size = kernel_size
        self.inputsize = inputsize
        self.num_outputs = num_outputs
        self.out_channel_fd = int(
            (dil_end - dil_start + 1) * 6 * bulge_size * 3)
        # Dilating layers
        if self.use_fixedConv:
            if trainable:
                self.conv1_dil = fixedDilatedConv(
                    dil_start=dil_start, dil_end=dil_end,
                    bulge_size=bulge_size, trainable=True,
                    binarize_fd=binarize_fd)
            else:
                self.conv1_dil = fixedDilatedConv(
                    dil_start=dil_start, dil_end=dil_end,
                    bulge_size=bulge_size, trainable=False,
                    binarize_fd=binarize_fd)
        else:
            out_channel = int(
                (dil_end - dil_start + 1) * 6 * bulge_size * 3)
            self.dil_dim = out_channel - ((dil_end - dil_start + 1) * 6)
            self.conv1_reg = stackedDilatedConv(
                inputsize=self.inputsize,
                dil_start=dil_start, dil_end=dil_end)
        self.pool_dil = nn.AdaptiveMaxPool1d(60)
        self.process_dil = nn.Sequential(
            nn.BatchNorm1d(60),
            nn.ReLU())
        # arm 1 dilations
        self.arm1_dil = ConvArm1(
            60,
            kernels=[16, 10, 5],
            pools=[2, 2, 2],
            dropout=dp,
            disable_conv_dp=self.disable_conv_dp)
        self.arm2_reg = ConvArm1(
            4,
            widths=[128, 64, 64],
            kernels=[12, 7, 5],
            pools=[2, 2, 2],
            dropout=dp,
            disable_conv_dp=self.disable_conv_dp)
        self.lin_dim = self.get_lindim()
        self.dense_1 = nn.Linear(
            self.lin_dim,
            64)
        self.dense_relu_1 = nn.ReLU()
        self.dense_2 = nn.Linear(
            64, 32)
        self.dense_relu_2 = nn.ReLU()
        self.dense_3 = nn.Linear(
            32, self.num_outputs)

        for m in self.modules():
            if isinstance(m, nn.Conv1d):
                nn.init.kaiming_normal_(
                    m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, (nn.BatchNorm1d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.xavier_uniform_(m.weight)
                m.bias.data.fill_(0.01)

    def get_lindim(self):
        x = torch.rand(1, 4, self.inputsize)
        out = self.forward_dil(x)
        return out.shape[1]

    def get_bn_dim(self):
        x = torch.rand(
            1, 4, self.inputsize)
        x_padded = nn.functional.pad(
            x, (25, 24))
        if self.use_fixedConv:
            out = self.conv1_dil(x_padded)
        else:
            out = self.conv1_reg(x_padded)
        return out.shape[2]

    def forward_dil(self, x):
        x_padded = nn.functional.pad(
            x, (x.shape[2], x.shape[2]))
        if self.use_fixedConv:
            out = self.conv1_dil(x_padded)
        else:
            out = self.conv1_reg(x_padded)
        out = self.pool_dil(out)
        out = out.reshape(
            out.shape[0], out.shape[2], -1)
        out = self.process_dil(out)
        out1 = self.arm1_dil(out)
        out2 = self.arm2_reg(x)
        out_merged = torch.cat(
            (out1, out2), 2)
        out = out_merged.view(
            out_merged.shape[0], -1)
        return out

    def forward(self, x):
        out = self.forward_dil(x)
        out_mfe = self.dense_1(out)
        out_mfe = self.dense_relu_1(out_mfe)
        out_mfe = self.dense_2(out_mfe)
        out_mfe = self.dense_relu_2(out_mfe)
        out_mfe = self.dense_3(out_mfe)
        return out_mfe

def conv1_dim(lin, kernel, pad=1, dil=1, stride=1):
    lout = (
        ((lin + 2 * pad - dil * (kernel - 1) - 1) / stride) + 1)
    return lout

def test_model(inputsize, use_fixedConv=True,
               kernel_size=16, trainable=False):
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print("i {} fixed {} k {} trainable {}".format(
            inputsize, use_fixedConv, kernel_size,
            trainable))
    net = PythiaModel(
        inputsize=inputsize,
        kernel_size=kernel_size,
        use_fixedConv=use_fixedConv,
        trainable=trainable)
    net.to(device)
    print(net)
    in_tensor = torch.rand(
        (10, 4, inputsize)).to(device)
    out_mfe = net(in_tensor)
    print("Input: {}\nOutput: {}".format(
            in_tensor.shape, out_mfe.shape))

def to_one_hot(a, num_classes):
    return np.squeeze(np.eye(num_classes)[a.reshape(-1)])

def get_x():
    np.random.seed(42)
    x = np.zeros((32, 4, 50))
    for i in range(32):
        temp_seq = np.random.choice(
            np.array([0, 1, 2, 3]),
            50, replace=True)
        ad_hot = to_one_hot(temp_seq, 4)
        x[i] = ad_hot.reshape(4, -1)
    return x

def test_fixed():
    np.random.seed(42)
    addict = {"A": 0, "C": 1, "G": 2, "U": 3}
    str_seq = np.random.choice(
        np.array(["A", "C", "G", "U"]),
        50, replace=True)
    str_seq[15:32] = list("ACUUAGGUAACCUAGAA")
    temp_seq = np.array(
        [addict[each] for each in str_seq])
    ad_hot = to_one_hot(temp_seq, 4)
    x = torch.from_numpy(ad_hot.T).reshape(1, 4, -1)
    x = torch.zeros(1, 4, 50)
    for i in range(50):
        x[0, :, i] = torch.from_numpy(ad_hot[i])
    self = fixedDilatedConv(dil_start=2, dil_end=21)
    dict_data = {"Seq": str_seq[:40]}
    for i in np.arange(self.weights.shape[0]):
        adname = self.weight_names[i]
        cur_weight = self.weights[i]
        res_val = torch.mul(cur_weight, x[0, :, :40])
        dict_data[adname] = sum(res_val).detach().numpy()
    import pandas as pd
    outdf = pd.DataFrame(dict_data)
    outdf.to_csv("Example_data_stemloop.tsv", sep="\t")

def test_fixed_2():
    np.random.seed(42)
    addict = {"A": 0, "C": 1, "G": 2, "U": 3}
    str_seq = "ACGCUUACAAAGUAAGCGU"
    filters = ["NNNNCNNNGNNNN",
               "NNNANNNNNUNNN",
               "NNUNNNNNNNANN",
               "NUNNNNNNNNNAN",
               "CNNNNNNNNNNNG"]
    temp_seq = np.array(
        [addict[each] for each in str_seq])
    ad_hot = to_one_hot(temp_seq, 4)
    x = torch.from_numpy(ad_hot.T).reshape(1, 4, -1)
    filters_temp = torch.zeros(5, 4, 19)
    i = 0
    for each_f in filters:
        j = 0
        for each_nuc in list(each_f):
            if each_nuc != "N":
                idx_nuc = addict[each_nuc]
                filters_temp[i, idx_nuc, j] = 1
            j += 1
        i += 1
    x_padded = nn.functional.pad(x, (9, 9))
    outval = torch.nn.functional.conv1d(x_padded.float(), filters_temp.float())
    print(outval[0].T)

if __name__ == "__main__":
    test_model(50, use_fixedConv=False, kernel_size=16)
    test_model(50, use_fixedConv=True, kernel_size=16)
    print("Script passed basic tests")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='MenuNode',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('path', models.CharField(unique=True, max_length=255)),
                ('depth', models.PositiveIntegerField()),
                ('numchild', models.PositiveIntegerField(default=0)),
                ('name', models.CharField(default=b'', max_length=63)),
                ('url', models.CharField(max_length=255, null=True, blank=True)),
            ],
            options={
                'abstract': False,
            },
        ),
    ]


"""Summary."""

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import DisallowedHost
from django.db import connection

from tenant_schemas.utils import (get_tenant_model,
                                  get_public_schema_name)
from tenant_schemas.middleware import TenantMiddleware

from panda.utils import remove_www

class SubdomainMiddleware(TenantMiddleware):
    """Custom subdamain middleware.

    This middleware should be placed at the very top of the middleware stack.
    Selects the proper database schema using the request host. Can fail in
    various ways which is better than corrupting or revealing data.

    Attributes:
        TENANT_NOT_FOUND_EXCEPTION (TYPE): Description
    """

    TENANT_NOT_FOUND_EXCEPTION = DisallowedHost

    def hostname_from_request(self, request):
        """Extract hostname from request. Used for custom requests filtering.

        By default removes the request's port and common prefixes.

        Args:
            request (TYPE): Description
        """
        return remove_www(request.get_host().split(':')[0])

    def process_request(self, request):
        """
        Summary.

        Args:
            request (TYPE): Description

        Raises:
            self.TENANT_NOT_FOUND_EXCEPTION: Description

        Returns:
            name (TYPE): Description
        """
        # Connection needs first to be at the public schema, as this is where
        # the tenant metadata is stored.
        connection.set_schema_to_public()
        hostname, www = self.hostname_from_request(request)

        tenant_model = get_tenant_model()
        try:
            request.tenant = tenant_model.objects.get(domain_url=hostname)
            connection.set_tenant(request.tenant)
        except tenant_model.DoesNotExist:
            try:
                request.tenant = tenant_model.objects.get(domain_url=www)
                connection.set_tenant(request.tenant)
            except tenant_model.DoesNotExist:
                raise self.TENANT_NOT_FOUND_EXCEPTION(
                    'No tenant for hostname "%s"' % hostname)

        # Content type can no longer be cached as public and tenant schemas
        # have different models. If someone wants to change this, the cache
        # needs to be separated between public and shared schemas. If this
        # cache isn't cleared, this can cause permission problems. For example,
        # on public, a particular model has id 14, but on the tenants it has
        # the id 15. if 14 is cached instead of 15, the permissions for the
        # wrong model will be fetched.
        ContentType.objects.clear_cache()

        # Do we have a public-specific urlconf?
        if hasattr(settings,
                   'PUBLIC_SCHEMA_URLCONF') and request.tenant\
                .schema_name == get_public_schema_name():
            request.urlconf = settings.PUBLIC_SCHEMA_URLCONF


from setuptools import setup, find_packages
from pathlib import Path
import sys

from spmimage import __version__ as version

LICENSE = 'Modified BSD'

if not (3, 5) <= sys.version_info[:2]:
    raise Exception('spm-image requires Python 3.5 or later. \n Now running on {0}'.format(sys.version))

with Path('requirements.txt').open() as f:
    INSTALL_REQUIRES = [line.strip() for line in f.readlines() if line]

setup(
    name='spm-image',
    author='Takashi Someda',
    author_email='takashi@hacarus.com',
    url='https://github.com/hacarus/spm-image',
    description='Sparse modeling and Compressive sensing in Python',
    version=version,
    packages=find_packages(),
    install_requires=INSTALL_REQUIRES,
    test_suite='tests',
    license=LICENSE
)


import puncher
from enum import Enum
import ctypes as ct
import logging
import commons
import math
import random
import pickle
import tracker
import protocol

class P2pCode(Enum):
    REQUEST = 1
    DATA = 2

CHUNK_SIZE = 1000

class P2pPacket(ct.Structure):
    _fields_ = [("action", ct.c_ushort),
                ("chunk_id", ct.c_ushort),
                ("md5_digest", ct.c_ubyte * 16),
                ("data", ct.c_ubyte * CHUNK_SIZE)]
    def set_action(self, action):
        self.action = action.value
    def get_action(self):
        return P2pCode(self.action)
    def __str__(self):
        return "%s:%s" % (self.get_action(),
            self.chunk_id)
    def set_md5(self, bytes):
        self.md5_digest = protocol.b2cb(bytes, 16)
    def get_md5(self):
        return protocol.cb2b(self.md5_digest)
    def compute_md5(self):
        self.md5_digest = commons.md5sum_bytes(self.data)
    def check_md5(self):
        return self.md5_digest == commons.md5sum_bytes(self.data)
    def set_data(self, bytes):
        self.data = protocol.b2cb(bytes, CHUNK_SIZE)
    def get_data(self):
        return protocol.cb2b(self.data)

class P2pEndpointClient(puncher.PuncherConnClient):
    def __init__(self, r, _id, server_addr=(puncher.PUB_IP, puncher.PORT)):
        puncher.PuncherConnClient.__init__(self, r, _id, server_addr)
        self.set_type(self.addr, P2pPacket)
    def incoming_endpoint(self, p):
        logging.fatal("Endpoint, processing new packet.")
        if p.get_action() != P2pCode.REQUEST:
            logging.fatal("Cannot process anything else but request messages!")

        p.set_data(self.fs.get_chunk(p.chunk_id))
        p.set_action(P2pCode.DATA)

        return p
    @classmethod
    def init_file_service(cls, fs):
        cls.fs = fs
    def request(self, chunk_id):
        p = P2pPacket()
        p.set_action(P2pCode.REQUEST)
        p.chunk_id = chunk_id
        self.send(p)
        p = self.recv()
        self.fs.save_chunk(p.data, p.chunk_id)

class P2pClient(object):
    def __init__(self, dodgy_p, server_ip, in_fd=None):
        self.dodgy_p = dodgy_p
        self.fs = FileService(dodgy_p, in_fd)
        self.t_client = tracker.TrackerClient(dodgy_p.get_addr())
        self.my_id = self.t_client.join()
        logging.info("Client joined with id=%d" % self.my_id)
        P2pEndpointClient.init_file_service(self.fs)
        # Now we start the puncher
        self.p_client = puncher.PuncherClient(self.my_id, P2pEndpointClient,
            (server_ip, puncher.PORT))
        self.p_client.join()
    def download(self, out_fd):

        if out_fd == None:
            out_fd = open(self.dodgy_p.get_name(), "wb")

        download_indices = self.fs.get_incomplete_chunks()
        while len(download_indices) > 0:

            for chunk_id in download_indices:

                logging.debug("Want chunk_id=%d" % chunk_id)
                peer_l = self.t_client.want(chunk_id)
                peer_id = random.choice(peer_l)

                # Now we connect to that peer
                peer = self.p_client.connect_cached(peer_id)
                peer.request(chunk_id)

                # Now we announce
                self.t_client.announce(chunk_id)

            download_indices = self.fs.get_incomplete_chunks()
        self.fs.mem_to_file(out_fd)
    def upload(self):
        n_chunks = self.fs.num_chunks
        logging.info("There are %d chunks in this file, announcing all of them." % n_chunks)
        # Now we announce the chunk_ids
        for chunk_id in range(1, n_chunks+1):
            self.t_client.announce(chunk_id)
    def shutdown(self):
        self.t_client.bye()
        self.p_client.bye()

class FileService(object):
    def __init__(self, dodgy_p, fd=None):
        self.file_name = dodgy_p.get_name()
        self.file_size = dodgy_p.file_size
        self.num_chunks = int(math.ceil(self.file_size/CHUNK_SIZE))
        self.total_chunks = []
        self.is_completed_chunks = []
        if fd==None:
            self.init_mem(dodgy_p)
        else:
            self.file_to_mem(dodgy_p, fd)
    def file_to_mem(self, dodgy_p, fd):
        self.file_mem = []
        for count in range(self.num_chunks):
            self.total_chunks.append(count)
            self.is_completed_chunks.append(True)
            fd.seek(count * CHUNK_SIZE)
            self.file_mem.append(fd.read(CHUNK_SIZE))
    def init_mem(self, dodgy_p):
        for count in range(1, self.num_chunks+1):
              self.total_chunks.append(count)
              self.is_completed_chunks.append(False)

        self.file_mem = [bytes()] * self.num_chunks
    def mem_to_file(self, fd):
        assert(self.get_incomplete_chunks() == [])
        logging.info("Saving to file: %s" % fd.name)
        file_mem_contiguous = bytes()
        for ea_b in self.file_mem:
            file_mem_contiguous+= ea_b
        file_mem_contiguous = file_mem_contiguous[:self.file_size]
        fd.write(file_mem_contiguous)
        fd.close()
    def save_chunk(self, data, chunk_num):
        logging.debug("Save chunk_id=%d: %s" % (chunk_num, str(data)))
        self.file_mem[chunk_num-1] = data
        self.is_completed_chunks[chunk_num-1] = True
    def get_chunk(self, chunk_id):
        data = self.file_mem[chunk_id-1]
        logging.debug("Get chunk_id=%d: %s" % (chunk_id, str(data)))
        return data
    def get_incomplete_chunks(self):
        incomplete_indices = []
        for i,is_complete in zip(self.total_chunks, self.is_completed_chunks):
            if not is_complete:
                incomplete_indices.append(i)
        return incomplete_indices
# vim: expandtab shiftwidth=4 softtabstop=4 textwidth=80:


from flask import Flask

# create a new server app
app = Flask(__name__)

# define all accessible routes
@app.route('/')
def hello_world():
    return 'Hello World!'

if __name__ == '__main__':
    addr = "localhost"         # the same as 127.0.0.1
    port = 1337
    debug = True               # activates the [1] debugger, [2] automatic reloader
    app.run(host=addr, port=port, debug=debug)


from __future__ import absolute_import
import arff # pip install liac-arff
import gzip

def arff2json(filename):

    if filename.endswith('.gz'):
        f=gzip.open(filename,'rb')
    else:
        f=open(filename,'rb')

    data = arff.load(f)

    # see https://pypi.python.org/pypi/liac-arff/1.1
    for record in data['data']:
        if not record:
            continue
        event = {}
        for i,attr in enumerate(data['attributes']):
            event.update({attr[0]:record[i]})

        yield event

def arff_iterator(filename, missing_values=['none','nan','na','n/a','','?','??','???','none or unspecified','unspecified','unknown']):

    return arff2json(filename)


from django.shortcuts import render, HttpResponse, redirect
from time import localtime, strftime

def index(request):
    return render(request, "session_words/index.html")

def add(request):
    if request.method == 'POST':
        if 'words' not in request.session:
            request.session['words']=[]
        temp_list = request.session['words']
        if 'big_font' in request.POST:
            showbig = '25pt'
        else:
            showbig = '12pt'
        temp_list.append({"word": request.POST['word'], "color": request.POST['color'], "showbig": showbig, "time": strftime("%a, %d %b %Y %H:%M:%S", localtime())})
        request.session['words'] = temp_list
        return redirect('/session_words')
    else:
        return redirect('/session_words')

def clear(request):
    del request.session['words']
    return redirect('/session_words')


import responses
from pytest import mark, param
from tests.utilities.mocks.mock_features import MOCK_FEATURE_RESPONSE
from tests.utilities.testing_constants import URL, APP_NAME, INSTANCE_ID, CUSTOM_HEADERS, CUSTOM_OPTIONS
from UnleashClient.constants import FEATURES_URL
from UnleashClient.api import get_feature_toggles

FULL_FEATURE_URL = URL + FEATURES_URL

@responses.activate
@mark.parametrize("response,status,expected", (
    param(MOCK_FEATURE_RESPONSE, 200, lambda result: result["version"] == 1, id="success"),
    param(MOCK_FEATURE_RESPONSE, 202, lambda result: not result, id="failure"),
    param({}, 500, lambda result: not result, id="failure"),
))
def test_get_feature_toggle(response, status, expected):
    responses.add(responses.GET, FULL_FEATURE_URL, json=response, status=status)

    result = get_feature_toggles(URL,
                                 APP_NAME,
                                 INSTANCE_ID,
                                 CUSTOM_HEADERS,
                                 CUSTOM_OPTIONS)

    assert len(responses.calls) == 1
    assert expected(result)


# encoding: utf-8
import pygame

from Constants import *
from groups import *
from functions import *
import Values

# pygame.init()
# size = width, height = 600, 95
# screen = pygame.display.set_mode(size)
# pygame.display.flip()

class Enemy(pygame.sprite.Sprite):
    def __init__(self, enemy, x, y, hp):
        super().__init__(enemy_group)

        self.stand_images = []
        self.run_images = []
        self.sword_images = []
        self.dance_images = []
        if enemy == GIRL:
            size = (GIRL_HEIGHT, GIRL_WIDTH)
            self.image = pygame.image.load("data/Girl/GirlPlayingWithBehemoth/0.gif")
            self.image = pygame.transform.scale(self.image, size)
            self.name = 'Girl'
            self.situation = G_PLAYING

            self.stand_images.append(all_pics(G_PLAY, G_PLAYING, size))

        self.rect = self.image.get_rect()
        self.rect.x = x
        self.rect.y = y

        self.start_pos = (x, y)
        self.state = ALIVE
        self.hp = hp
        self.direction = LEFT
        self.moving = [False, False, False, False, False]
        self.anim_count = 0
        self.speed = PLAYER_SPEED
        self.fps = FPS
        self.first_time = True
        self.gravity = GRAVITY
        self.last_sender = None

        self.mask = pygame.mask.from_surface(self.image)

        # self.add(all_sprites)

    def update(self, player, situation):
        if pygame.sprite.spritecollideany(self, player_group) and Values.GIRL:
            self.mini_game = True
            Values.MINIGAME = True

    def go_render(self, images, sender, move_flag=True, move=None):
        s = images
        flag = move_flag
        if self.last_sender != sender:
            self.anim_count = 0
        self.last_sender = sender
        return s, flag, move

    def render(self):
        """ rendering player """
        # print(1)
        images = []
        move_flag = True
        if self.situation == RUNNING:
            images = self.go_render(self.run_images, self.situation)[0]
        elif self.situation == G_PLAYING:
            images = self.go_render(self.stand_images, self.situation)[0]
        # print(images)
        if not move_flag:
            self.anim_count = self.anime(images, move_flag, self.anim_count, move)
        else:
            self.anim_count = self.anime(images, move_flag, self.anim_count)

    def anime(self, images, move_flag, anim_count, move=None):
        if anim_count > (self.situation - 1) and move_flag:
            anim_count = 0

        # print(anim_count)

        if anim_count > self.situation - 1:
            self.moving[move] = False
            self.first_time = True
            self.gravity = GRAVITY
            self.rect.y += 22
            self.rect.x += 40
        else:
            self.image = images[self.direction][anim_count]
            anim_count += 1
        return anim_count


import sys
import os.path
from data.base_dataset import BaseDataset, get_transform
from data.image_folder import make_dataset
from PIL import Image
import random
import torch

class MMUnalignedSegDataset(BaseDataset):
	def name(self):
		return 'MMUnalignedSegDataset'

	@staticmethod
	def modify_commandline_options(parser, is_train):
		return parser

	def initialize(self, opt):
		self.opt = opt
		self.root = opt.dataroot
		self.cat_num = opt.cat_num
		self.dir, self.cat_size, self.cat_path = list(), list(), list()
		self.max_instances = 20  # default: 20
		self.seg_dir = 'seg'  # default: 'seg'
		for i in range(opt.cat_num):
			self.dir.append(os.path.join(opt.dataroot, opt.phase + chr(i + 65)))
			self.cat_path.append(sorted(make_dataset(self.dir[i])))
			self.cat_size.append(len(self.cat_path[i]))
		self.transform = get_transform(opt)

	def fixed_transform(self, image, seed):
		random.seed(seed)
		return self.transform(image)

	def read_segs(self, seg_path, seed):
		segs = list()
		for i in range(self.max_instances):
			path = seg_path.replace('.png', '_{}.png'.format(i))
			if os.path.isfile(path):
				seg = Image.open(path).convert('L')
				seg = self.fixed_transform(seg, seed)
				segs.append(seg)
			else:
				segs.append(-torch.ones(segs[0].size()))
		return torch.cat(segs)

	def __getitem__(self, index):
		cat_index = list()
		for i in range(self.cat_num):
			if i == 0: # first as base index
				cat_index.append(index % self.cat_size[i])
			else:
				if self.opt.serial_batches:
					cat_index.append(index % self.cat_size[i])
				else:
					cat_index.append(random.randint(0, self.cat_size[i] - 1))
		# index_A = index % self.A_size
		# if self.opt.serial_batches:
		# 	index_B = index % self.B_size
		# else:
		# 	index_B = random.randint(0, self.B_size - 1)
		seed = random.randint(-sys.maxsize, sys.maxsize)
		imgs_list, segs_list = list(), list()
		for i in range(self.cat_num):
			cur_path = self.cat_path[i][cat_index[i]]
			cur_seg_path = cur_path.replace(chr(i + 65), chr(i + 65) + '_{}'.format(self.seg_dir))
			cur_idx = cur_path.split('/')[-1].split('.')[0]
			img = Image.open(cur_path).convert('RGB')
			imgs_list.append(self.fixed_transform(img, seed))
			segs_list.append(self.read_segs(cur_seg_path, seed))

		# A_path = self.A_paths[index_A]
		# B_path = self.B_paths[index_B]
		# A_seg_path = A_path.replace('A', 'A_{}'.format(self.seg_dir))
		# B_seg_path = B_path.replace('B', 'B_{}'.format(self.seg_dir))
		#
		# A_idx = A_path.split('/')[-1].split('.')[0]
		# B_idx = B_path.split('/')[-1].split('.')[0]

		# print('(A, B) = (%d, %d)' % (index_A, index_B))
		# seed = random.randint(-sys.maxsize, sys.maxsize)
		#
		# A = Image.open(A_path).convert('RGB')
		# B = Image.open(B_path).convert('RGB')
		# A = self.fixed_transform(A, seed)
		# B = self.fixed_transform(B, seed)
		#
		# A_segs = self.read_segs(A_seg_path, seed)
		# B_segs = self.read_segs(B_seg_path, seed)

		# if self.opt.direction == 'BtoA':
		# 	input_nc = self.opt.output_nc
		# 	output_nc = self.opt.input_nc
		# else:
		# 	input_nc = self.opt.input_nc
		# 	output_nc = self.opt.output_nc
		#
		# if input_nc == 1:  # RGB to gray
		# 	tmp = A[0, ...] * 0.299 + A[1, ...] * 0.587 + A[2, ...] * 0.114
		# 	A = tmp.unsqueeze(0)
		# if output_nc == 1:  # RGB to gray
		# 	tmp = B[0, ...] * 0.299 + B[1, ...] * 0.587 + B[2, ...] * 0.114
		# 	B = tmp.unsqueeze(0)

		# return {'A': A, 'B': B,
		# 		'A_idx': A_idx, 'B_idx': B_idx,
		# 		'A_segs': A_segs, 'B_segs': B_segs,
		# 		'A_paths': A_path, 'B_paths': B_path}
		return {'imgs_list': imgs_list, 'segs_list': segs_list}

	def __len__(self):
		return max(self.cat_size)


import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
from time import time
from tqdm import tqdm

print("---> SHORTEST PATH:")

def generar_mapa(n1, n2, p):
    a = 0
    while (np.linalg.matrix_rank(a)!=n1):
        a = n2 * np.random.rand(n1,n1)
        np.fill_diagonal(a,0)
        a = np.tril(a) + np.tril(a, -1).T
        a = np.where(a < (p * n2), a, 0)
    return a

def dijkstra_sp(net, paquetes, C_xi_shape):
    source = []
    destin = []
    paths = []
    for i in range(paquetes):
        rng = np.random.default_rng()
        s, d = rng.choice(C_xi_shape, size=2, replace=False)
        tiempo_begin = time()
        s_path = nx.dijkstra_path(net, s, d)
        tiempo_prom = (time() - tiempo_begin) / paquetes
        source.append(s)
        destin.append(d)
        paths.append(s_path)
    return source, destin, paths, tiempo_prom

desde = 10
hasta = 100
print("Total de iteraciones: {}.".format(int(hasta/desde)-1))
n1_list = np.arange(desde, hasta, desde)    # cantidad de nodos
n2 = 1                                      # distancia máxima
p = 0.5                                     # probabilidad de cerrar camino
temp = np.zeros(n1_list.shape)
paquetes = 1
iteraciones = 1

for t, n1 in tqdm(enumerate(n1_list)):
    print("\n\nCálculo de tiempo del Shortest Path para {} nodos.".format(n1))
    for it in tqdm(range(iteraciones)):
        C_xi = generar_mapa(n1, n2, p)
        net = nx.from_numpy_matrix(C_xi)
        source, destin, paths, tiempo = dijkstra_sp(net, paquetes, C_xi.shape[0])
        temp[t] += tiempo
    print("\nTotal de iteraciones: {}/{}.".format(t+1, int(hasta/desde)-1))

#print(np.round(C_xi, 3))
print("Origines: {}".format(source))
print("Destinos: {}".format(destin))
print("Caminos: {}".format(paths))
#print("Tiempo promedio en calcular tiempos: {}".format(tiempo))

plt.plot(n1_list, (temp/iteraciones))
plt.show()

#nx.draw(net, with_labels = True)           # graficar red
#plt.show()

"""
print("---> HOPFIELD NETWORK:")

n = 500
p = 10
print("NEURONAS: {}. PATRONES: {}.".format(n, p))

patrones = np.random.choice([1, -1], size=[p,n])
w = np.zeros((n, n))
for p_i in range(p):
    w += np.dot(patrones[p_i].reshape(-1,1),np.transpose(patrones[p_i].reshape(-1,1)))
np.fill_diagonal(w,0)
w /= n

j = 0
j_lim = 100
r_aux = np.copy(patrones)
while (j<j_lim):
    reconstruido = np.sign(np.dot(r_aux,w))
    reconstruido = np.where(reconstruido==0, 1, reconstruido)
    if (reconstruido==r_aux).all():
        print("Convergió con {} iteraciones. ".format(j+1), end='')
        break
    else:
        r_aux = np.copy(reconstruido)
        j += 1
if j==j_lim:
    print("No convergió. ", end='')

m = 0
for k in range(p):
    m += np.dot(patrones[k], reconstruido[k].T)
m = m / (n * p)
print("<m> = {}.".format(m))
"""


import pandas as pd
from sklearn.model_selection import train_test_split
import keras
from keras.models import Sequential
from keras.layers import Dense
from sklearn.metrics import confusion_matrix, accuracy_score #importa a matriz confusao e a matriz de acerto

previsores = pd.read_csv('entradas-breast.csv')
classe = pd.read_csv('saidas-breast.csv')

previsores_treinamento, previsores_teste, classe_treinamento, classe_teste = train_test_split(previsores, classe, test_size=0.25)

classificador = Sequential()
classificador.add(Dense(units = 16, activation = 'relu',
                        kernel_initializer = 'random_uniform', input_dim=30)) #primeira camada oculta
classificador.add(Dense(units = 16, activation = 'relu',
                        kernel_initializer = 'random_uniform')) #segunda camada oculta
classificador.add(Dense(units = 1, activation = 'sigmoid')) #camada de saida
classificador.compile('adam', loss='binary_crossentropy', metrics = ['binary_accuracy']) #optimizer padrao
            #otimizador = keras.optimizers.Adam(lr = 0.001, decay = 0.0001, clipvalue = 0.5) #mudando parametros do optimizer => abaixou muito a precisao
            #classificador.compile(otimizador, loss='binary_crossentropy', metrics = ['binary_accuracy']) #usando o optimizer configurado
classificador.fit(previsores_treinamento, classe_treinamento, batch_size = 10, epochs = 100)

pesos0 = classificador.layers[0].get_weights() #mostra os pesos que a IA encontrou, o 0 indica a primeira camada
pesos1 = classificador.layers[1].get_weights()
pesos2 = classificador.layers[2].get_weights()
print(pesos0)
print(len(pesos0))

previsoes = classificador.predict(previsores_teste)
previsoes = (previsoes > 0.5)

precisao = accuracy_score(classe_teste, previsoes) #essa matriz mostra a precisao da rede
matriz = confusion_matrix(classe_teste, previsoes) #essa matriz mostra quantos a rede errou ou acertou
resultado = classificador.evaluate(previsores_teste, classe_teste) #outra maneira de achar a precisao, mas agr usando o keras


from __future__ import absolute_import, division, print_function
__metaclass__ = type

DOCUMENTATION = '''
module: auth_approle
author: "Paul Arthur (@flowerysong)"
short_description: Authenticate to HashiCorp Vault using AppRole
description:
  - Authenticate to HashiCorp Vault using AppRole.
version_added: 0.1.0
extends_documentation_fragment:
  - flowerysong.hvault.base
options:
  role_id:
    type: str
    required: true
    description:
      - Role ID to use for authentication.
  secret_id:
    type: str
    required: false
    description:
      - SecretID to use for authentication.
  mount_point:
    type: str
    default: approle
    description:
      - Path under auth/ where the backend is mounted.
'''

EXAMPLES = '''
'''

RETURN = '''
'''

from ansible.module_utils.basic import AnsibleModule

from ..module_utils.base import (
    HVaultClient,
    hvault_argument_spec,
)
from ..module_utils.module import optspec_to_config

def main():
    argspec = hvault_argument_spec()
    argspec.pop('token')
    argspec.update(
        dict(
            mount_point=dict(
                default='approle',
            ),
        )
    )

    optspec = dict(
        secret_id=dict(
            no_log=True,
        ),
        role_id=dict(
            required=True,
            no_log=True,
        ),
    )
    argspec.update(optspec)

    module = AnsibleModule(
        supports_check_mode=False,
        argument_spec=argspec,
    )

    module.params['token'] = None
    client = HVaultClient(module.params, module)

    base_path = '/'.join([
        'auth',
        module.params['mount_point'].strip('/'),
        'login',
    ])

    config = optspec_to_config(optspec, module.params)

    result = client.post(base_path, config)['auth']

    module.exit_json(changed=True, auth=result)

if __name__ == '__main__':
    main()


# import modules
import json
import pandas as pd
import dash
import dash_html_components as html
from dash.dependencies import Input, Output, State
import dash_table

# prepare dash_table
size = 5
df = pd.DataFrame([], index=range(size))
df['num'] = range(size)
df['char'] = list('abcdefghijk')[:size]
tab = dash_table.DataTable(
    id='table',
    columns=[{"name": i, "id": i} for i in df.columns],
    data=df.to_dict('records'),
    tooltip_data=[
        {
            # (B) multiply cell value by 10 for demonstration purpose
            column: {'value': str(value*10), 'type': 'markdown'}
            for column, value in row.items()
        } for row in df.to_dict('records')
    ],
    tooltip_delay=0,
    tooltip_duration=None
)

# set layout
app = dash.Dash('SimpleExample')
app.layout = html.Div([
    tab,
    html.Div(id='click-data', style={'whiteSpace': 'pre-wrap'}),

])

# define callback
@app.callback(
    Output('click-data', 'children'),
    [Input('table', 'active_cell')],
     # (A) pass table as data input to get current value from active cell "coordinates"
    [State('table', 'data')]
)
def display_click_data(active_cell, table_data):
    if active_cell:
        cell = json.dumps(active_cell, indent=2)
        row = active_cell['row']
        col = active_cell['column_id']
        value = table_data[row][col]
        out = '%s\n%s' % (cell, value)
    else:
        out = 'no cell selected'
    return out

# run app
if __name__ == '__main__':
    app.run_server(debug=True)


import numpy
import cupy as cp
import timeit
from scipy import sparse

# x = pycuda.driver.managed_zeros(shape=4, dtype=numpy.int32, mem_flags=cuda.mem_attach_flags.GLOBAL)
# y = pycuda.driver.managed_zeros(shape=4, dtype=numpy.int32, mem_flags=cuda.mem_attach_flags.GLOBAL)
# x[0:4] = [1, 2, 3, 4]
# y[0:4] = [1, 2, 3, 4]
# l2_gpu = cupy.linalg.norm(y)
# a = cupy.multiply(x,y)
# print(l2_gpu)

# a = cupy.random.rand(44, 20)
# b = cupy.random.rand(20, 1)

pool = cp.cuda.MemoryPool(cp.cuda.malloc_managed)
cp.cuda.set_allocator(pool.malloc)

sizes = [100, 1000, 2000]
for size in sizes:
    print("---- Size: ",size, " x ", size, " ----")
    a = cp.ones((size, size), dtype=numpy.float32)
    b = cp.ones((size, size), dtype=numpy.float32)

    a[0:size] = 2

    start_time = timeit.default_timer()
    cp.dot(a, b)    # cupy.matmul(a, b, out=None)
    cp.cuda.Device(0).synchronize()
    print("duration gpu: ", timeit.default_timer() - start_time)

    start_time = timeit.default_timer()
    for i in range(10):
        a[0:size] = i
        c = cp.dot(a, b)
        cp.cuda.Device(0).synchronize()
    print("avg duration gpu: ", (timeit.default_timer() - start_time)/10.0)

    # numpy
    a_np = numpy.random.random((size, size))
    b_np = numpy.random.random((size, size))
    result_time = timeit.timeit("c_np = numpy.dot(a_np, b_np)", globals=globals(), number=10)
    print("duration numpy: ", result_time / 10.0)

    # # numpy int
    # BLAS, which is used in numpy, only works with float32 and float64 See https://www.benjaminjohnston.com.au/matmul
    a_np = numpy.random.randint(0, 10, (size, size))
    b_np = numpy.random.randint(0, 10, (size, size))
    result_time = timeit.timeit("c_np = numpy.dot(a_np, b_np)", globals=globals(), number=1)
    print("duration numpy int: ", result_time / 1.0)

    # # numpy int32
    # BLAS, which is used in numpy, only works with float32 and float64 See https://www.benjaminjohnston.com.au/matmul
    a_np = numpy.random.randint(0, 10, (size, size), dtype=numpy.int32)
    b_np = numpy.random.randint(0, 10, (size, size), dtype=numpy.int32)
    result_time = timeit.timeit("c_np = numpy.dot(a_np, b_np)", globals=globals(), number=1)
    print("duration numpy int32: ", result_time / 1.0)

    # sparse
    M = sparse.random(size, size, .30)
    result_time = timeit.timeit("c_sparse = M.dot(M)", globals=globals(), number=10)
    print("avg duration sparse: ", result_time/10.0)

    print()

# "C:\Program Files\Python36\python.exe" C:/Users/tfischle/Github/PyCuda_examples/matrix_multiplication.py
# ---- Size:  100  x  100  ----
# duration gpu:  0.5852928000000001
# avg duration gpu:  0.00014606999999999815
# duration numpy:  0.0003451800000000005
# duration numpy int:  0.0009719999999999729
# duration numpy int32:  0.0009557000000000038
# avg duration sparse:  0.0009325600000000045
#
# ---- Size:  1000  x  1000  ----
# duration gpu:  0.0015964000000000533
# avg duration gpu:  0.0031100099999999964
# duration numpy:  0.013901710000000001
# duration numpy int:  0.6493503
# duration numpy int32:  0.633626
# avg duration sparse:  0.2823481
#
# ---- Size:  2000  x  2000  ----
# duration gpu:  0.006359999999999921
# avg duration gpu:  0.00414393999999998
# duration numpy:  0.07829159999999993
# duration numpy int:  27.2362269               # BLAS, which is used in numpy, only works with float32 and float64 See https://www.benjaminjohnston.com.au/matmul
# duration numpy int32:  27.470195999999994
# avg duration sparse:  2.313961


# Importações necessárias para o projeto
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb
import pandas as pd
from pathlib import Path

# Classe criada para executar as funções
class RegressaoSimples:
    def __init__(self, x, y, idade):
        self.x = x
        self.y = y
        self.idade = idade

        # Tratamento para caso seja inserido um numero já utilizado
        if not np.isin(self.idade, self.x):
            self.idade = idade
        else:
            raise 'Idade já está em uso favor escolher outra'

    def correlacao(self):
        raiz_quadrada = np.sqrt(np.var(self.x) * np.var(self.y))
        covarianca = np.cov(self.x, self.y, bias=True)[0][1]
        return covarianca / raiz_quadrada

    def inclinacao(self):
        desvio_x = np.std(self.y)
        desvio_y = np.std(self.x)
        return self.correlacao() * (desvio_x / desvio_y)

    def interceptacao(self):
        media_y = np.mean(self.y)
        media_x = np.mean(self.x)
        return media_y - self.inclinacao() * media_x

    def previsao(self):

        # Foi utilizado o round para arredondar o resultado
        return round(self.interceptacao() + self.inclinacao() * self.idade)

    # novo dataset criado para receber os dados da previsão, gerando um documento no formato csv localmente
    def novo_dataset(self):
        add_idade = np.sort(np.append(self.idade, self.x))
        add_valor = np.sort(np.append(self.previsao(), self.y))
        dados = pd.DataFrame({'Idade': add_idade,
                              'Valor': add_valor})
        filepath = Path('documento_atualizado.csv')
        filepath.parent.mkdir(parents=True, exist_ok=True)
        return dados.to_csv(filepath, index=False)

    # Função para o plot do gráfico
    def grafico(self):
        novo_dataset = pd.read_csv('documento_atualizado.csv')
        plt.figure(figsize=(8, 6))
        sb.regplot(x='Idade',
                   y='Valor',
                   ci=None,
                   data=novo_dataset,
                   color='red',
                   scatter_kws={'color': 'blue'},
                   line_kws={'color': 'black'})
        plt.title('Regressão Linear Simples', fontsize=10)
        plt.xlabel('Idade', fontsize=10)
        plt.ylabel('Valor', fontsize=10)
        plt.legend(['Valores', 'Linha de melhor ajuste'],
                   loc='lower right', bbox_to_anchor=(1.13, -0.14), fontsize=8)
        return plt.show()

# Para executar aqui mesmo e não ser usado como módulo.
if __name__ == '__main__':

    # Digite o caminho em que se encontra o arquivo csv, lembre-se de colocar o caminho e o nome do arquivo também.
    dataset = pd.read_csv(input('Coloque o caminho: '))

    '''
    O nome dentro do dataset refere-se as colunas, no caso a primeira coluna seria Idade e a segunda Valor do documento
    Convenio.csv. Se quiser testar previsões com outras variaveis, deve-se trocar o nome das colunas dentro do Convenio.csv
    OU pode-se usar outro arquivo trocando o nomedos respectivos dataset abaixo, pois se não for feito o ocorrerá erro.
    '''
    posicao_x = dataset['Idade']
    x_array = np.array(posicao_x)
    posicao_y = dataset['Valor']
    y_array = np.array(posicao_y)

    idade_previsao = input('Coloque a idade que deseja prever o valor: ')

    # Tratamento básico caso seja digitado letra ao invés de numero
    if idade_previsao.isnumeric():
        idade_previsao = int(idade_previsao)
    else:
        raise 'Coloque somente numeros!'

    regressao = RegressaoSimples(x_array, y_array, idade_previsao)
    regressao.novo_dataset()

    # impressao direto no console
    print(f'Com a idade de {idade_previsao} anos, a previsão de fica em R${regressao.previsao()}')

    regressao.grafico()


'''
Finds and returns pieces of raw COVID-19 press-release that look
like tables. There are 3 such tables in typical release:

- COVID-19 cases in Russian regions
- new deaths
- new recovered per

Скрипт находит таблицы новых случев и смертей в пресс-релизе оперштаба
и возвращает обработанные скриптом regioncounter.py абзацы с отсортированными
регионами.

Первая таблица - с новыми случаями - должна быть обрезана (убираем все регионы,
где менее 100 случаев за последние сутки. Вторую таблицу даем полностью.
'''

import re

import ifaxbotcovid.parser.regioncounter as r
from ifaxbotcovid.parser.regexp import table_detect

# задаем regex'ы для парсинга двух таблиц в релизе

REGEXES = table_detect.detector_dict

# здесь будут храниться таблицы
TABLES = dict(newcases='NO_TABLE', newdeaths='NO_TABLE')

def get_key(dct, v):
    '''
    Возвращает key в dict, зная value
    '''
    return [key for key, value in dct.items() if value == v][0]

def choose_value(value):
    '''
    Сокращает tuple вида ('', 'цифра', '') до 'цифра'
    '''
    if type(value) == tuple:
        if len(value) > 1:
            for item in value:
                if item == '':
                    None
                else:
                    return item
            return ''
        else:
            return value[0]
    else:
        return value

def find_tables(raw_txt):
    global TABLES
    for regex in REGEXES.values():
        try:
            table_name = get_key(REGEXES, regex)
        except Exception:
            print('Не удалось определить формат таблицы')
            return
        try:
            table_regex = re.compile(regex, re.DOTALL)
            raw_table = table_regex.findall(raw_txt)[0]
            raw_table = choose_value(raw_table)
            TABLES[table_name] = raw_table
        except Exception:
            print('Не удалось найти таблицу {}'.format(table_name))
    return

def get_prepared(ready_table, table_type, short=100):
    if table_type == 'ready_cases':
        splitted_ready_cases = ready_table.split(',')
        if 'Москв' in splitted_ready_cases[0]:
            first_reg = splitted_ready_cases[1]
            splitted_ready_cases[1] = first_reg.replace(
                '-', 'новых случаев COVID-19 зафиксировано', 1
            )
            joined_ready_cases = ','.join(
                splitted_ready_cases[1:]
            )  # убираем Москву
        else:
            joined_ready_cases = ','.join(
                splitted_ready_cases
            )  # Москва не в начале, не трогаем
        cases_final = ''.join((
            'По информации оперативного штаба, еще',
            joined_ready_cases,
            'В других регионах России суточный прирост не превышает ',
            str(short),
            '.'
        ))
        return cases_final
    if table_type == 'ready_deaths':
        splitted_deaths = ready_table.split(',')
        first_reg = splitted_deaths[0]
        if ' - ' in first_reg:
            splitted_deaths[0] = first_reg.replace(
                '-', 'пациентов скончалось за сутки', 1
            )
        joint_deaths = ','.join(splitted_deaths)
        cases_final = 'Согласно данным оперштаба о смертности, ' + joint_deaths
        return cases_final

def process_tables(short=100):
    rc_ready_cases = r.RegionCounter(
        TABLES['newcases'],
        table_type='new_cases',
        short=short
    )
    rc_ready_deaths = r.RegionCounter(
        TABLES['newdeaths'],
        table_type='dead'
    )
    ready_cases, log1 = rc_ready_cases(), rc_ready_cases.log
    ready_deaths, log2 = rc_ready_deaths(), rc_ready_deaths.log
    log = log1 + log2
    ready_cases = get_prepared(ready_cases, 'ready_cases', short=short)
    ready_deaths = get_prepared(ready_deaths, 'ready_deaths', short=short)
    return ready_cases, ready_deaths, log

def tables(rawtext, short=100):
    # для вызова за пределами скрипта
    try:
        find_tables(rawtext)
        ready_cases, ready_deaths, log = process_tables(short=short)
        return ready_cases, ready_deaths, log
    except Exception:
        print(
            'Не удалось обработать и добавить таблицы новых случаев и умерших'
        )

# for testing usage only
if __name__ == '__main__':
    import pyperclip
    if pyperclip.paste is not None:
        rawtext = pyperclip.paste()
        ready_cases, ready_deaths, log = tables(rawtext, short=150)
        print(ready_cases)
        print(ready_deaths)
        print(log)
    else:
        print('Нет текста в буфере обмена')


import scanpy as sc
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd

import os
from datetime import date

def get_counts(data):
    # number of counts per spot
    data.obs['n_counts'] = data.X.sum(1)
    # number of genes per spot
    data.obs['n_genes'] = (data.X > 0).sum(1)

    data.obs['mean_counts'] = data.obs['n_counts'] / data.obs['n_genes']

    return data

def get_mean_std(data):
    np.std(data.X, axis=1)

def add_disease(data, disease_labels):
    data = data.copy()
    data.obs['disease'] = 'Unknown'
    data.obs['disease'] = data.obs['disease'].astype('<U16')
    for spot_label in disease_labels:
        m_spots = data.obs[spot_label] == 1
        data.obs['disease'][m_spots] = spot_label
    data.obs['disease'] = data.obs['disease'].astype('string').astype('category')

    return data

def plot(data, save_folder, key):
    df = data.obs

    # Plots
    # get mean counts of spots
    _, ax = plt.subplots(figsize=(12, 6))
    ax.set_yscale("log")
    ax = sns.boxplot(x="sample", y="mean_counts", data=df, ax=ax, fliersize=1, linewidth=1, hue="disease")
    ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
    ax.set_xlabel("Samples")
    ax.set_ylabel("Mean UMI-counts per spot")
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    # Put the legend out of the figure
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.tight_layout()
    plt.savefig(os.path.join(save_folder, "{}_{}".format(key, "Mean_UMI_counts_per_spot.png")), dpi=300)
    plt.close()

    # mean counts per sample
    _, ax = plt.subplots(figsize=(12, 6))
    ax.set_yscale("log")
    ax = sns.boxplot(x="sample", y="n_counts", data=df, ax=ax, fliersize=1, linewidth=1, hue="disease")
    ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
    ax.set_xlabel("Samples")
    ax.set_ylabel("Total UMI-counts per spot")
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    # Put the legend out of the figure
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.tight_layout()
    plt.savefig(os.path.join(save_folder, "{}_{}".format(key, "Total_UMI_counts_per_spot.png")), dpi=300)
    plt.close()

def main(save_folder):
    adata_wo_4_7 = sc.read(
        '/Users/christina.hillig/Documents/Projects/annData_objects/spatial/2021-07-29/Spatial Transcriptomics_unpp.h5')

    print("Dimension of samples without 4 and 7", adata_wo_4_7.shape)

    # rename LICHEN to LP
    adata_wo_4_7.obs = adata_wo_4_7.obs.rename(columns={"LICHEN": "LP"})
    adata_wo_4_7 = get_counts(data=adata_wo_4_7)
    adata_wo_4_7 = add_disease(data=adata_wo_4_7, disease_labels=['PSO', 'AE', 'LP', 'PRP'])

    # per spot
    df_spot = pd.DataFrame(columns=['sample', 'mean', 'var', 'std'])
    df_spot.loc[:, 'sample'] = adata_wo_4_7.obs['sample'].values
    nan_data = adata_wo_4_7.X.copy()
    # 1. Set zero counts to nan
    nan_data[nan_data == 0] = np.nan
    # 2. Calculate mean, std, and var
    df_spot.loc[:, 'mean'] = np.nanmean(nan_data, axis=1)
    df_spot.loc[:, 'var'] = np.nanvar(nan_data, axis=1)
    df_spot.loc[:, 'std'] = np.nanstd(nan_data, axis=1)

    del nan_data

    # save in .csv file
    df_spot.to_csv(os.path.join(save_folder, "Statistics_per_spot_wo_4_7.csv"))
    df_spot.to_excel(os.path.join(save_folder, "Statistics_per_spot_wo_4_7.xlsx"))

    # TODO per sample
    df_sample = pd.DataFrame(
        columns=['sample', 'min_count_spot', 'max_count_spot', 'mean_count_spot', 'var_count_spot', 'std_count_spot',
                 'median_counts_spot', 'total_genes_detected', 'median_genes_spot'])
    # 1. Sum up counts per sample
    for ind, sample_id in enumerate(adata_wo_4_7.obs['sample'].cat.categories):
        df_sample.loc[ind, 'sample'] = sample_id

        mask_sample = adata_wo_4_7.obs['sample'] == sample_id

        matrix_sample = adata_wo_4_7.X[mask_sample, :].copy()

        sum_over_spots = np.sum(matrix_sample, axis=0)
        sum_over_spots[sum_over_spots > 0] = 1
        df_sample.loc[ind, 'total_genes_detected'] = sum_over_spots.sum()

        #  Calculate mean, std, and var
        # 1. Get counts per sample
        counts_per_sample = np.sum(matrix_sample, axis=0)

        # 2. Set zero counts to nan
        counts_per_sample[counts_per_sample == 0] = np.nan

        # Min and Max count pro sample
        df_sample.loc[ind, 'min_count_spot'] = np.nanmin(counts_per_sample)
        df_sample.loc[ind, 'max_count_spot'] = np.nanmax(counts_per_sample)

        # 3. Calculate mean, std, and var
        mean_count_gene_sample = np.nanmean(counts_per_sample)
        df_sample.loc[ind, 'mean_count_spot'] = mean_count_gene_sample

        var_count_gene_sample = np.nanvar(counts_per_sample)
        df_sample.loc[ind, 'var_count_spot'] = var_count_gene_sample

        std_count_gene_sample = np.nanstd(counts_per_sample)
        df_sample.loc[ind, 'std_count_spot'] = std_count_gene_sample

        # Median UMI Counts per Spot
        sum_over_genes = np.sum(matrix_sample, axis=1)
        median_counts_sample = np.median(sum_over_genes)
        df_sample.loc[ind, 'median_counts_spot'] = median_counts_sample

        # Median Genes per Spot
        matrix_sample[matrix_sample > 0] = 1
        num_genes_spot_sample = np.sum(matrix_sample, axis=1, dtype='int64')
        median_genes_sample = np.median(num_genes_spot_sample)
        df_sample.loc[ind, 'median_genes_spot'] = median_genes_sample

        np.sum(matrix_sample, axis=1, dtype='int64')

    df_sample.to_csv(os.path.join(save_folder, "Statistics_per_sample_wo_4_7.csv"))
    df_sample.to_excel(os.path.join(save_folder, "Statistics_per_sample_wo_4_7.xlsx"))

    plot(data=adata_wo_4_7, save_folder=save_folder, key='wo_4_7')

    del adata_wo_4_7

    adata = sc.read(
        '/Users/christina.hillig/Documents/Projects/annData_objects/spatial/2021-03-19/adata_P15509_P16357_unpp.h5')

    print("Dimension of all samples", adata.shape)

    adata.obs = adata.obs.rename(columns={"LICHEN": "LP"})
    adata = add_disease(data=adata, disease_labels=['PSO', 'AE', 'LP', 'PRP'])
    adata = get_counts(data=adata)
    plot(data=adata, save_folder=save_folder, key='all')

if __name__ == '__main__':
    today = date.today()
    path = os.path.join("..", "..")
    # create saving folder in current project path
    savepath = os.path.join(path, "output", "UMI-counts_per_spots", str(today))
    os.makedirs(savepath, exist_ok=True)

    main(save_folder=savepath)


#
# Author: Alex R. Carney
# Date: 4/9/2020
# Summary: Creates a scatterplot comparing the College Enrollment Rate
# to the School Instruction Score of Chicago Public Schools in the
# 2011-2012 school year. This scatterplot shows a moderately strong,
# positive, linear association between higher Instruction Scores and College
# Enrollment Rates.
#
# Dataset:
# https://data.cityofchicago.org/Education/Chicago-Public-Schools-Progress-Report-Cards-2011-/9xs2-f89t
#

import numpy as np
import csv
import pandas as pd
import matplotlib.pyplot as plt

#Func Name: csv_to_df
#Purpose: Converts .csv dataset to a pandas dataframe
#Params: f_name: the file name of the dataset
#Returns: df: pandas dataframe conversion of the .csv dataset
def csv_to_df(f_name):
    df = pd.read_csv(f_name,index_col =0)
    return df

#Func Name: clean_and_type_data
#Purpose: Replaces NDA values with np.nan. Ignores rows
# in desired columns with NaN values. Converts col values to float.
#Params: df: dataframe, col1 is first desired col, col2 is second desired col
#Returns: df: original data frame but with desired columns properly typed
# and rows with NaN values removed.
def clean_and_type_data(df, col1, col2):
    df =df.replace('NDA', np.nan)
    df.dropna(subset = [col1], inplace = True)
    df.dropna(subset = [col2], inplace = True)
    df[col1] = df[col1].astype(float)
    df[col2] = df[col2].astype(float)
    return df

#Func Name: plot_data
#Purpose: Plots the data to a scatterplot.
#Params: df: dataframe, col1 is first desired col, col2 is second desired col
#Returns:
def plot_data(df, col1, col2):
    df.plot(kind='scatter',x=col1,y=col2, color='red')
    plt.xlabel("Instruction Score", fontsize = 14)
    plt.ylabel("College Enrollment Rate (%)", fontsize = 14)
    plt.title("College Enrollment Rate Vs. School Instruction Score", fontsize = 14)

#Func Name: plot_regression
#Purpose: Plots a  linear regression line to the scatterplot.
#Params: df: dataframe, col1 is first desired col, col2 is second desired col
#Returns:
def plot_regression(df,col1,col2):
    x = np.array(df[col1])
    y = np.array(df[col2])
    m,b = np.polyfit(x,y,1)

    plt.plot(x, m*x + b)

def main():
    f_name = 'Chicago_Public_Schools_-_Progress_Report_Cards__2011-2012_.csv'
    col1 = 'Instruction Score'
    col2 = 'College Enrollment Rate %'
    df = csv_to_df(f_name)
    df = clean_and_type_data(df,col1,col2)
    plot_data(df, col1, col2)
    plot_regression(df, col1,col2)
    plt.show()

main()


import requests
from ckanapi import RemoteCKAN

#secret = API Key
secret = ''
demo = RemoteCKAN('http://beta.stlouisdata.org', apikey=secret)

#Create as many resources as desired
def update_resources():
    filename = " "

    #While the name input is NOT blank
    while filename != "":
        #Name displayed in big letters and on resource list
        filename = input("Enter the name of a resource to to add to a dataset.  Leave blank to finish with resources\n"
                         "'exit' to exit\n")
        if filename == "":
            break
        elif filename == "exit":
            exit()

        #Choose where the resource goes
        dataset = input("Enter the name of the existing dataset to add the file to\n")

        #Enter a description for the resource
        desc = input("Enter a description for the resource\n")

        #URL of the initial values to input into the file
        res_url = input("Enter the url for the source file\n")

        #Create resource with all of those variables
        package = demo.action.resource_create(package_id=dataset,
                                              name=filename,
                                              url=res_url,
                                              notes=desc)

        #Verify in the logs that it worked by seeing what values are in what places
        print(package)

    run()

def update_datasets():
    filename = " "

    #While the name input is NOT blank
    while filename != "":
        #Name displayed in big letters and URL
        filename = input("Enter the name of a dataset to create.  \n"
                         "Will be converted to Lowercase.  Leave blank to finish with datasets\n"
                         "'exit' to exit\n").lower()
        if filename == "":
            break
        elif filename == "exit":
            exit()

        #Enter a description for the dataset
        desc = input("Enter a description for the dataset\n")

        #Create resource with all of those variables
        package = demo.action.package_create(name=filename,
                                              notes=desc)

        #Verify in the logs that it worked by seeing what values are in what places
        print(package)
    run()

def run():
    print("OpenSTL-DataExchange")

    usr_in = input("Do you wish to create 'datasets' or 'resources'?  Leave blank to exit\n")

    #Decide if user wants to make new Datasets or Resources
    if usr_in == "datasets":
        update_datasets()
    elif usr_in == "resources":
        update_resources()
    update_resources()

    ua = 'ckanapiexample/1.0 (+http://example.com/my/website)'

    demo = RemoteCKAN('http://beta.stlouisdata.org', apikey=secret)

    groups = demo.action.package_list(id='test_data')
    print(groups)

    # try:
    #     pkg = demo.action.package_create(name='ya_set', title='not going to work')
    # except:
    #     print("create_error")

    pkg = demo.action.package_show(id='ya_set')

    groups = demo.action.package_list(id='test_data')
    print(groups)
    print("\nPackage_Create {}".format(pkg))

    pkg['title'] = "WORKING!!"
    #pkg['tags'] = ['a_tag', 'b_tag']
    pkg['notes'] = 'Just playing around really...' # this is the description field

    #print("\nPackage_Create {}".format(pkg))

    #pkg = demo.action.package_update(**pkg)
    #print("\nPackage_Create {}".format(pkg))

    npkg = demo.action.package_show(id='test_data')
    print("\nPackage_Create {}".format(npkg))

    # pkg['resources'] = npkg['resources']
    # pkg = demo.action.package_update(**pkg)

''' ToDo:
- parse_sources: 'sources dictionary" -> {source_name: source_url, data_set, group, org}
- for_each source in sources -> create/update/patch

'''

    #pkg = demo.action.resource_update(package_id='my-dataset3', name='test-data', id='ec2191c7-fd13-44c6-a51f-71c1081330df', url='https://raw.githubusercontent.com/OpenDataSTL/pyckan/Clarys-Branch/testdata.csv', upload='true')

    #res = demo.action.resource_show(package_id='my-dataset3', name='test-data',  id='ec2191c7-fd13-44c6-a51f-71c1081330df')

if __name__ == "__main__":
    run()


# Import modules
import tensorflow as tf
import numpy as np
import argparse
import os
import time
import h5py
import sys
import matplotlib.pyplot as plt
import horovod.tensorflow as hvd

# Prevent python from saving out .pyc files
sys.dont_write_bytecode = True

# Add models and tasks to path
sys.path.insert(0, './models')
sys.path.insert(0, './tasks')

# Import methods
from util import log
from input_ops import *

def check_path(path):
    if not os.path.exists(path):
        os.mkdir(path)

class Trainer(object):

    def __init__(self,config):

        # Initialize Horovod
        hvd.init()

        # Pin GPU to be used to process local rank (one GPU per process)
        self.session_config = tf.ConfigProto()
        self.session_config.gpu_options.visible_device_list = str(hvd.local_rank())
        self.session_config.gpu_options.allow_growth = True
        log.info('# of GPUs = ' + str(hvd.size()))

        # Config
        self.config = config
        self.config.N_GPUs = hvd.size()
        self.model_name = config.model_name

        # Get dataset path
        self.dataset_path = './datasets'

        # Get variable names to create input operations
        model_script_name = "model_" + self.model_name
        create_input_var_names = getattr(__import__(model_script_name, fromlist=["create_input_var_names"]), "create_input_var_names")
        input_var_names = create_input_var_names()

        ## Training set
        # Read file
        train_set_name = self.config.dset_names[self.config.train_set_ind]
        train_filename = 'analogy_' + train_set_name + '.hy'
        train_file = os.path.join(self.dataset_path, train_filename)
        log.info("Reading %s...", train_file)
        train_data = h5py.File(train_file, 'r')
        # Create dataset structure
        train_ids = np.arange(len(train_data))
        np.random.shuffle(train_ids)
        train_set = Dataset(train_ids, train_data, 'analogy_' + train_set_name, input_var_names)
        # Create input operation
        self.train_batch_ops = create_input_ops(train_set, int(self.config.batch_size), scope = 'analogy_' + train_set_name + '_inputs')

        # Import model class
        Model = getattr(__import__(model_script_name, fromlist=["Model"]), "Model")
        log.info("Using Model class: %s", self.model_name)

        # Create 'global_step' variable
        log.info("Initializing global step and optimizer...")
        self.global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False, dtype=tf.int64)

        # Build model
        log.info("Building training model...")
        with tf.variable_scope(tf.get_variable_scope()):
            self.model = Model(self.config, self.train_batch_ops)
            tf.get_variable_scope().reuse_variables()
            self.train_loss = self.model.train_loss

        # Create optimizer
        log.info("Create optimizer...")
        self.opt = tf.train.AdamOptimizer(learning_rate=self.config.learning_rate)
        self.opt = hvd.DistributedOptimizer(self.opt)

        # Set up training operation (to apply gradients)
        log.info('Setting up training operation...')
        self.apply_gradient_op = self.opt.minimize(self.train_loss, global_step=self.global_step)

        # Model name (for output directory)
        self.model_name = self.config.model_name + '_trained_on_' + train_set_name + '_run' + self.config.run

        # Create directory for training output
        check_path('./train_dir')
        self.train_dir = './train_dir/%s' % (self.model_name) + '/'
        check_path(self.train_dir)
        log.info("Train Dir: %s", self.train_dir)

        # Summaries and saving output
        self.summary_op = tf.summary.merge_all()
        self.saver = tf.train.Saver(max_to_keep=len(self.config.save_checkpoint_steps))

        # Set up session
        log.info("Setting up session...")
        init = tf.global_variables_initializer()
        bcast = hvd.broadcast_global_variables(0)
        self.session = tf.Session(config=self.session_config)
        self.session.run(init)
        self.session.run(bcast)
        tf.train.start_queue_runners(sess=self.session)

    def train(self):

        # Use CPU to coordinate training
        with tf.Graph().as_default(), tf.device('/cpu:0'):

            log.info("Training begins...")

            # Setup file for recording timecourses
            timecourse_dir = './timecourses'
            check_path(timecourse_dir)
            timecourse_filename = self.model_name + '.txt'
            timecourse_filename_path = os.path.join(timecourse_dir, timecourse_filename)
            timecourse_file_ID = open(timecourse_filename_path, 'w')
            header_line = 'step loss'
            for key in self.model.all_out.keys():
                header_line = header_line + ' ' + key
            header_line = header_line + '\n'
            timecourse_file_ID.write(header_line)

            train_steps = int(self.config.train_steps)
            save_checkpoint_steps = np.round(np.array(self.config.save_checkpoint_steps)).astype(np.int)
            for s in range(train_steps):

                ## Training
                # Single training step
                summary, step_time, train_loss, train_all_out = self.run_single_step(s)
                # Periodic output
                if s % 10 == 0:
                    # Output performance
                    self.log_summary(s+1, step_time, train_loss, train_all_out)
                    # Write performance to timecourse file
                    timecourse_line = str(int(s+1)) + ' ' + str(train_loss)
                    for key in self.model.all_out.keys():
                        timecourse_line += ' ' + str(float("{0:.2f}".format(train_all_out[key])))
                    timecourse_line += '\n'
                    timecourse_file_ID.write(timecourse_line)

                # Save parameters
                if np.any(s+1 == save_checkpoint_steps):
                    log.info('Saving parameters...')
                    self.saver.save(self.session, self.train_dir, global_step=self.global_step)

            # Close timecourse file
            timecourse_file_ID.close()

    def run_single_step(self, step):

        # Start time
        start_time = time.time()

        # Read in batch and run step
        _, summary, train_loss, train_all_out = self.session.run(
            [self.apply_gradient_op, self.summary_op,
            self.model.train_loss, self.model.all_out])

        # End time
        end_time = time.time()
        step_time = end_time - start_time

        return summary, step_time, train_loss, train_all_out

    def log_summary(self, step, step_time, train_loss, train_out):

        if step_time == 0:
            step_time = 0.001

        # Step number
        if step == 'final':
            text = " [train step final] "
        else:
            text = " [train step {step:4d}] ".format(step = step)

        # Training loss
        text = text + "{var_name:s}: {var_val:.4f} ".format(var_name = 'train_loss', var_val = train_loss)

        # Training outputs
        for key, val in train_out.items():
            text = text + "{var_name:s}: {var_val:.4f} ".format(var_name = key, var_val = val)

        # Step time
        text = text + "({sec_per_batch:.3f} sec/batch)".format(sec_per_batch = step_time)

        # Print
        log.info(text)

def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('--model_name', type=str, default='context_norm')
    parser.add_argument('--dset_names', nargs="*", type=str, default=['train', 'test1', 'test2', 'test3', 'test4', 'test5'])
    parser.add_argument('--train_set_ind', type=int, default=0)
    parser.add_argument('--run', type=str, default='1')
    parser.add_argument('--batch_size', type=int, default=32)
    parser.add_argument('--train_steps', type=int, default=10000)
    parser.add_argument('--save_checkpoint_steps', nargs="*", type=int, default=[100,200,500,1000,2000,10000])
    parser.add_argument('--learning_rate', type=float, default=5e-4)

    config = parser.parse_args()

    # Construct trainer
    trainer = Trainer(config)

    # Train
    trainer.train()

if __name__ == '__main__':
    main()


import asyncio

class AsyncIOIntegration:
	def __init__(self):
		self.loop = asyncio.get_running_loop()
		self._timer_ids = dict()

		# Not using range() as it requires defining a maximum
		def ids():
			x = 0
			while True:
				yield x
				x += 1
		self._timerid = iter(ids())

	def _get_timerid(self):
		return next(self._timerid)

	def _timer_cb(self, timer_id, callback, oneshot, _interval):
		now = self.loop.time()
		callback(timer_id)
		if not oneshot:
			self.create_timer(
				_interval, callback,
				oneshot=oneshot,
				_timer_id=timer_id, _start_at=now
			)
		else:
			del self._timer_ids[timer_id]

	def create_timer(self,
		interval_in_s, callback, oneshot=False,
		_timer_id=None, _start_at=None
	):
		# We need a timer_id in _timer_cb() callback.
		# As AsyncIO doesn't actually send it as argument
		# we have to roll our own id.
		if _timer_id is None:
			_timer_id = self._get_timerid()
		if _start_at is None:
			_start_at = self.loop.time()
		timer_obj = self.loop.call_at(
			_start_at + interval_in_s,
			self._timer_cb,
			_timer_id, callback, oneshot, interval_in_s
		)
		self._timer_ids[_timer_id] = timer_obj
		return _timer_id

	def remove_timer(self, timer_id):
		obj = self._timer_ids[timer_id]
		del self._timer_ids[timer_id]
		obj.cancel()

	def create_reader(self, fd, callback):
		self.loop.add_reader(fd, callback, fd)

	def remove_reader(self, fd):
		self.loop.remove_reader(fd)


# Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
/install endpoint for tecs API
"""
import os
import copy
import subprocess
import time
import commands

import traceback
import webob.exc
from oslo_config import cfg
from oslo_log import log as logging
from webob.exc import HTTPBadRequest
from webob.exc import HTTPForbidden
from webob.exc import HTTPServerError

import threading
from threading import Thread

from daisy import i18n
from daisy import notifier

from daisy.api import policy
import daisy.api.v1

from daisy.common import exception
import daisy.registry.client.v1.api as registry
from daisy.api.backends.tecs import config
from daisy.api.backends import driver
from daisy.api.network_api import network as neutron
from ironicclient import client as ironic_client
import daisy.api.backends.os as os_handle
import daisy.api.backends.common as daisy_cmn
import daisy.api.backends.tecs.common as tecs_cmn
import daisy.api.backends.tecs.install as instl
import daisy.api.backends.tecs.uninstall as unstl
import daisy.api.backends.tecs.upgrade as upgrd

try:
    import simplejson as json
except ImportError:
    import json

LOG = logging.getLogger(__name__)
_ = i18n._
_LE = i18n._LE
_LI = i18n._LI
_LW = i18n._LW

tecs_state = tecs_cmn.TECS_STATE

class API(driver.DeploymentDriver):
    """
    The hosts API is a RESTful web service for host data. The API
    is as follows::

        GET  /hosts -- Returns a set of brief metadata about hosts
        GET  /hosts/detail -- Returns a set of detailed metadata about
                              hosts
        HEAD /hosts/<ID> -- Return metadata about an host with id <ID>
        GET  /hosts/<ID> -- Return host data for host with id <ID>
        POST /hosts -- Store host data and return metadata about the
                        newly-stored host
        PUT  /hosts/<ID> -- Update host metadata and/or upload host
                            data for a previously-reserved host
        DELETE /hosts/<ID> -- Delete the host with id <ID>
    """

    def __init__(self):
        super(API, self).__init__()
        return

    def install(self, req, cluster_id):
        """
        Install TECS to a cluster.

        param req: The WSGI/Webob Request object
        cluster_id:cluster id
        """

        tecs_install_task = instl.TECSInstallTask(req, cluster_id)
        tecs_install_task.start()

    def _get_roles_and_hosts_ip_list(self, req, cluster_id):
        host_ha_list = set()
        host_ip_list = set()
        role_id_list = set()
        hosts_id_list = []
        hosts_list = []

        roles = daisy_cmn.get_cluster_roles_detail(req,cluster_id)
        cluster_networks = daisy_cmn.get_cluster_networks_detail(req, cluster_id)
        for role in roles:
            if role['deployment_backend'] != daisy_cmn.tecs_backend_name:
                continue
            role_hosts = daisy_cmn.get_hosts_of_role(req, role['id'])
            if role_hosts:
                for role_host in role_hosts:
                    host = daisy_cmn.get_host_detail(req, role_host['host_id'])
                    host_ip = tecs_cmn.get_mngt_network_ip(host, cluster_networks)
                    if role['name'] == "CONTROLLER_HA":
                        host_ha_list.add(host_ip)
                    host_ip_list.add(host_ip)
                    hosts_id_list.append({host['id']:host_ip})
                role_id_list.add(role['id'])
        for host in hosts_id_list:
            if host not in hosts_list:
                hosts_list.append(host)
        return (role_id_list, host_ip_list, host_ha_list, hosts_list)

    def _query_progress(self, req, cluster_id, action=""):
        nodes_list = []
        roles = daisy_cmn.get_roles_detail(req)
        (role_id_list,host_ip_list,host_ha_list,hosts_list) = self._get_roles_and_hosts_ip_list(req, cluster_id)
        for host in hosts_list:
            node = {}
            host_id = host.keys()[0]
            host = daisy_cmn.get_host_detail(req, host_id)
            node['id'] = host['id']
            node['name'] = host['name']

            if 0 == cmp("upgrade", action):
                node['os-progress'] = host['os_progress']
                node['os-status'] = host['os_status']
                node['os-messages'] = host['messages']

            if host['status'] == "with-role":
                host_roles = [ role for role in roles if role['name'] in host['role'] and role['cluster_id'] == cluster_id]
                if host_roles:
                    node['role-status'] = host_roles[0]['status']
                    node['role-progress'] = str(host_roles[0]['progress'])
                    # node['role-message'] = host_roles[0]['messages']
            nodes_list.append(node)
        return {'tecs_nodes': nodes_list}

    def uninstall(self, req, cluster_id):
        """
        Uninstall TECS to a cluster.

        :param req: The WSGI/Webob Request object

        :raises HTTPBadRequest if x-install-cluster is missing
        """

        (role_id_list, host_ip_list,host_ha_list, hosts_list) = self._get_roles_and_hosts_ip_list(req, cluster_id)
        if role_id_list:
            if not host_ip_list:
                msg = _("there is no host in cluster %s") % cluster_id
                raise exception.ThreadBinException(msg)

            unstl.update_progress_to_db(req, role_id_list, tecs_state['UNINSTALLING'], hosts_list, 0.0)
            uninstall_progress_percentage = round(1*1.0/len(host_ip_list), 2)*100

            threads = []
            for host_ip in host_ip_list:
                t = threading.Thread(target=unstl.thread_bin,args=(req,host_ip,role_id_list,uninstall_progress_percentage))
                t.setDaemon(True)
                t.start()
                threads.append(t)
            LOG.info(_("uninstall threads have started, please waiting...."))

            try:
                for t in threads:
                    t.join()
            except:
                LOG.warn(_("Join uninstall thread %s failed!" % t))
            else:
                uninstall_failed_flag = False
                for role_id in role_id_list:
                    role = daisy_cmn.get_role_detail(req, role_id)
                    # if progress is 100, it means some thing is not ready when uninstalling
                    # here can't use status ==uninstalling or progress==0
                    if role['progress'] == 100:
                        unstl.update_progress_to_db(req, role_id_list, tecs_state['UNINSTALL_FAILED'], hosts_list)
                        uninstall_failed_flag = True
                        break
                    if role['status'] == tecs_state['UNINSTALL_FAILED']:
                        uninstall_failed_flag = True
                        break
                if not uninstall_failed_flag:
                    LOG.info(_("all uninstall threads have done, set all roles status to 'init'!"))
                    unstl.update_progress_to_db(req, role_id_list, tecs_state['INIT'], hosts_list)

        try:
            (status, output) = commands.getstatusoutput('rpm -e --nodeps openstack-packstack\
                        openstack-packstack-puppet openstack-puppet-modules puppet')
        except exception.Invalid as e:
            raise HTTPBadRequest(explanation=e.msg, request=req)

    def uninstall_progress(self, req, cluster_id):
        return self._query_progress(req, cluster_id, "uninstall")

    def upgrade(self, req, cluster_id):
        """
        update TECS to a cluster.

        :param req: The WSGI/Webob Request object

        :raises HTTPBadRequest if x-install-cluster is missing
        """
        (role_id_list,host_ip_list,host_ha_list,hosts_list) = self._get_roles_and_hosts_ip_list(req, cluster_id)
        if role_id_list:
            if not host_ip_list:
                msg = _("there is no host in cluster %s") % cluster_id
                raise exception.ThreadBinException(msg)
            unreached_hosts = daisy_cmn.check_ping_hosts(host_ip_list, 1)
            if unreached_hosts:
                self.message = "hosts %s ping failed" % unreached_hosts
                raise exception.NotFound(message=self.message)

            daisy_cmn.subprocess_call('rm -rf /root/.ssh/known_hosts')

            if os_handle.check_tfg_exist():
                os_handle.upgrade_os(req, hosts_list)
                unreached_hosts = daisy_cmn.check_ping_hosts(host_ip_list, 30)
                if unreached_hosts:
                    self.message = "hosts %s ping failed after tfg upgrade" % unreached_hosts
                    raise exception.NotFound(message=self.message)
            upgrd.update_progress_to_db(req, role_id_list, tecs_state['UPDATING'], 0.0)
            update_progress_percentage = round(1*1.0/len(host_ip_list), 2)*100
            threads = []
            LOG.info(_("begin to update TECS controller nodes, please waiting...."))
            for host_ip in host_ha_list:
                LOG.info(_("update TECS controller node %s..." % host_ip))
                upgrd.thread_bin(req,host_ip,role_id_list,update_progress_percentage)
            LOG.info(_("begin to update TECS other nodes, please waiting...."))
            for host_ip in (host_ip_list - host_ha_list):
                t = threading.Thread(target=upgrd.thread_bin,args=(req,host_ip,role_id_list,update_progress_percentage))
                t.setDaemon(True)
                t.start()
                threads.append(t)
            try:
                for t in threads:
                    t.join()
            except:
                LOG.warn(_("Join update thread %s failed!" % t))
            else:
                update_failed_flag = False
                for role_id in role_id_list:
                    role = registry.get_role_metadata(req.context, role_id)
                    # if progress is 0, it means some thing is not ready when updating
                    # here can't use status ==updating or progress==100
                    if role['progress'] == 0:
                        upgrd.update_progress_to_db(req, role_id_list, tecs_state['UPDATE_FAILED'])
                        update_failed_flag = True
                        break
                    if role['status'] == tecs_state['UPDATE_FAILED']:
                        update_failed_flag = True
                        break
                if not update_failed_flag:
                    LOG.info(_("all update threads have done, set all roles status to 'active'!"))
                    upgrd.update_progress_to_db(req, role_id_list, tecs_state['ACTIVE'])

    def upgrade_progress(self, req, cluster_id):
        return self._query_progress(req, cluster_id, "upgrade")

    def export_db(self, req, cluster_id):
        """
        Export daisy db data to tecs.conf and HA.conf.

        :param req: The WSGI/Webob Request object

        :raises HTTPBadRequest if x-install-cluster is missing
        """

        (tecs_config, mgnt_ip_list) = instl.get_cluster_tecs_config(req, cluster_id)

        config_files = {'tecs_conf':'','ha_conf':''}
        tecs_install_path = "/home/tecs_install"
        tecs_config_file = ''
        if tecs_config:
            cluster_conf_path = tecs_install_path + "/" + cluster_id
            create_cluster_conf_path = "rm -rf %s; mkdir %s" % (cluster_conf_path, cluster_conf_path)
            daisy_cmn.subprocess_call(create_cluster_conf_path)
            config.update_tecs_conf(tecs_config, cluster_conf_path)

            get_tecs_conf = "ls %s|grep tecs.conf" % cluster_conf_path
            obj = subprocess.Popen(get_tecs_conf, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stdoutput, erroutput) = obj.communicate()
            tecs_conf_file = ""
            if stdoutput:
                tecs_conf_file = stdoutput.split('\n')[0]
                config_files['tecs_conf'] = cluster_conf_path + "/" + tecs_conf_file

            get_ha_conf_cmd = "ls %s|grep HA_1.conf" % cluster_conf_path
            obj = subprocess.Popen(get_ha_conf_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stdoutput, erroutput) = obj.communicate()
            ha_conf_file = ""
            if stdoutput:
                ha_conf_file = stdoutput.split('\n')[0]
                config_files['ha_conf'] = cluster_conf_path + "/" + ha_conf_file
        return config_files


import torch
from torch.nn import Module, Linear, Conv2d
from torch.nn.utils import spectral_norm
from torch.nn.functional import tanh, interpolate
from .spade_resblk import SPADEResBlk

class SPADEGenerator(Module):
    def __init__(self, args):
        super().__init__()
        self.linear = Linear(args.gen_input_size, args.gen_hidden_size)
        self.spade_resblk1 = SPADEResBlk(args, 1024)
        self.spade_resblk2 = SPADEResBlk(args, 1024)
        self.spade_resblk3 = SPADEResBlk(args, 1024)
        self.spade_resblk4 = SPADEResBlk(args, 512)
        self.spade_resblk5 = SPADEResBlk(args, 256)
        self.spade_resblk6 = SPADEResBlk(args, 128)
        self.spade_resblk7 = SPADEResBlk(args, 64)
        self.conv = spectral_norm(Conv2d(64, 3, kernel_size=(3,3), padding=1))

    def forward(self, x, seg):
        b, c, h, w = seg.size()
        x = self.linear(x)
        x = x.view(b, -1, 4, 4)

        x = interpolate(self.spade_resblk1(x, seg), size=(2*h, 2*w), mode='nearest')
        x = interpolate(self.spade_resblk2(x, seg), size=(4*h, 4*w), mode='nearest')
        x = interpolate(self.spade_resblk3(x, seg), size=(8*h, 8*w), mode='nearest')
        x = interpolate(self.spade_resblk4(x, seg), size=(16*h, 16*w), mode='nearest')
        x = interpolate(self.spade_resblk5(x, seg), size=(32*h, 32*w), mode='nearest')
        x = interpolate(self.spade_resblk6(x, seg), size=(64*h, 64*w), mode='nearest')
        x = interpolate(self.spade_resblk7(x, seg), size=(128*h, 128*w), mode='nearest')

        x = tanh(self.conv(x))

        return x