import pandas as pd
import matplotlib.pyplot as plt
from wordcloud import WordCloud
wordclouddata=pd.read_csv('wordclouddata.csv')
d = {}
for a, x in wordclouddata.values:
    d[a] = x
wordcloud = WordCloud(width = 800, height = 800,
                background_color ='white',
                min_font_size = 10).generate_from_frequencies(frequencies=d)
# plot the WordCloud image
plt.figure(figsize = (8, 8), facecolor = None)
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad = 0)
plt.title('All Count features Word Cloud')
plt.show()
wordcloud.to_file('all.png')

PerClusterWordCounts=pd.read_csv('PerClusterWordCounts.csv')
PerClusterWordCounts['Word']=PerClusterWordCounts['Unnamed: 0']
PerClusterWordCounts=PerClusterWordCounts[PerClusterWordCounts['Word']!='label']
for i in range(7):
    t='cluster'+str(i)
    data=PerClusterWordCounts[['Word',t]]
    d={}
    for r in data.itertuples():
        d[r[1]]= r[2]
    wordcloud = WordCloud(width = 800, height = 800,
                    background_color ='white',
                    min_font_size = 10).generate_from_frequencies(frequencies=d)
    # plot the WordCloud image
    plt.figure(figsize = (8, 8), facecolor = None)
    plt.imshow(wordcloud)
    plt.axis("off")
    plt.tight_layout(pad = 0)
    plt.title('cluster: '+str(i)+' counts word cloud')
    plt.show()
    wordcloud.to_file('cluster'+str(i)+'.png')


import importlib
import json
import logging
import os
from sys import stdout
from urlparse import urljoin

from cartoview.app_manager.forms import AppInstanceEditForm
from django.conf import settings
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django.core.files import File
from django.core.urlresolvers import reverse
from django.db.models import F, Max, Min
from django.forms.util import ErrorList
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import render, render_to_response
from django.template import RequestContext, loader
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from geonode.base.forms import CategoryForm
from geonode.base.models import TopicCategory
from geonode.people.forms import ProfileForm
from geonode.security.views import _perms_info_json
from geonode.utils import build_social_links, resolve_object
from guardian.shortcuts import get_perms
from models import App, AppInstance

from .installer import AppInstaller

formatter = logging.Formatter(
    '[%(asctime)s] p%(process)s  { %(name)s %(pathname)s:%(lineno)d} \
                              %(levelname)s - %(message)s', '%m-%d %H:%M:%S')
logger = logging.getLogger(__name__)
handler = logging.StreamHandler(stdout)
handler.setFormatter(formatter)
logger.addHandler(handler)
_PERMISSION_MSG_DELETE = _("You are not permitted to delete this document")
_PERMISSION_MSG_GENERIC = _("You do not have permissions for this document.")
_PERMISSION_MSG_MODIFY = _("You are not permitted to modify this document")
_PERMISSION_MSG_METADATA = _(
    "You are not permitted to modify this document's metadata")
_PERMISSION_MSG_VIEW = _("You are not permitted to view this document")

current_folder, filename = os.path.split(os.path.abspath(__file__))
temp_dir = os.path.join(current_folder, 'temp')
if not os.path.exists(temp_dir):
    os.makedirs(temp_dir)

# TODO: Review the following function Legacy?!!!

def save_thumbnail(filename, image):
    thumb_folder = 'thumbs'
    upload_path = os.path.join(settings.MEDIA_ROOT, thumb_folder)
    if not os.path.exists(upload_path):
        os.makedirs(upload_path)

    with open(os.path.join(upload_path, filename), 'wb') as f:
        thumbnail = File(f)
        thumbnail.write(image)

    url_path = os.path.join(
        settings.MEDIA_URL,
        thumb_folder,
        filename).replace(
        '\\',
        '/')
    url = urljoin(settings.SITEURL, url_path)

    return url

def get_apps_names():
    return [
        n for n in os.listdir(settings.APPS_DIR)
        if os.path.isdir(os.path.join(settings.APPS_DIR, n))
    ]

def installed_apps():
    from .models import App
    apps = App.objects.filter().order_by('order')
    return apps

@staff_member_required
def manage_apps(request):
    apps = App.objects.all()
    context = {
        'apps': apps,
        'site_apps': get_apps_names(),
    }
    logger.warning(context['site_apps'])
    return render(request, 'app_manager/manage.html', context)

def index(request):
    Apps = installed_apps()
    for app in Apps:
        module = importlib.import_module(app.name)
        if hasattr(module, 'urls_dict'):
            urls_dict = getattr(module, 'urls_dict')
            if 'admin' in urls_dict.keys():
                app.admin_urls = urls_dict['admin']
            else:
                app.admin_urls = None
            if 'logged_in' in urls_dict.keys():
                app.logged_in_urls = urls_dict['logged_in']
            else:
                app.logged_in_urls = None
            if 'anonymous' in urls_dict.keys():
                app.anonymous_urls = urls_dict['anonymous']
            else:
                app.anonymous_urls = None
        else:
            app.admin_urls = app.logged_in_urls = app.anonymous_urls = None

    context = {'Apps': Apps}
    return render(request, 'app_manager/apps.html', context)

@staff_member_required
@require_POST
def install_app(request, store_id, app_name, version):
    response_data = {'success': False, 'messages': []}
    try:
        installer = AppInstaller(app_name, store_id, version)
        # installedApps = installer.install()
        installer.install()
        response_data["success"] = True
    except Exception as ex:
        logger.error(ex.message)
        response_data["messages"].append({"type": "error", "msg": ex.message})

    return HttpResponse(
        json.dumps(response_data), content_type="application/json")

@staff_member_required
@require_POST
def uninstall_app(request, store_id, app_name):
    response_data = {"success": False, "errors": []}
    try:
        installer = AppInstaller(app_name, store_id)
        installer.uninstall()
        response_data["success"] = True
    except Exception as ex:
        logger.error(ex.message)
        response_data["errors"].append(ex.message)
    return HttpResponse(
        json.dumps(response_data), content_type="application/json")

@login_required
def move_up(request, app_id):
    app = App.objects.get(id=app_id)
    prev_app = App.objects.get(order=App.objects.filter(
        order__lt=app.order).aggregate(Max('order'))['order__max'])
    order = app.order
    app.order = prev_app.order
    prev_app.order = order
    app.save()
    prev_app.save()
    return HttpResponse(
        json.dumps({
            "success": True
        }), content_type="application/json")

@login_required
def move_down(request, app_id):
    app = App.objects.get(id=app_id)
    next_app = App.objects.get(order=App.objects.filter(
        order__gt=app.order).aggregate(Min('order'))['order__min'])
    order = app.order
    app.order = next_app.order
    next_app.order = order
    app.save()
    next_app.save()
    return HttpResponse(
        json.dumps({
            "success": True
        }), content_type="application/json")

def save_app_orders(request):
    if request.method == 'POST':
        apps_list = request.POST.get('apps', None)

        if apps_list:
            try:
                apps = json.loads(apps_list)
                menu_apps = apps['menu_apps']
                non_menu_apps = apps['non_menu_apps']
                for idx, val in enumerate(menu_apps):
                    app = App.objects.get(id=int(val['id']))
                    app.order = idx
                    app.in_menu = True
                    app.save()

                for idx, val in enumerate(non_menu_apps):
                    app = App.objects.get(id=int(val['id']))
                    app.order = idx + len(menu_apps)
                    app.in_menu = False
                    app.save()
                ajax_vars = {'success': True}
            except BaseException:
                ajax_vars = {'success': False}
                return HttpResponse(
                    json.dumps(ajax_vars), content_type="application/json")

    return HttpResponse(json.dumps(ajax_vars), content_type="application/json")

def _resolve_appinstance(request,
                         appinstanceid,
                         permission='base.change_resourcebase',
                         msg=_PERMISSION_MSG_GENERIC,
                         **kwargs):
    """
    Resolve the document by the provided primary key
    and check the optional permission.
    """
    return resolve_object(
        request,
        AppInstance, {'pk': appinstanceid},
        permission=permission,
        permission_msg=msg,
        **kwargs)

def appinstance_detail(request, appinstanceid):
    """
    The view that show details of each document
    """
    appinstance = None
    try:
        appinstance = _resolve_appinstance(request, appinstanceid,
                                           'base.view_resourcebase',
                                           _PERMISSION_MSG_VIEW)

    except Http404:
        return HttpResponse(
            loader.render_to_string('404.html', RequestContext(request, {})),
            status=404)

    except PermissionDenied:
        return HttpResponse(
            loader.render_to_string(
                '401.html',
                RequestContext(request, {
                    'error_message':
                    _("You are not allowed to view this document.")
                })),
            status=403)

    if appinstance is None:
        return HttpResponse(
            'An unknown error has occured.', mimetype="text/plain", status=401)

    else:
        if request.user != appinstance.owner and not request.user.is_superuser:
            AppInstance.objects.filter(id=appinstance.id).update(
                popular_count=F('popular_count') + 1)
        set_thumbnail_link = appinstance.link_set.filter(
            link_type='appinstance_thumbnail')
        context_dict = {
            'perms_list':
                get_perms(request.user, appinstance.get_self_resource()),
            'permissions_json':
                _perms_info_json(appinstance),
            'resource':
                appinstance,
            # 'appinstance_links': appinstance_links,
            'set_thumbnail_link':
                set_thumbnail_link
            # 'imgtypes': IMGTYPES,
            # 'related': related
        }

        if settings.SOCIAL_ORIGINS:
            context_dict["social_links"] = build_social_links(
                request, appinstance)

        if getattr(settings, 'EXIF_ENABLED', False):
            try:
                from geonode.contrib.exif.utils import exif_extract_dict
                exif = exif_extract_dict(appinstance)
                if exif:
                    context_dict['exif_data'] = exif
            except BaseException as e:
                logger.error(e.message + "Exif extraction failed.")

        return render_to_response("app_manager/appinstance_detail.html",
                                  RequestContext(request, context_dict))

@login_required
def appinstance_metadata(request,
                         appinstanceid,
                         template='app_manager/appinstance_metadata.html'):
    appinstance = None
    try:
        appinstance = _resolve_appinstance(request, appinstanceid,
                                           'base.change_resourcebase_metadata',
                                           _PERMISSION_MSG_METADATA)

    except Http404:
        return HttpResponse(
            loader.render_to_string('404.html', RequestContext(request, {})),
            status=404)

    except PermissionDenied:
        return HttpResponse(
            loader.render_to_string(
                '401.html',
                RequestContext(request, {
                    'error_message':
                    _("You are not allowed to edit this instance.")
                })),
            status=403)

    if appinstance is None:
        return HttpResponse(
            'An unknown error has occured.', mimetype="text/plain", status=401)

    else:
        poc = appinstance.poc
        metadata_author = appinstance.metadata_author
        topic_category = appinstance.category

        if request.method == "POST":
            appinstance_form = AppInstanceEditForm(
                request.POST, instance=appinstance, prefix="resource")
            category_form = CategoryForm(
                request.POST,
                prefix="category_choice_field",
                initial=int(request.POST["category_choice_field"])
                if "category_choice_field" in request.POST else None)
        else:
            appinstance_form = AppInstanceEditForm(
                instance=appinstance, prefix="resource")
            category_form = CategoryForm(
                prefix="category_choice_field",
                initial=topic_category.id if topic_category else None)

        if request.method == "POST" and appinstance_form.is_valid(
        ) and category_form.is_valid():
            new_poc = appinstance_form.cleaned_data['poc']
            new_author = appinstance_form.cleaned_data['metadata_author']
            new_keywords = appinstance_form.cleaned_data['keywords']
            new_category = TopicCategory.objects.get(
                id=category_form.cleaned_data['category_choice_field'])

            if new_poc is None:
                if poc is None:
                    poc_form = ProfileForm(
                        request.POST, prefix="poc", instance=poc)
                else:
                    poc_form = ProfileForm(request.POST, prefix="poc")
                if poc_form.is_valid():
                    if len(poc_form.cleaned_data['profile']) == 0:
                        # FIXME use form.add_error in django > 1.7
                        errors = poc_form._errors.setdefault(
                            'profile', ErrorList())
                        errors.append(
                            _('You must set a point of contact for this\
                             resource'))
                        poc = None
                if poc_form.has_changed and poc_form.is_valid():
                    new_poc = poc_form.save()

            if new_author is None:
                if metadata_author is None:
                    author_form = ProfileForm(
                        request.POST, prefix="author",
                        instance=metadata_author)
                else:
                    author_form = ProfileForm(request.POST, prefix="author")
                if author_form.is_valid():
                    if len(author_form.cleaned_data['profile']) == 0:
                        # FIXME use form.add_error in django > 1.7
                        errors = author_form._errors.setdefault(
                            'profile', ErrorList())
                        errors.append(
                            _('You must set an author for this resource'))
                        metadata_author = None
                if author_form.has_changed and author_form.is_valid():
                    new_author = author_form.save()

            if new_poc is not None and new_author is not None:
                the_appinstance = appinstance_form.save()
                the_appinstance.poc = new_poc
                the_appinstance.metadata_author = new_author
                the_appinstance.keywords.add(*new_keywords)
                AppInstance.objects.filter(id=the_appinstance.id).update(
                    category=new_category)

                return HttpResponseRedirect(
                    reverse('appinstance_detail', args=(appinstance.id,)))
            else:
                the_appinstance = appinstance_form.save()
                if new_poc is None:
                    the_appinstance.poc = appinstance.owner
                if new_author is None:
                    the_appinstance.metadata_author = appinstance.owner
                the_appinstance.keywords.add(*new_keywords)
                AppInstance.objects.filter(id=the_appinstance.id).update(
                    category=new_category)

                return HttpResponseRedirect(
                    reverse('appinstance_detail', args=(appinstance.id,)))

        if poc is not None:
            appinstance_form.fields['poc'].initial = poc.id
            poc_form = ProfileForm(prefix="poc")
            poc_form.hidden = True
        else:
            poc_form = ProfileForm(prefix="poc")
            poc_form.hidden = True
        if metadata_author is not None:
            appinstance_form.fields[
                'metadata_author'].initial = metadata_author.id
            author_form = ProfileForm(prefix="author")
            author_form.hidden = True
        else:
            author_form = ProfileForm(prefix="author")
            author_form.hidden = True

        return render_to_response(template,
                                  RequestContext(request, {
                                      "appinstance": appinstance,
                                      "appinstance_form": appinstance_form,
                                      "poc_form": poc_form,
                                      "author_form": author_form,
                                      "category_form": category_form,
                                  }))

def appinstance_remove(request, appinstanceid):
    try:
        appinstance = _resolve_appinstance(request, appinstanceid,
                                           'base.delete_resourcebase',
                                           _PERMISSION_MSG_DELETE)
        appinstance.delete()
        return HttpResponseRedirect(reverse('appinstance_browse'))
    except PermissionDenied:
        return HttpResponse(
            'You are not allowed to delete this Instance',
            mimetype="text/plain",
            status=401)


#saludos a la Clase
import json

m = []
lado = 9
pausa = False
guardar = False
cargar = False

def setup():
    global m, lado
    fullScreen()
    background(0)#______________________Color de fondo
    m = generarm(height/lado, width/lado)
    dibujarm(m)

def draw():
    global m, pausa, lado, cargar, guardar
    #fill(0,0,0,100)
    #rect(0,0, width, height)
    #noFill()
    background(0)
    dibujarm(m)
    #m = siguientem(m)
    if not pausa:
        m = siguientem(m)
    if guardar:
        archivo = open("guardar.txt", "w")
        json.dump(m, archivo)
        archivo.close()
        guardar = False
    if cargar:
        archivo = open("guardar.txt", "r")
        m = json.load(archivo)
        archivo.close()
        dibujarm(m)
        cargar = False
    if mousePressed:
        f = mouseY/lado
        c = mouseX/lado
        if f >= len(m):
            f %= len(m)
        if c >= len(m[0]):
            c %= len(m[0])
        m[f][c] = 1

def generarm(fil, col):
    return [[int(random(3)) for _ in range(col)] for _ in range(fil)]

def dibujarm(m):
    global lado
    for fil in range(len(m)):
        for col in range(len(m[0])):
            if m[fil][col] == 1:
                fill(0,40 , 255)
                rect(col * lado, fil * lado, lado, lado)
                #ellipse(col*lado, fil*lado, lado, lado)
                noFill()
            elif m[fil][col] == 2:
                fill(152,159, 193)
                rect(col * lado, fil * lado, lado, lado)
                #ellipse(col*lado, fil*lado, lado, lado)
                noFill()

def vecinosv(m, fil, col):
    total = 0
    topefil = len(m)
    topecol = len(m[0])
    for _ in range(fil-1, fil+2):
        for j in range(col-1, col+2):
            if _ >= topefil:
                _ %= topefil
            if j >= topecol:
                j %= topecol
            if _ != fil or j != col:
                if m[_][j] == 1:
                    total += m[_][j]
                    #print(_, j)
                    #total += m[_ % len(m)][j% len(m[0])]
                else:
                    pass
            else:
                pass
    #print(total)
    return total

def siguientem(m):
    nueva = [[0 for _ in range(len(m[0]))] for _ in range(len(m))]
    for fil in range(len(m)):
        for col in range(len(m[0])):
            v = vecinosv(m, fil, col)
            if m[fil][col] == 0 and v == 2:
                nueva[fil][col] = 1
            elif m[fil][col] == 1:
                nueva[fil][col] = 2
            elif m[fil][col] == 2:
                nueva[fil][col] = 0
    return nueva

def keyPressed():
    global pausa, guardar, cargar
    if key == " ":
        pausa = not pausa
    if key == "g" or key == "G":
        guardar = True
    if key == "c" or key == "C":
        cargar = True


import argparse
import os, sys, stat

def change_permissions(files_to_convert):
    """Set permissions for files using os.chmod

    Parameters
    ----------
    files_to_convert : list like
        List of files to change permissions of.
    """
    for f in files_to_convert:
        os.chmod(f, stat.S_IREAD|
                    stat.S_IWRITE|
                    stat.S_IEXEC|
                    stat.S_IRGRP|
                    stat.S_IWGRP|
                    stat.S_IXGRP)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--files', default=None, dest='files',
                        nargs='+', help='Path of files')
    args = parser.parse_args()

    change_permissions(args.files)

if __name__ == '__main__':
    main()


#! /usr/bin/env python
# -*- coding: utf-8 -*-

import sys
from PySide6.QtWidgets import QApplication, QMainWindow, QDialog
from PySide6 import QtCore
# импортируем связанный py файл с нашим ui файлом
from design_main import Ui_MainWindow

class MainWindow(QMainWindow):
    def __init__(self):
		# вызовем метод родительского класса
        super(MainWindow, self).__init__()

        # создадим объект
        self.ui = Ui_MainWindow()
        # инициализируем нашу форму
        self.ui.setupUi(self)

        # Соединим сигналы со слотами
        self.ui.pushButton.clicked.connect(self.pushed_button)

    # функция при нажатии на кнопку
    def pushed_button(self):
        print('ok')

if __name__ == "__main__":
    # Создадим Qt Application
    app = QApplication(sys.argv)
    # Создадим и покажем главное окно
    window = MainWindow()
    # Показываем окно
    window.show()
    # Запустим приложение
    sys.exit(app.exec_())


import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models
from torch.nn import init
from torchvision.models.resnet import resnet18

def init_weights(net, init_type='normal', gain=0.02):
    def init_func(m):
        classname = m.__class__.__name__
        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
            if init_type == 'normal':
                init.normal_(m.weight.data, 0.0, gain)
            elif init_type == 'xavier':
                init.xavier_normal(m.weight.data, gain=gain)
            elif init_type == 'kaiming':
                init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
            elif init_type == 'orthogonal':
                init.orthogonal(m.weight.data, gain=gain)
            else:
                raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
            if hasattr(m, 'bias') and m.bias is not None:
                init.constant_(m.bias.data, 0.0)
        elif classname.find('BatchNorm2d') != -1:
            init.normal_(m.weight.data, 1.0, gain)
            init.constant_(m.bias.data, 0.0)

    print('initialize network with %s' % init_type)
    net.apply(init_func)

def fix_grad(net):
    print(net.__class__.__name__)
    def fix_func(m):
        classname = m.__class__.__name__
        if classname.find('Conv') != -1 or classname.find('BatchNorm2d') != -1:
            m.weight.requires_grad = False
            if m.bias is not None:
                m.bias.requires_grad = False

    net.apply(fix_func)

def unfix_grad(net):
    def fix_func(m):
        classname = m.__class__.__name__
        if classname.find('Conv') != -1 or classname.find('BatchNorm2d') != -1 or classname.find('Linear') != -1:
            m.weight.requires_grad = True
            if m.bias is not None:
                m.bias.requires_grad = True

    net.apply(fix_func)

def define_TrecgNet(cfg, use_noise=None, using_semantic_branch=None, device=None):
    if using_semantic_branch is None:
        using_semantic_branch = cfg.USING_SEMANTIC_BRANCH

    if 'resnet' in cfg.ARCH:
        model = TRecgNet_Upsample_Resiual(cfg, encoder=cfg.ARCH, using_semantic_branch=using_semantic_branch,
                            use_noise=use_noise, device=device)

    return model

def print_network(net):
    num_params = 0
    for param in net.parameters():
        num_params += param.numel()
    # print(net)
    print('Total number of parameters: %d' % num_params)

def conv3x3(in_planes, out_planes, stride=1):

    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=1, bias=False)

def conv_norm_relu(dim_in, dim_out, kernel_size=3, stride=1, padding=1, norm=nn.BatchNorm2d,
                   use_leakyRelu=False, use_bias=False, is_Sequential=True):
    if use_leakyRelu:
        act = nn.LeakyReLU(0.2, True)
    else:
        act = nn.ReLU(True)

    if is_Sequential:
        result = nn.Sequential(
            nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, stride=stride,
                      padding=padding, bias=use_bias),
            norm(dim_out, affine=True),
            act
        )
        return result
    return [nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, stride=stride,
                      padding=padding, bias=False),
            norm(dim_out, affine=True),
            act]

def expand_Conv(module, in_channels):
    def expand_func(m):
        classname = m.__class__.__name__
        if classname.find('Conv2d') != -1:
            m.in_channels = in_channels
            m.out_channels = m.out_channels
            mean_weight = torch.mean(m.weight, dim=1, keepdim=True)
            m.weight.data = mean_weight.repeat(1, in_channels, 1, 1).data

    module.apply(expand_func)

##############################################################################
# Moduels
##############################################################################
class Upsample_Interpolate(nn.Module):

    def __init__(self, dim_in, dim_out, kernel_size=3, padding=1, norm=nn.BatchNorm2d, scale=2, mode='bilinear', reduce_dim=False):
        super(Upsample_Interpolate, self).__init__()
        self.scale = scale
        self.mode = mode
        if reduce_dim:
            dim_out = int(dim_out / 2)
            self.conv_norm_relu1 = conv_norm_relu(dim_in, dim_out, kernel_size=1, stride=1, padding=0, norm=norm)
            self.conv_norm_relu2 = conv_norm_relu(dim_out, dim_in, kernel_size=3, stride=1, padding=1, norm=norm)
        else:
            self.conv_norm_relu1 = conv_norm_relu(dim_in, dim_out, kernel_size=kernel_size, stride=1, padding=padding, norm=norm)
            self.conv_norm_relu2 = conv_norm_relu(dim_out, dim_out, kernel_size=3, stride=1, padding=1, norm=norm)

    def forward(self, x, activate=True):
        x = nn.functional.interpolate(x, scale_factor=self.scale, mode=self.mode, align_corners=True)
        x = self.conv_norm_relu1(x)
        x = self.conv_norm_relu2(x)
        return x

class Upconv_ConvTransposed(nn.Module):
    def __init__(self, dim_in, dim_out, kernel_size=3, stride=1, padding=1, output_padding=1):
        super(Upconv_ConvTransposed, self).__init__()
        self.conv_bn_relu = nn.Sequential(
            nn.ConvTranspose2d(dim_in, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, output_padding=output_padding),
            nn.BatchNorm2d(dim_out),
            nn.ReLU(inplace=True)
        )

    def forward(self, x):
        return self.conv_bn_relu(x)

class UpBasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, norm, using_semantic_branch=None):
        super(UpBasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, stride=1, padding=0, bias=False)
        self.bn1 = norm(planes)
        self.relu = nn.ReLU(inplace=True)

        self.conv2 = conv3x3(planes, planes)
        self.bn2 = norm(planes)

        self.using_semantic_branch = using_semantic_branch

    def forward(self, x):

        residual = x
        if self.using_semantic_branch is not None:
            x, conv_out = self.using_semantic_branch(x, activate=False)
            residual = conv_out

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        out += residual
        out = self.relu(out)

        return out
#########################################
class Discriminator(nn.Module):
    # initializers
    def __init__(self, cfg):
        super(Discriminator, self).__init__()
        self.cfg = cfg
        # norm = nn.BatchNorm2d
        self.d_downsample_num = 4
        norm = nn.InstanceNorm2d
        convs = nn.Sequential(
            nn.Conv2d(3, 32, 3, 1, 1),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(32, 64, 3, 2, 1),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(64, 128, 3, 2, 1),
            norm(128),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(128, 128, 3, 1, 1),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(128, 256, 3, 2, 1),
            norm(256),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(256, 256, 3, 1, 1),
            norm(256),
            nn.LeakyReLU(0.2, True),
            nn.Conv2d(256, 1, 3, 2, 1)
        )

        if self.cfg.NO_LSGAN:
            convs.add_module('sigmoid', nn.Sigmoid())

        self.model = nn.Sequential(*convs)
        init_weights(self.model, 'normal')

    def forward(self, x):
        return self.model(x)

##############################################################################
# Translate to recognize
##############################################################################
class Content_Model(nn.Module):

    def __init__(self, cfg, criterion=None):
        super(Content_Model, self).__init__()
        self.cfg = cfg
        self.criterion = criterion
        self.net = cfg.WHICH_CONTENT_NET

        if 'resnet' in self.net:
            from .pretrained_resnet import ResNet
            self.model = ResNet(self.net, cfg)

        fix_grad(self.model)
        # print_network(self)

    def forward(self, x, target, in_channel=3, layers=None):

        # important when set content_model as the attr of trecg_net
        self.model.eval()

        layers = layers
        if layers is None or not layers:
            layers = self.cfg.CONTENT_LAYERS.split(',')

        input_features = self.model((x + 1) / 2, layers)
        target_targets = self.model((target + 1) / 2, layers)
        len_layers = len(layers)
        loss_fns = [self.criterion] * len_layers
        alpha = [1] * len_layers

        content_losses = [alpha[i] * loss_fns[i](gen_content, target_targets[i])
                          for i, gen_content in enumerate(input_features)]
        loss = sum(content_losses) * self.cfg.ALPHA_CONTENT
        return loss

class TRecgNet_Upsample_Resiual(nn.Module):

    def __init__(self, cfg, num_classes=37,encoder='resnet18', using_semantic_branch=True,
                 use_noise=False, device=None):
        super(TRecgNet_Upsample_Resiual, self).__init__()

        self.encoder = encoder
        self.cfg = cfg
        self.using_semantic_branch = using_semantic_branch
        self.dim_noise = 128
        self.use_noise = use_noise
        self.device = device
        # self.avg_pool_size = 7

        dims = [32, 64, 128, 256, 512, 1024, 2048]

        fc_input_nc = dims[4] if encoder == 'resnet18' else dims[6]

        if cfg.PRETRAINED == 'imagenet' or cfg.PRETRAINED == 'place':
            pretrained = True
        else:
            pretrained = False

        if cfg.PRETRAINED == 'place':
            resnet = models.__dict__['resnet18'](num_classes=365)
            load_path = "/home/dudapeng/workspace/pretrained/place/resnet18_places365.pth"
            checkpoint = torch.load(load_path, map_location=lambda storage, loc: storage)
            state_dict = {str.replace(k, 'module.', ''): v for k, v in checkpoint['state_dict'].items()}
            resnet.load_state_dict(state_dict)
            print('place resnet18 loaded....')
        else:
            resnet = resnet18(pretrained=pretrained)
            print('{0} pretrained:{1}'.format(encoder, str(pretrained)))

        self.conv1 = resnet.conv1
        self.bn1 = resnet.bn1
        self.relu = resnet.relu
        self.maxpool = resnet.maxpool  # 1/4
        self.layer1 = resnet.layer1  # 1/4
        self.layer2 = resnet.layer2  # 1/8
        self.layer3 = resnet.layer3  # 1/16
        self.layer4 = resnet.layer4  # 1/32
        # self.build_upsample_layers(dims)
        # if self.upsample:
        # self.avg_pool_size = 14
        self.final=nn.Conv2d(dims[1],num_classes,kernel_size=1)
        self.build_upsample_segmentation_layers(dims)
        # if self.using_semantic_branch:
        #     self.build_upsample_content_layers(dims)

        # self.avgpool = nn.AvgPool2d(self.avg_pool_size, 1)
        # self.fc = nn.Linear(fc_input_nc, cfg.NUM_CLASSES)

        if pretrained:
            if using_semantic_branch:
                # init_weights(self.deconv1_content, 'normal')
                # init_weights(self.deconv2_content, 'normal')
                # init_weights(self.deconv3_content, 'normal')
                # init_weights(self.deconv4_content, 'normal')
                # init_weights(self.skip_3_content, 'normal')
                # init_weights(self.skip_2_content, 'normal')
                # init_weights(self.skip_1_content, 'normal')
                init_weights(self.up_image_content, 'normal')

            init_weights(self.deconv1_seg, 'normal')
            init_weights(self.deconv2_seg, 'normal')
            init_weights(self.deconv3_seg, 'normal')
            init_weights(self.deconv4_seg, 'normal')
            init_weights(self.deconv5_seg, 'normal')
            init_weights(self.skip_3_seg, 'normal')
            init_weights(self.skip_2_seg, 'normal')
            init_weights(self.skip_1_seg, 'normal')

        elif not pretrained:

            init_weights(self, 'normal')

    def set_content_model(self, content_model):
        self.content_model = content_model

    def set_pix2pix_criterion(self, criterion):
        self.pix2pix_criterion = criterion.to(self.device)

    def set_cls_criterion(self, criterion):
        self.cls_criterion = criterion.to(self.device)

    def build_upsample_segmentation_layers(self, dims):

        norm_seg = nn.InstanceNorm2d

        self.deconv1_seg = Upsample_Interpolate(dims[4], dims[3], kernel_size=1, padding=0, norm=norm_seg)
        self.deconv2_seg = Upsample_Interpolate(dims[3], dims[2], kernel_size=1, padding=0, norm=norm_seg)
        self.deconv3_seg = Upsample_Interpolate(dims[2], dims[1], kernel_size=1, padding=0, norm=norm_seg)
        self.deconv4_seg = Upsample_Interpolate(dims[1], dims[1], kernel_size=3, padding=1, norm=norm_seg)
        self.deconv5_seg = Upsample_Interpolate(dims[1], dims[1], kernel_size=3, padding=1, norm=norm_seg)

        skip3_channel_seg = dims[3]
        skip2_channel_seg = dims[2]
        skip1_channel_seg = dims[1]
        self.skip_3_seg = conv_norm_relu(skip3_channel_seg, skip3_channel_seg, kernel_size=1, padding=0, norm=norm_seg)
        self.skip_2_seg = conv_norm_relu(skip2_channel_seg, skip2_channel_seg, kernel_size=1, padding=0, norm=norm_seg)
        self.skip_1_seg = conv_norm_relu(skip1_channel_seg, skip1_channel_seg, kernel_size=1, padding=0, norm=norm_seg)
        self.up_image_content = nn.Sequential(
            nn.Conv2d(64, 3, 7, 1, 3, bias=False),
            nn.Tanh()
        )

    # def build_upsample_content_layers(self, dims):

    #     norm_content = nn.InstanceNorm2d

    #     self.deconv1_content = Upsample_Interpolate(dims[4], dims[3], kernel_size=1, padding=0, norm=norm_content)
    #     self.deconv2_content = Upsample_Interpolate(dims[3], dims[2], kernel_size=1, padding=0, norm=norm_content)
    #     self.deconv3_content = Upsample_Interpolate(dims[2], dims[1], kernel_size=1, padding=0, norm=norm_content)
    #     self.deconv4_content = Upsample_Interpolate(dims[1], dims[1], kernel_size=3, padding=1, norm=norm_content)

    #     skip3_channel_content = dims[3]
    #     skip2_channel_content = dims[2]
    #     skip1_channel_content = dims[1]
    #     self.skip_3_content = conv_norm_relu(skip3_channel_content, skip3_channel_content, kernel_size=1, padding=0, norm=norm_content)
    #     self.skip_2_content = conv_norm_relu(skip2_channel_content, skip2_channel_content, kernel_size=1, padding=0, norm=norm_content)
    #     self.skip_1_content = conv_norm_relu(skip1_channel_content, skip1_channel_content, kernel_size=1, padding=0, norm=norm_content)

    #     self.up_image_content = nn.Sequential(
    #         nn.Conv2d(64, 3, 7, 1, 3, bias=False),
    #         nn.Tanh()
    #     )

    def forward(self, source=None, target=None, label=None, out_keys=None, phase='train', content_layers=None, return_losses=True):
        out={}

        if self.using_semantic_branch:
            out['0'] = self.relu(self.bn1(self.conv1(source)))
        else:
            out['0'] = self.maxpool(self.relu(self.bn1(self.conv1(source))))

        out['1'] = self.layer1(out['0'])
        out['2'] = self.layer2(out['1'])
        out['3'] = self.layer3(out['2'])
        out['4'] = self.layer4(out['3'])
            #content model branch
            # skip1_cot = self.skip_1_content(out['1'])
            # skip2_cot = self.skip_2_content(out['2'])
            # skip3_cot = self.skip_3_content(out['3'])

            # upconv4 = self.deconv1_content(out['4'])
            # upconv3 = self.deconv2_content(upconv4 + skip3_cot)
            # upconv2 = self.deconv3_content(upconv3 + skip2_cot)
            # upconv1 = self.deconv4_content(upconv2 + skip1_cot)

            # out['gen_img'] = self.up_image_content(upconv1)
            # # segmentation branch
            # score_pool4 = self.deconv2_seg(upconv4)
            # score_pool3 = self.deconv3_seg(upconv3)
            # score_pool2 = self.deconv4_seg(upconv2)
            # score_pool1 = self.deconv5_seg(upconv1)
        # else:
            # segmentation branch
        skip1 = self.skip_1_seg(out['1'])
        skip2 = self.skip_2_seg(out['2'])
        skip3 = self.skip_3_seg(out['3'])

        upconv4 = self.deconv1_seg(out['4'])
        upconv3 = self.deconv2_seg(upconv4 + skip3)
        upconv2 = self.deconv3_seg(upconv3 + skip2)
        upconv1 = self.deconv4_seg(upconv2 + skip1)
        if self.using_semantic_branch and phase=='train':
            out['gen_img'] = self.up_image_content(upconv1)

        if self.using_semantic_branch:
            out['cls'] = self.final(upconv1)
        else:
            upconv0 = self.deconv5_seg(upconv1)
            out['cls'] = self.final(upconv0)

        loss_content = None
        loss_cls = None
        loss_pix2pix=None
        if return_losses and not self.cfg.INFERENCE:

            if self.using_semantic_branch and phase == 'train':

                loss_content = self.content_model(out['gen_img'], target, layers=content_layers) * self.cfg.ALPHA_CLS

            loss_cls= self.cls_criterion(out['cls'] , label) * self.cfg.ALPHA_CLS

        result = []
        for key in out_keys:
            # print(key)
            if isinstance(key, list):
                item = [out[subkey] for subkey in key]
            else:
                item = out[key]
            result.append(item)

        return result, {'cls_loss': loss_cls, 'content_loss': loss_content, 'pix2pix_loss': loss_pix2pix}


# Twitch Modpack Server Updater

# [User configuration]
	# Mods profile
PROFILE = "profile.zip"

	# By changing the setting below to True you are indicating your agreement to the Minecraft EULA (https://account.mojang.com/documents/minecraft_eula)
MC_EULA = False

	# JVM settings
MEMORY_MIN = 2048
MEMORY_MAX = 4096
JAVA_PATH = ""

	# Number of threads a threadpool should spawn
	# Typically, this should be set to the amount of cores available times two
NUM_THREADS = 16
# ---

# --- TOUCHING ANYTHING BEYOND THIS POINT MAY BREAK TMSU --- #

# [Imports]
import os
import sys
import json
import requests

from io import BytesIO
from time import sleep
from zipfile import ZipFile
from subprocess import call as spcall
from multiprocessing.dummy import Pool
# ---

# [Internal configuration]
FORGE_INSTALLER_TEMPLATE = "forge-{}-{}-installer.jar"
VANILLA_SERVER_TEMPLATE = "minecraft_server.{}.jar"
FORGE_SERVER_TEMPLATE = "forge-{}-{}-universal.jar"
VERSION_MANIFEST_URL = "https://launchermeta.mojang.com/mc/game/version_manifest.json"
CURSE_URL_TEMPLATE = "https://minecraft.curseforge.com/projects/{}/files/{}/download"
FORGE_URL_TEMPLATE = "http://files.minecraftforge.net/maven/net/minecraftforge/forge/{0}-{1}/forge-{0}-{1}-installer.jar"
HOUSEKEEPING_FILE = "tmsu.json"
OVERRIDES_FOLDER = "./"
MODS_FOLDER = "./mods/"

FORGE_SERVER_JAR = ""
FORGE_DOUBLE_VERSIONS = [ "1.7.10", "1.8.9" ]
FORGE_URL_TEMPLATE_DOUBLE = "http://files.minecraftforge.net/maven/net/minecraftforge/forge/{0}-{1}-{0}/forge-{0}-{1}-{0}-installer.jar"
FORGE_SERVER_TEMPLATE_DOUBLE = "forge-{0}-{1}-{0}-universal.jar"
# ---

# [Helper functions]
def Request( url ):
	print( "> " + url, end = " " )
	sys.stdout.flush()
	result = requests.get( url )
	print( "[OK]" )

	return result

def Path():
    return os.path.dirname( os.path.realpath( sys.argv[ 0 ] ) )
# ---

# [Download functions]
def DownloadVanilla( version ): # Download vanilla server jar by version (e.g. version="1.12.2")
	# Fetch version manifest
	versionManifest = Request( VERSION_MANIFEST_URL ).json()

	# Search for requested version
	mfEntry = [ v for v in versionManifest[ "versions" ] if v[ "id" ] == version ]

	if len( mfEntry ) == 0:
		raise KeyError( "Requested version not found in manifest file" )

	# Request and return the actual version json after finding it
	versionInfo = Request( mfEntry[ 0 ][ "url" ] ).json()

	with open( VANILLA_SERVER_TEMPLATE.format( version ), "wb" ) as f:
		f.write( Request( versionInfo[ "downloads" ][ "server" ][ "url" ] ).content )

def DownloadForge( mcVersion, forgeVersion ):
	filename = FORGE_INSTALLER_TEMPLATE.format( mcVersion, forgeVersion )
	with open( filename, "wb" ) as f:
		if mcVersion in FORGE_DOUBLE_VERSIONS:
			f.write( Request( FORGE_URL_TEMPLATE_DOUBLE.format( mcVersion, forgeVersion ) ).content )
		else:
			f.write( Request( FORGE_URL_TEMPLATE.format( mcVersion, forgeVersion ) ).content )

	print( "> Installing Forge", end = " " )
	sys.stdout.flush()
	with open( os.devnull, "w" ) as fnull:
		spcall( [ JAVA_PATH + "java", "-jar", filename, "--installServer" ], stdout = fnull )
	os.remove( filename )
	print( "[OK]" )

def DownloadMod( projectID, fileID ):
	mod = requests.get( CURSE_URL_TEMPLATE.format( str( projectID ), str( fileID ) ) )
	filename = mod.url[ mod.url.rfind( "/" ) + 1 : ]

	key = str( projectID ) + "-" + str( fileID )

	with open( MODS_FOLDER + filename, "wb" ) as f:
			f.write( mod.content )

	print( "> " + mod.url + " [OK]" )
	return [ key, filename ]

def ExtractOverride( zf, filename, overrides ):
	with zf.open( filename ) as f:
		target = OVERRIDES_FOLDER + filename[ len( overrides ) : ]
		if os.path.isfile( target ):
			os.remove( target )

		with open( target, "wb" ) as ff:
			ff.write( f.read() )

def UpdateFromZip( zipFile ): # Update install by zip file (e.g. file="MyCustomPack-1.0a.zip")
	modManifest = {}

	# Read mod manifest from zip
	with ZipFile( zipFile ) as zf:
		with zf.open( "manifest.json", "r" ) as f:
			modManifest = json.loads( f.read().decode( "utf-8" ) )

	# Check if vanilla server jar is installed
	mcVersion = modManifest[ "minecraft" ][ "version" ]
	if not os.path.isfile( VANILLA_SERVER_TEMPLATE.format( mcVersion ) ):
		DownloadVanilla( mcVersion )

	# Check if forge universal jar is installed
	forgeVersion = modManifest[ "minecraft" ][ "modLoaders" ][ 0 ][ "id" ].split( "-" )[ 1 ]

	global FORGE_SERVER_JAR
	if mcVersion in FORGE_DOUBLE_VERSIONS:
		FORGE_SERVER_JAR = FORGE_SERVER_TEMPLATE_DOUBLE.format( mcVersion, forgeVersion )
	else:
		FORGE_SERVER_JAR = FORGE_SERVER_TEMPLATE.format( mcVersion, forgeVersion )

	if not os.path.isfile( FORGE_SERVER_JAR ):
		DownloadForge( mcVersion, forgeVersion )
	# Check if mods folder exsits
	if not os.path.isdir( MODS_FOLDER ):
		os.mkdir( MODS_FOLDER )

	# Housekeeping
	print( "> Housekeeping (1/2)", end = " " )
	sys.stdout.flush()
	hkNew = {}
	for file in modManifest[ "files" ]:
		key = str( file[ "projectID" ] ) + "-" + str( file[ "fileID" ] )
		hkNew[ key ] = key + ".jar"

	hkOld = {}
	if os.path.isfile( HOUSEKEEPING_FILE ):
		with open( HOUSEKEEPING_FILE, "rb" ) as f: # Load old housekeeping file
			hkOld = json.loads( f.read().decode( "utf-8" ) )

		os.remove( HOUSEKEEPING_FILE )

		for key in hkOld: # Remove obsolete mods
			if not key in hkNew:
				if os.path.isfile( MODS_FOLDER + hkOld[ key ] ):
					os.remove( MODS_FOLDER + hkOld[ key ] )
			else:
				hkNew[ key ] = hkOld[ key ] # Move filename from old to new housekeeping file

	print( "[OK]" )

	# Download new mods and build new housekeeping file
	job = [] # Make a job for threaded downloads
	jobResult = []

	for file in modManifest[ "files" ]:
		key = str( file[ "projectID" ] ) + "-" + str( file[ "fileID" ] )
		if not os.path.isfile( MODS_FOLDER + hkNew[ key ] ):
			job.append( ( file[ "projectID" ], file[ "fileID" ] ) )

	with Pool( NUM_THREADS ) as p:
		jobResult = p.starmap( DownloadMod, job )

	print( "> Housekeeping (2/2)", end = " " )
	sys.stdout.flush()
	for v in jobResult:
		hkNew[ v[ 0 ] ] = v[ 1 ]

	with open( HOUSEKEEPING_FILE, "w" ) as f: # Write new housekeeping file
		f.write( json.dumps( hkNew, sort_keys = True ) )

	print( "[OK]" )

	overrides = modManifest[ "overrides" ] + "/"

	job = [] # New job for threaded extraction

	print( "> Overrides", end = " " )
	with ZipFile( zipFile ) as zf:
		for filename in zf.namelist():
			if filename.startswith( overrides ) and not filename.endswith( "/" ):
				dirs = os.path.dirname( filename[ len( overrides ) : ] )
				if not os.path.isdir( dirs ): # Make directory if it does not exist
					os.makedirs( dirs )

				job.append( ( zf, filename, overrides ) )

		with Pool( NUM_THREADS ) as p:
			jobResult = p.starmap( ExtractOverride, job )
	print( "[OK]" )

def UpdateFromURL( url ): # Update install by url (e.g. url="http://my.host/my-pack.zip")
	mods = Request( url )

	with BytesIO() as buffer:
		buffer.write( mods.content )
		UpdateFromZip( buffer )
# ---

# [Run updater]
if PROFILE.startswith( "http://" ) or PROFILE.startswith( "https://" ) or PROFILE.startswith( "ftp://" ):
	UpdateFromURL( PROFILE )
else:
	UpdateFromZip( PROFILE )
# ---

# [MC EULA]
if MC_EULA == True:
	if os.path.isfile( "eula.txt" ):
		os.remove( "eula.txt" )

	with open( "eula.txt", "w" ) as f:
		f.write( "eula=true" )
# ---

# [Run server]
spcall( [ JAVA_PATH + "java", "-Xms" + str( MEMORY_MIN ) + "M", "-Xmx" + str( MEMORY_MAX ) + "M", "-jar", FORGE_SERVER_JAR, "nogui" ] )
# ---


import yaml
import os
#获取元素表格路径
# os.path.realpath(__file__)获取当前执行脚本的绝对路径 #D:\workspace\ui_test\common\configuration\configuration.py
# os.path.dirname()：去掉脚本的文件名，返回目录。
# os.path.dirname(os.path.realname(__file__))：指的是，该语句所在py文件的绝对路径，__file__为内置属性。
# os.path.join()函数：连接两个或更多的路径名组件
# yaml_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../config.yml")#这是两个点“..”，也就是上级目录的表示方法../../上上级
# D:\workspace\ui_test\common\configuration\../../config.yml
# 1.如果各组件名首字母不包含’/’，则函数会自动加上
# 2.如果有一个组件是一个绝对路径，则在它之前的所有组件均会被舍弃
# 3.如果最后一个组件为空，则生成的路径以一个’/’分隔符结尾
def filePath(filename):
    file = os.path.join(os.path.dirname(os.path.realpath(__file__)), filename)
    # file = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), filename)
    # print(file)
    # print(os.path.dirname(os.path.realpath(__file__)))
    return file
if __name__ == '__main__':
    # 传入建打印值
    print(filePath("AssetElement.xlsx"))


import os
import time
import threading
from datetime import datetime, timedelta

from sqlalchemy import func

from app_parser import app
from app_parser import db
from app_parser.domain.model import Download, Config
from app_parser.service.torrent_service import Torrent

download_pool_ids = set()

def run():
    while True:
        db.session.close()
        if not is_downloader_active() \
                or not is_space_enough() \
                or len(download_pool_ids) >= Config.get('BT_POOL_LIMIT', int):
            time.sleep(5)
            continue

        d = get_from_queue()
        if not d:
            app.logger.warn('no item to download')
            time.sleep(10)
            continue

        t = threading.Thread(target=download, daemon=True, args=(d,), name='thread_download_id_{}'.format(d.id))
        t.start()
        app.logger.info('new download thread started {}, pid {}'.format(t, os.getpid()))
        time.sleep(3)

def download(d: Download):
    download_pool_ids.add(d.id)
    d = Download.query.filter_by(id=d.id).first()

    try:
        d.save_path = os.path.join(Config.get('BT_DOWNLOAD_DIR'), str(int(d.search_id / 1000)), str(d.search_id))
        torrent = Torrent(d)
        torrent.download()
    except Exception as e:
        d.status = Download.Statuses.ERROR
        d.error = str(e)
    finally:
        db.session.commit()
        download_pool_ids.remove(d.id)

def get_from_queue() -> Download:
    min_changed_at = datetime.utcnow() - timedelta(minutes=30)
    return Download.query\
        .filter(
            Download.id.notin_(download_pool_ids) &
            (
                (Download.status.in_([Download.Statuses.NEW, Download.Statuses.PAUSED])) |
                ((Download.status == Download.Statuses.DOWNLOADING) & (Download.changed_at < min_changed_at))
            )
        )\
        .order_by(func.rand())\
        .first()

def is_downloader_active():
    is_active = Config.get('BT_IS_ACTIVE', bool)
    if not is_active:
        app.logger.warn('Further downloads stopped via configs, downloads in queue {}'.format(len(download_pool_ids)))

    return is_active

def is_space_enough():
    from app_parser.utils.system import get_disk_usage_perc
    max = Config.get('BT_USE_DISK_SPACE_PERC', float)
    current = get_disk_usage_perc()
    is_enough = current < max

    if not is_enough:
        app.logger.warn('Further downloads stopped disk space limit reached (limit={}%, used={}%), '
                        'downloads in queue {}'.format(max, current, len(download_pool_ids)))

    return is_enough

run()


# -----------------------------------
# Undersampling with bagging
# Author: Victoria A. Lestari
# June 12, 2017
# -----------------------------------
import numpy as np
# from imblearn.under_sampling import RandomUnderSampler
# from imblearn.ensemble import EasyEnsemble
import collections
from sklearn.linear_model import LogisticRegression

class EnsembleClassifier:
	def __init__(self, size):
		self.models = [] # empty list for models
		self.size = size # the number of classifiers

		for i in range(size):
			model = LogisticRegression()
			self.models.append(model)

	def fit(self, X_train_agg, y_train_agg, sample_weight=None):
		print("-----------------------------------------------")
		print("X, y train agg")
		print("-----------------------------------------------")
		print(len(X_train_agg), np.array(X_train_agg[0]).shape)
		print(len(y_train_agg), y_train_agg)
		print("-----------------------------------------------")
		for i in range(self.size):
			print(np.array(X_train_agg[i]).shape)
			self.models[i].fit(X_train_agg[i], y_train_agg[i], sample_weight=sample_weight)

	def predict(self, X_test):
		y_preds = []

		for i in range(self.size):
			y_pred = self.models[i].predict(X_test)[0]
			# print "y pred", y_pred
			y_preds.append(y_pred)

		return collections.Counter(y_preds).most_common(1)[0][0]

	def predict_proba(self, X_test):
		y_prob = [0, 0]

		for i in range(self.size):
			tmp = self.models[i].predict_proba(X_test)[0]
			y_prob[0] += tmp[0]
			y_prob[1] += tmp[1]

		y_prob[0] /= self.size
		y_prob[1] /= self.size

		# print "y_prob", y_prob
		return y_prob

	# ------------------------------------
	# decide the label of items given a set of labels produced by ensemble classifiers
	# ------------------------------------
	def aggregate_vote(self, y_aggregate):
	    ensemble_size = len(y_aggregate)
	    sample_size = len(y_aggregate[0])
	    y_decision = []

	    for i in range(sample_size):
	        tmp_y = collections.Counter([ens[i] for ens in y_aggregate]).most_common(1)[0][0]
	        y_decision.append(tmp_y)

	    return y_decision

# y_aggregate = [[1,0,0],[1,0,0],[1,1,1],[1,0,1],[0,0,0]]

# print aggregate_vote(y_aggregate)

# ------------------------------------
# X_train_agg and y_train_agg are outputs of EasyEnsemble, n sets of training data
# We build n classifiers and test them on X_test and y_test
# And then we use the function aggregate_vote to vote for the majority
# model is the model we use to train the classifier
# ------------------------------------
# def ensemble_classifier(X_train_agg, y_train_agg, X_test, y_test, model, proba=False):
# 	n = len(X_train_agg)
# 	y_test_agg = []

# 	for i in range(n):
# 		curr_X = X_train_agg[i]
# 		curr_y = y_train_agg[i]
# 		model.fit(curr_X, curr_y)

# 		y_prob = None
# 		if proba:
# 			y_prob = model.predict_proba(X_test)[0]
# 		else:
# 			y_prob = model.predict(X_test)[0]

# 		y_test_agg.append(y_prob)

# 	return aggregate_vote(y_test_agg)


# -*- coding: utf-8 -*
__author__ = 'double k'

"""
唤醒APP
工作位置
x/2
y/1.05
"""

import time
import traceback
import os
import re
import configparser
import datetime
import random
import sched
import requests
scheduler = sched.scheduler(time.time, time.sleep)
path = os.getcwd() + "\\DingDingClockIn\\"
config = configparser.ConfigParser(allow_no_value=False)
# .cfg路径根据自己实际情况修改
config.read( path + "dingding.cfg")
go_hour = int(config.get("time", "go_hour"))
back_hour = int(config.get("time", "back_hour"))
# go_hour = 8
# back_hour = 17
def wakeUpTheScreen():
    displayPowerState = os.popen(
        "adb shell \"dumpsys power | grep 'Display Power: state=' \"").read().strip('\n')
    if displayPowerState == 'Display Power: state=OFF':
        print("唤醒屏幕")
        os.system("adb shell \"input keyevent 26\"")
    else:
        print("屏幕已开启不需要唤醒")
def deblocking():
    isStatusBarKeyguard = os.popen(
        "adb shell \"dumpsys window policy|grep isStatusBarKeyguard \"").read().strip(
        '\n')
    # print(isStatusBarKeyguard)
    # return
    if "isStatusBarKeyguard=false" in isStatusBarKeyguard:
        time.sleep(2)
        print("解锁屏保")
        # 滑动解锁
        os.system('adb shell \"input swipe  300 1000 300 500\"')
        time.sleep(1)
        print("输入密码")
        os.system('adb shell \"input text 910510\"')
    else:
        print("屏幕已解锁不需要再次解锁")

def screencap(hourtype):
    if hourtype == 2:
        pref = "go"
    else:
        pref = "back"
    fileName = pref + "-" + time.strftime("%Y%m%d%H%M%S") + ".png"
    recordPath = path + "record\\"
    dir = recordPath + fileName
    os.system("adb shell screencap -p sdcard/screen.png")
    os.system("adb pull sdcard/screen.png %s" % dir)
    os.system("adb shell rm -r sdcard/screen.png")
    print("screencap to computer success")
def screenshot_prepare(hourtype):
    """
    打开APP
    :return:
    """
    try:
        """
        打开app
        """
        appName = "com.alibaba.android.rimet"
        # 唤醒屏幕
        wakeUpTheScreen()
        # 解锁
        deblocking()

        time.sleep(1)
        # 获取正在启动的APP
        mFocusedActivity = os.popen(
            "adb shell \"dumpsys activity | grep 'mFocusedActivity' \"").read().strip('\n')
        if appName in mFocusedActivity:
            print("APP已启动，停止APP，等待重新启动")
            os.system("adb shell \"am force-stop %s\"" % (appName,))
        time.sleep(1)
        print("启动app")
        os.system(
            "adb shell \"monkey -p %s -c android.intent.category.LAUNCHER 1\"" % (appName, ))
        xy = os.popen("adb shell wm size").read().strip('\n')

        xyobj = re.search(re.compile("\d+x\d+"), xy).group().split("x")
        if len(xyobj) == 2:
            time.sleep(8)
            # 截屏
            screencap(hourtype)
            time.sleep(2)
            x = int(xyobj[0]) / 2
            y = int(xyobj[1]) / 1.05
            os.system('adb shell \"input tap %s %s\"' % (x, y))

        os.system("adb shell \"input keyevent 26\"")
    except Exception:
        print("screenshot_prepare error")
        # traceback.print_exc()
        # exit(-1)

# 随机打卡时间段
def random_minute():
    return random.randint(30,50)

# 包装循环函数，传入随机打卡时间点
def incode_loop(func,minute):
    """
    包装start_loop任务调度函数，主要是为了传入随机分钟数。保证在不打卡的情况下能保持分钟数不变。
    :param func: star_loop
    :param minute: 随机分钟数
    :return: None
    """
    # 判断时间当超过上班时间则打下班卡。否则则打上班卡。
    if datetime.datetime.now().hour >=go_hour and datetime.datetime.now().hour < back_hour :
        # 用来分类上班和下班。作为参数传入任务调度
        hourtype = 1
        print("下班打卡-下次将在", str(back_hour), ":", str(minute), "打卡")
    else:
        hourtype = 2
        print("上班打卡-下次将在", str(go_hour), ":", str(minute), "打卡")
    #执行任务调度函数
    func(hourtype, minute)

# 任务调度
def start_loop(hourtype,minute):
    """
    每次循环完成，携带生成的随机分钟数来再次进行循环，当打卡后，再重新生成随机数
    :param hourtype: 设置的上班时间点
    :param minute: 随机生成的分钟数（30-55）
    :return: None
    """
    now_time = datetime.datetime.now()
    now_hour = now_time.hour
    now_minute = now_time.minute
    hourtype = hourtype
    # 上班，不是周末（双休），小时对应，随机分钟对应
    if hourtype == 2 and now_hour == go_hour and now_minute == minute and is_weekend():
        print("hourtype", str(hourtype), "now_hour", str(now_hour), "now_minute", str(now_minute))
        random_time = random_minute()
        screenshot_prepare(hourtype)
        scheduler.enter(0,0,incode_loop,(start_loop,random_time,))
        return
    if hourtype == 1 and now_hour == back_hour and now_minute == minute and is_weekend():
        print("hourtype", str(hourtype), "now_hour", str(now_hour), "now_minute", str(now_minute))
        random_time = random_minute()
        screenshot_prepare(hourtype)
        scheduler.enter(0, 0, incode_loop, (start_loop,random_time,))
        return
    else:
        if hourtype ==1:
            nextHour = back_hour
        else:
            nextHour = go_hour
        print("现在时间：", now_hour, ':', now_minute, "--下次执行事件", nextHour, ":", minute)
        scheduler.enter(60, 0, start_loop, (hourtype, minute, ))
        return

"""
是否休息
判断是否需要调休
判断是否是法定节假日
"""
def is_weekend():
    """
    :return: 如果需要休息返回 False 否则 返回True
    """
    holiday = False
    try:
        url = "http://timor.tech/api/holiday/info/" + time.strftime("%Y-%m-%d")
        r = requests.get(url)
        result = r.json()
        code = result['code']
        holidayInfo = result['holiday']
        if code == 0:
            if holidayInfo:
                holiday = True
                if holidayInfo['holiday']:
                    print("今天是%s不打卡" % holidayInfo['name'])
                    return False
                else:
                    print("今天是%s需要打卡" % holidayInfo['name'])
                    return True
    except Exception:
        holiday = False

    if not holiday:
        now_time = datetime.datetime.now().strftime("%w")
        if now_time == "6" or now_time == "0":
            print("今天周末不打卡")
            return False
        else:
            return True
if __name__ == "__main__":
    # ======formal
    scheduler.enter(0, 0, incode_loop, (start_loop, random_minute(),))
    scheduler.run()


# https://leetcode.com/problems/longest-consecutive-sequence

class Solution:
    def longestConsecutive(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        longest = 0
        set_nums = set(nums)

        for n in set_nums:
            if n-1 not in set_nums:
                curr = n
                curr_seq = 1

                while curr + 1 in set_nums:
                    curr += 1
                    curr_seq += 1

                longest = max(longest, curr_seq)
        return longest

s = Solution()
arr = [100, 4, 200, 1, 3, 2]
print(s.longestConsecutive(arr))


import os
import subprocess
import numpy as np
import calendar
import pdb

def join_files(files_in, file_name_out):

    subprocess.call('cdo mergetime '+files_in+' '+file_name_out, shell=True)

def add_to_list(file_in, file_out, nc_file_string_in):

    if not os.path.isfile(file_out):
        subprocess.call('cdo -f nc copy '+file_in + ' ' + file_out, shell=True)

    if not (file_out in nc_file_string_in):
        nc_file_string=nc_file_string_in+' '+file_out
    else:
        nc_file_string = nc_file_string_in

    return nc_file_string

if __name__=="__main__":

    variable = 'swhr'
    file_prefix = 'fcst_phy3m125.250_swhr'
#     file_prefix = 'anl_surf125.002_prmsl'
    #file_prefix = 'anl_p25.034_vgrd'
    do_remap = True
    do_day_mean = True
    do_month_mean = True

    avg_or_daily = '6hourly'
    folder = variable+'_'+avg_or_daily

    years = np.arange(1958,2017)
    months = np.arange(12)+1

    nc_file_string = ''
    if avg_or_daily=='monthly':
        for year_val in years:
            file_in = '/disca/share/rg419/JRA_55/'+folder+'/'+file_prefix+'.'+str(year_val)+'010100_'+str(year_val)+'123100'
            file_out = '/disca/share/rg419/JRA_55/'+folder+'/'+file_prefix+'_'+str(year_val)+'010100_'+str(year_val)+'123100.nc'
            add_to_list(file_in, file_out, nc_file_string_in)

    if avg_or_daily=='6hourly':
        for year_val in years:
            for month_val in months:
                end_month_day = calendar.monthrange(year_val, month_val)[1]

                month_string ='%02d' % month_val

                file_in = '/disca/share/rg419/JRA_55/'+folder+'/'+file_prefix+'.'+str(year_val)+month_string+'0100_'+str(year_val)+month_string+str(end_month_day)+'18'
                file_out = '/disca/share/rg419/JRA_55/'+folder+'/'+file_prefix+'_'+str(year_val)+month_string+'0100_'+str(year_val)+month_string+str(end_month_day)+'18.nc'

                if not(os.path.isfile(file_in) or os.path.isfile(file_out)):
                    file_in = '/disca/share/rg419/JRA_55/'+folder+'/'+file_prefix+'.'+str(year_val)+'010100_'+str(year_val)+'123118'
                    file_out = '/disca/share/rg419/JRA_55/'+folder+'/'+file_prefix+'_'+str(year_val)+'010100_'+str(year_val)+'123118.nc'

                nc_file_string = add_to_list(file_in, file_out, nc_file_string)

    nc_file_out='/disca/share/rg419/JRA_55/'+folder+'/atmos_'+avg_or_daily+'_together.nc'
    if not os.path.isfile(nc_file_out):
        print('joining files')
        join_files(nc_file_string,nc_file_out)

#     pdb.set_trace()

    if do_day_mean:
        if not os.path.isdir('/disca/share/rg419/JRA_55/'+variable+'_daily/'):
            os.mkdir('/disca/share/rg419/JRA_55/'+variable+'_daily/')
        daymean_file_out = '/disca/share/rg419/JRA_55/'+variable+'_daily/atmos_daily_together.nc'
        if not os.path.isfile(daymean_file_out):
            print('doing daily mean')
            subprocess.call('cdo daymean '+nc_file_out+' '+daymean_file_out,shell=True)

    if do_day_mean and do_month_mean:
        if not os.path.isdir('/disca/share/rg419/JRA_55/'+variable+'_monthly/'):
            os.mkdir('/disca/share/rg419/JRA_55/'+variable+'_monthly/')
        monmean_file_out = '/disca/share/rg419/JRA_55/'+variable+'_monthly/atmos_monthly_together.nc'
        if not os.path.isfile(monmean_file_out):
            print('doing monthly mean from daily data')
            subprocess.call('cdo monmean '+daymean_file_out+' '+monmean_file_out,shell=True)
    elif (not do_day_mean) and do_month_mean:
        if not os.path.isdir('/disca/share/rg419/JRA_55/'+variable+'_monthly/'):
            os.mkdir('/disca/share/rg419/JRA_55/'+variable+'_monthly/')
        monmean_file_out = '/disca/share/rg419/JRA_55/'+variable+'_monthly/atmos_monthly_together.nc'
        if not os.path.isfile(monmean_file_out):
            print('doing monthly mean from 6hourly data')
            subprocess.call('cdo monmean '+nc_file_out+' '+monmean_file_out,shell=True)

    if do_remap:
        remap_file_out = nc_file_out[:-3]+'_2_5.nc'
        if not os.path.isfile(remap_file_out):
            print('doing remapping')
            subprocess.call('cdo remapbil,r144x73 '+nc_file_out+' '+remap_file_out,shell=True)
        if do_day_mean:
            daymean_remap_file_out = daymean_file_out[:-3]+'_2_5.nc'
            if not os.path.isfile(daymean_remap_file_out):
                print('doing remapping')
                subprocess.call('cdo remapbil,r144x73 '+daymean_file_out+' '+daymean_remap_file_out,shell=True)
        if do_month_mean:
            monmean_remap_file_out = monmean_file_out[:-3]+'_2_5.nc'
            if not os.path.isfile(monmean_remap_file_out):
                print('doing remapping')
                subprocess.call('cdo remapbil,r144x73 '+monmean_file_out+' '+monmean_remap_file_out,shell=True)


# -*- coding: utf-8 -*-

class Paginator(object):
    def __init__(self, data, count=10, page=1):
        self.data = data

        if count <= 1:
            self.count = 1
        else:
            self.count = int(count)

        try:
            page = int(page)
        except:
            page = 1
        if page <=1:
            self.page = 1
        else:
            self.page = page

    @property
    def item_num(self):
        try:
            num = self.data.count()
        except (AttributeError, TypeError):
            num = len(self.data)
        return num

    @property
    def page_num(self):
        item_num = self.item_num
        count = self.count
        if item_num % count:
            return item_num / count + 1
        return item_num / count

    @property
    def has_next(self):
        p1 = self.page
        p2 = self.page_num
        if p1 < p2:
            return True
        return False

    @property
    def next_num(self):
        return int(self.page + 1)

    @property
    def has_previous(self):
        p1 = self.page
        if p1 > 1:
            return True
        return False

    @property
    def previous_num(self):
        return int(self.page - 1)

    def page_range(self, num=4):
        current = self.page
        xlist = range(current-num, current+num+1)
        plist = range(1, self.page_num + 1)
        data = []
        for p in xlist:
            if p in plist:
                data.append(p)
        return data

    @property
    def show_first(self):
        if self.page > 5:
            return True
        return False

    @property
    def show_first_dash(self):
        p = self.page
        if p > 5 and p != 6:
            return True
        return False

    @property
    def show_last(self):
        n = self.page_num - self.page
        if n > 5:
            return True
        return False

    @property
    def show_last_dash(self):
        n = self.page_num - self.page
        if n > 5 and n != 6:
            return True
        return False

    def get_items(self):
        limit = self.count
        offset = (self.page - 1)*limit
        try:
            mvdata = self.data.fetch(limit, offset)
        except:
            mvdata = self.data[offset:limit*self.page]
        return mvdata
    object_list = property(get_items)


import re

f = open('input.txt','r')
myfile = f.read()
rule1 = re.compile(r'([a-z]*[aeiou]){3,}')
rule2 = re.compile(r'([a-z])\1')
rule3 = re.compile(r'ab|cd|pq|xy')

srule1 = re.compile(r'([a-z]{2})[a-z]*(\1)')
srule2 = re.compile(r'([a-z]).(\1)')

niceCount = 0
naughtyCount = -1
newNiceCount = 0
newNaughtyCount = -1
for line in myfile.split('\n'):
    r1match = rule1.search(line) != None
    r2match = rule2.search(line) != None
    r3match = rule3.search(line) == None
    nice = r1match and r2match and r3match
    if nice:
        niceCount+=1
    else:
        naughtyCount+=1
    #Part 2
    nr1match = srule1.search(line) != None
    nr2match = srule2.search(line) != None
    newnice = nr1match and nr2match
    if newnice:
        newNiceCount+=1
    else:
        newNaughtyCount+=1
print('Nice: '+str(niceCount)+' Naughty: '+str(naughtyCount))
print('New Nice: '+str(newNiceCount)+' New Naughty: '+str(newNaughtyCount))


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import tempfile
import glob
import os
from turbo_seti_sofia.plot_event_pipeline import plot_event_pipeline
freqDictLoc1 = {
    '0704-0832' : ['/mnt_blpd2/datax/PKSUWL/blc01/PKSUWL/','blc01'],
    '0832-0960' : ['/mnt_blpd2/datax/PKSUWL/blc02/PKSUWL/','blc02'],
    '0960-1088' : ['/mnt_blpd2/datax/PKSUWL/blc03/PKSUWL/','blc03'],
    '1088-1216' : ['/mnt_blpd2/datax/PKSUWL/blc04/PKSUWL/','blc04'],
    '1216-1344' : ['/mnt_blpd2/datax/PKSUWL/blc05/PKSUWL/','blc05'],
    '1344-1472' : ['/mnt_blpd2/datax/PKSUWL/blc06/PKSUWL/','blc06'],
    '1472-1600' : ['/mnt_blpd2/datax/PKSUWL/blc07/PKSUWL/','blc07'],
    '1600-1728' : ['/mnt_blpd2/datax/PKSUWL/blc10/PKSUWL/','blc10'],
    '1728-1856' : ['/mnt_blpd2/datax/PKSUWL/blc11/PKSUWL/','blc11'],
    '1856-1984' : ['/mnt_blpd2/datax/PKSUWL/blc12/PKSUWL/','blc12'],
    '1984-2112' : ['/mnt_blpd2/datax/PKSUWL/blc13/PKSUWL/','blc13'],
    '2112-2240' : ['/mnt_blpd2/datax/PKSUWL/blc14/PKSUWL/','blc14'],
    '2240-2368' : ['/mnt_blpd2/datax/PKSUWL/blc15/PKSUWL/','blc15'],
    '2368-2496' : ['/mnt_blpd2/datax/PKSUWL/blc16/PKSUWL/','blc16'],
    '2496-2624' : ['/mnt_blpd2/datax/PKSUWL/blc17/PKSUWL/','blc17'],
    '2624-2742' : ['/mnt_blpd2/datax/PKSUWL/blc20/PKSUWL/','blc20'],
    '2742-2880' : ['/mnt_blpd2/datax/PKSUWL/blc21/PKSUWL/','blc21'],
    '2880-3008' : ['/mnt_blpd2/datax/PKSUWL/blc22/PKSUWL/','blc22'],
    '3008-3136' : ['/mnt_blpd2/datax2/PKSUWL/blc23/PKSUWL/','blc23'],
    '3136-3264' : ['/mnt_blpd2/datax2/PKSUWL/blc24/PKSUWL/','blc24'],
    '3264-3392' : ['/mnt_blpd2/datax2/PKSUWL/blc25/PKSUWL/','blc25'],
    '3392-3520' : ['/mnt_blpd2/datax2/PKSUWL/blc26/PKSUWL/','blc26'],
    '3520-3648' : ['/mnt_blpd2/datax2/PKSUWL/blc27/PKSUWL/','blc27'],
    '3648-3776' : ['/mnt_blpd2/datax2/PKSUWL/blc30/PKSUWL/','blc30'],
    '3776-3904' : ['/mnt_blpd2/datax2/PKSUWL/blc31/PKSUWL/','blc31'],
    '3904-4032' : ['/mnt_blpd2/datax2/PKSUWL/blc32/PKSUWL/','blc32']
}

freqDictLoc = {
    'blc01' : ['/mnt_blpd2/datax/PKSUWL/blc01/PKSUWL/','0704-0832'],
    'blc02' : ['/mnt_blpd2/datax/PKSUWL/blc02/PKSUWL/','0832-0960'],
    'blc03' : ['/mnt_blpd2/datax/PKSUWL/blc03/PKSUWL/','0960-1088'],
    'blc04' : ['/mnt_blpd2/datax/PKSUWL/blc04/PKSUWL/','1088-1216'],
    'blc05' : ['/mnt_blpd2/datax/PKSUWL/blc05/PKSUWL/','1216-1344'],
    'blc06' : ['/mnt_blpd2/datax/PKSUWL/blc06/PKSUWL/','1344-1472'],
    'blc07' : ['/mnt_blpd2/datax/PKSUWL/blc07/PKSUWL/','1472-1600'],
    'blc10' : ['/mnt_blpd2/datax/PKSUWL/blc10/PKSUWL/','1600-1728'],
    'blc11' : ['/mnt_blpd2/datax/PKSUWL/blc11/PKSUWL/','1728-1856'],
    'blc12' : ['/mnt_blpd2/datax/PKSUWL/blc12/PKSUWL/','1856-1984'],
    'blc13' : ['/mnt_blpd2/datax/PKSUWL/blc13/PKSUWL/','1984-2112'],
    'blc14' : ['/mnt_blpd2/datax/PKSUWL/blc14/PKSUWL/','2112-2240'],
    'blc15' : ['/mnt_blpd2/datax/PKSUWL/blc15/PKSUWL/','2240-2368'],
    'blc16' : ['/mnt_blpd2/datax/PKSUWL/blc16/PKSUWL/','2368-2496'],
    'blc17' : ['/mnt_blpd2/datax/PKSUWL/blc17/PKSUWL/','2496-2624'],
    'blc20' : ['/mnt_blpd2/datax/PKSUWL/blc20/PKSUWL/','2624-2742'],
    'blc21' : ['/mnt_blpd2/datax/PKSUWL/blc21/PKSUWL/','2742-2880'],
    'blc22' : ['/mnt_blpd2/datax/PKSUWL/blc22/PKSUWL/','2880-3008'],
    'blc23' : ['/mnt_blpd2/datax2/PKSUWL/blc23/PKSUWL/','3008-3136'],
    'blc24' : ['/mnt_blpd2/datax2/PKSUWL/blc24/PKSUWL/','3136-3264'],
    'blc25' : ['/mnt_blpd2/datax2/PKSUWL/blc25/PKSUWL/','3264-3392'],
    'blc26' : ['/mnt_blpd2/datax2/PKSUWL/blc26/PKSUWL/','3392-3520'],
    'blc27' : ['/mnt_blpd2/datax2/PKSUWL/blc27/PKSUWL/','3520-3648'],
    'blc30' : ['/mnt_blpd2/datax2/PKSUWL/blc30/PKSUWL/','3648-3776'],
    'blc31' : ['/mnt_blpd2/datax2/PKSUWL/blc31/PKSUWL/','3776-3904'],
    'blc32' : ['/mnt_blpd2/datax2/PKSUWL/blc32/PKSUWL/','3904-4032']
}

# def exclude_rfi():

#     def check_rfi(freq_range):
#         rfi_ranges = [[703,798],[867.5,897.4],[939.3,965.1],[1141.795,1318],[1551.795,1602],
#                       [1795,1882.5],[2105,2125],[2137,2152.5],[2162.3,2172.7],[2292.8,2388.8],
#                       [2660,2700],[3435.1,3474.9],[3540.1,3579.9]]

#         for rfi in rfi_ranges:
#             if my_ints[0] > rfi[0]:
#                 if my_ints[1] < rfi[1]:
#                     return True

#         return False

#     group_list = glob.glob('/datax/scratch/ssmith/event_groups/ProxCen*.csv')
#     exclusive_list = []

#     for group in group_list:
#         myrng = group.split('/')[-1].split('_')[1].split('-')
#         my_ints = [int(i) for i in myrng]

#         if check_rfi(my_ints) == False:
#             exclusive_list.append(group)
#     exclusive_list = exclusive_list[968::]
#     return exclusive_list

def acquire_cad_list(filename):
    os.chdir('/home/ssmith/proxcen/turboSETI/search_dirs/')

    try:
        filepath = freqDictLoc[filename.split('_')[0]][0]
        freq = freqDictLoc[filename.split('_')[0]][1]
        file_loc = filepath + os.path.splitext(filename)[0] + '.fil'
    except:
        filepath = freqDictLoc[filename.split('_')[0][13:]][0]
        freq = freqDictLoc[filename.split('_')[0][13:]][1]

        file_loc = filepath + os.path.splitext(filename[13:])[0] + '.fil'

    txt_file = freq + 'MHz/' + filename.split('_')[-1].split('T')[0].strip().replace('-','') + '.txt'
    files_list = open(txt_file).read().splitlines()
    file_index = files_list.index(file_loc)

    return_list = [file_loc]
    j = 1
    for i in range(1,12):
        try:
            filei = files_list[file_index + i]
            return_list.append(filei)
        except:
            filei = files_list[file_index - j]
            return_list.insert(0,filei)
            j += 1
        #print(filei)
    return return_list

def sample_and_plot(filelist):   #where file list is a list of group .csvs to run on

    for file in filelist:
        #randomly sample
        df = pd.read_csv(file)
        if len(df) > 1:
            rand_index = np.random.randint(0,len(df))
            rand_event = df.iloc[[rand_index]]
        else:
            rand_event = df

        #build cadence
        file_name = rand_event.FileID.item()
        # print(file_name)
        cadence = acquire_cad_list(file_name)

        #temporary .lst
        temp_lst = tempfile.NamedTemporaryFile(suffix='.lst', mode='w+t')
        for fil in cadence:
            temp_lst.writelines(fil+'\n')
        temp_lst.seek(0)
        #temporary .csv
        os.chdir('/datax/scratch/ssmith/event_groups/images/')
        rand_event.to_csv('temporary.csv', index=False)
        #generate plot
        plot_event_pipeline('temporary.csv',temp_lst.name)

        temp_lst.close()

        plt.clf()
        plt.close()

def plot_all(df):   #where file list is a list of group .csvs to run on

    for file in df['FileID']:
        idf = pd.read_csv(file)
        for i in range(idf.shape[0]):
            rand_event = idf.iloc[[i]]

            #build cadence
            file_name = rand_event.FileID.item()
            # print(file_name)
            cadence = acquire_cad_list(file_name)

            #temporary .lst
            temp_lst = tempfile.NamedTemporaryFile(suffix='.lst', mode='w+t')
            for fil in cadence:
                temp_lst.writelines(fil+'\n')
            temp_lst.seek(0)
            #temporary .csv
            os.chdir('/datax/scratch/ssmith/event_groups/images3/')
            rand_event.to_csv('temporary.csv', index=False)
            #generate plot
            plot_event_pipeline('temporary.csv',temp_lst.name)

            temp_lst.close()

            plt.clf()
            plt.close()

def plot_all_events(df):

    for i in range(len(df)):
        temp_df = df.iloc[[i]]
        cadence = acquire_cad_list(temp_df.FileID.item())
        os.chdir('/datax/scratch/ssmith/new_event_groups/')

        #temporary .lst
        temp_lst = tempfile.NamedTemporaryFile(suffix='.lst', mode='w+t')
        for fil in cadence:
            temp_lst.writelines(fil+'\n')
        temp_lst.seek(0)

        temp_df.to_csv('temporary.csv', index=False)
        plot_event_pipeline('temporary.csv', temp_lst.name)

        temp_lst.close()
        plt.clf()
        plt.close()

def look_for_event(filename):

    date = filename.split('_')[2] + '.000.h5'

    #look for node
    freq = filename.split('_')[6].split('.')[0] +'.'+ filename.split('_')[6].split('.')[1]
    for key in freqDictLoc1.keys():
        upper = key.split('-')[1]
        lower = key.split('-')[0]
        if float(lower) <= float(freq) < float(upper):
            node = freqDictLoc1[key][1]
            break

    fil_name = node+'_ProxCen_S_'+date

    file_df = big_df.loc[big_df['FileID'] == fil_name]

    freq = float(freq)
    final_event = file_df.query('FreqStart <= @freq < FreqEnd')
    return final_event

from datetime import datetime

os.chdir('/home/ssmith/proxcen/turboSETI/search_dirs2/')
csv_list = glob.glob('**/ProxCen*cadlen4.csv', recursive=True)
csv_list.sort(key = lambda x: (x.split('/')[0], datetime.strptime(x.split('_')[1].split('T')[0], '%Y-%m-%d')))
df_list = [0] * len(csv_list)

for i, file in enumerate(csv_list):
    df_list[i] = pd.read_csv(file, index_col=0)

big_df = pd.concat(df_list)

new_list = open('/home/ssmith/proxcen/turboSETI/follow_up.txt').read().splitlines()
new_list = [image.split(' ')[0] for image in new_list]

for img in new_list:
    temp_df = look_for_event(img)

    try:
        cadence = acquire_cad_list(temp_df.FileID.item())
    except:
        continue

    os.chdir('/datax/scratch/ssmith/new_event_groups/round2')

    #temporary .lst
    temp_lst = tempfile.NamedTemporaryFile(suffix='.lst', mode='w+t')
    for fil in cadence:
        temp_lst.writelines(fil+'\n')
    temp_lst.seek(0)

    temp_df.to_csv('temporary.csv', index=False)
    plot_event_pipeline('temporary.csv', temp_lst.name)

    temp_lst.close()
    plt.clf()
    plt.close()


import math

from Homework_to_lesson_OOP import shape_class

class ITriangle(shape_class.IShape):

    def __init__(self, side_one, side_two, angle=90, name='abc'):
        self.name = name
        self.angles = 3
        self.side_one = side_one
        self.side_two = side_two
        self.angle = angle
        self.side_three = self.get_side_three(self.side_one, self.side_two, self.angle)
        self.perimeter = self.get_perimeter(self.side_one, self.side_three, self.side_two)
        self.area = self.get_area(self.side_one, self.side_two, self.angle)

    @staticmethod
    def get_side_three(side_one, side_two, angle):
        side_three = math.sqrt(side_one**2 + side_two**2 - 2 * side_one * side_two * math.cos(math.radians(angle)))
        return side_three

    @staticmethod
    def get_perimeter(side_one, side_two, side_three):
        perimeter = side_one + side_two + side_three
        return perimeter

    @staticmethod
    def get_area(side_ab, side_ac, angle):
        area = 1/2 * side_ab * side_ac * math.sin(math.radians(angle))
        return area


import socket
import struct
import time

UDP_IP = "127.0.0.1"
UDP_PORT = 6000

sock = socket.socket(socket.AF_INET, # Internet
                     socket.SOCK_DGRAM) # UDP
sock.bind((UDP_IP, UDP_PORT))

data, addr = sock.recvfrom(4 + 3*8)
(i, x, y, psi) = struct.unpack('=Iddd', data)
print("id: {}, x: {}, y: {}, psi: {}".format(i, x, y, psi))
assert i == 1
assert x == 2
assert y == 3
assert psi == -40000


"""
description:
has additional properties than binary tree
left subtree values are less than or equal to parent
right subtree values are greater than or equal to parent

pros:
quick insertion and deletion
half - half - half

operations:
create - node with empty data, empty left child, empty right child
insert - find correct spot by traversing from root
traverse(4 ways) - same as binary tree
search - traverse left or right depending on small or big
delete node
    leaf - search and delete directly
    with one child - search and just delete and move the child upwards
    with two children - find second largest element to that i.e. smallest node in right subtree, swap and delete
delete tree - make root node data, leftchild and rightchild as none

data structure used:
linked list
list (possible)
"""

queue = __import__("052_queue_from_linkedlist_create_del_peek_enq_deq_isempty_isfull")

class Node:
    # O(1), O(1)
    def __init__(self, data):
        self.data = data
        self.leftchild = None
        self.rightchild = None

# O(logn), O(logn)
def insert(rootnode, value):
    if rootnode.data == None:
        rootnode.data = value
    elif value <= rootnode.data:
        if rootnode.leftchild == None:
            rootnode.leftchild = Node(value)
        else:
            insert(rootnode.leftchild, value)
    else:
        if rootnode.rightchild == None:
            rootnode.rightchild = Node(value)
        else:
            insert(rootnode.rightchild, value)
    return "inserted"

# O(n) - half trees traversed on both sides, O(n)
def preordertraversal(rootnode):
    if not rootnode:
        return
    print(rootnode.data)
    preordertraversal(rootnode.leftchild)
    preordertraversal(rootnode.rightchild)

# O(n) - half trees traversed on both sides, O(n)
def inordertraversal(rootnode):
    if not rootnode:
        return
    inordertraversal(rootnode.leftchild)
    print(rootnode.data)
    inordertraversal(rootnode.rightchild)

# O(n) - half trees traversed on both sides, O(n)
def postordertraversal(rootnode):
    if not rootnode:
        return
    postordertraversal(rootnode.leftchild)
    postordertraversal(rootnode.rightchild)
    print(rootnode.data)

# O(n) - half trees traversed on both sides, O(n)
def levelordertraversal(rootnode):
    if not rootnode:
        return
    q = queue.Queue()
    q.enqueue(rootnode)
    while not q.isempty():
        root = q.dequeue()
        print(root.value.data)
        if root.value.leftchild:
            q.enqueue(root.value.leftchild)
        if root.value.rightchild:
            q.enqueue(root.value.rightchild)

# O(logn), O(logn)
# learn more about elements equal to the one present in list
# write on own - learn more
def search(rootnode, value):
    if rootnode:
        if rootnode.data == value:
            print("found")
        elif value < rootnode.data:
            if rootnode.leftchild:
                if rootnode.leftchild.data == value:
                    print("found")
                else:
                    search(rootnode.leftchild, value)
        elif rootnode.rightchild:
            if rootnode.rightchild.data == value:
                print("found")
            else:
                search(rootnode.rightchild, value)

def minvaluenode(node):
    currentnode = node
    while currentnode.leftchild:
        currentnode = currentnode.leftchild
    return currentnode

# O(logn) , O(logn) - learn more
# write on own - learn more
def deletenode(rootnode, nodevalue):
    if not rootnode:
        return None
    if nodevalue < rootnode.data:
        rootnode.leftchild = deletenode(rootnode.leftchild, nodevalue)
    elif nodevalue > rootnode.data:
        rootnode.rightchild = deletenode(rootnode.rightchild, nodevalue)
    else:
        if rootnode.leftchild is None:
            temp = rootnode.rightchild
            rootnode = None
            return temp
        if rootnode.rightchild is None:
            temp = rootnode.leftchild
            rootnode = None
            return temp
        temp = minvaluenode(rootnode.rightchild)
        rootnode.data = temp.data
        rootnode.rightchild = deletenode(rootnode.rightchild, temp.data)
    return rootnode

# O(1), O(1)
def delete(rootnode):
    rootnode.data = None
    rootnode.leftchild = None
    rootnode.rightchild = None
    return "deleted"

n = Node(None)
print(insert(n, 70))
print(insert(n, 50))
print(insert(n, 90))
print(insert(n, 30))
print(insert(n, 60))
print(insert(n, 80))
print(insert(n, 100))
print(insert(n, 20))
print(insert(n, 40))
print(insert(n, 95))
print(insert(n, 105))
print("####################")
preordertraversal(n)
print("####################")
inordertraversal(n)
print("####################")
postordertraversal(n)
print("####################")
levelordertraversal(n)
print("####################")
search(n, 0)
print("####################")
deletenode(n, 90)
levelordertraversal(n)
print("####################")
print(delete(n))
levelordertraversal(n)


# coding=utf-8
import pandas
from datetime import date
# import matplotlib.pyplot as plt

transactions = pandas.read_csv('cardstrxsmarch2017.csv', sep=';', thousands=',')
# transactions = pandas.read_csv('cards-test.csv', sep=';', thousands=',')
transactions.columns = ['SOU_SERVICE', 'CARD_SYSTEM_NAME', 'COUNTRY_NAME_FULL', 'CARD_CURRENCY', 'CURRENCY_TRANS_NAME',
                        'OPERATION_TYPE_NAME', 'DATE_TRANS', 'CR_SID', 'TRANSACTION_DETAILS', 'OPERATION_AMOUNT',
                        'ACCOUNT_CUR_AMOUNT', 'ACCOUNT_CUR_AMOUNT_LCY', 'PROD_TYPE', 'IS_INTERNET_TRANS']

# transactions = transactions.set_index(['CR_SID'])

pandas.set_option('display.max_rows', 10)

crsids = transactions.drop_duplicates("CR_SID")
crsids = crsids.drop(['SOU_SERVICE', 'CARD_SYSTEM_NAME', 'COUNTRY_NAME_FULL', 'CARD_CURRENCY', 'CURRENCY_TRANS_NAME',
                        'OPERATION_TYPE_NAME', 'DATE_TRANS', 'TRANSACTION_DETAILS', 'OPERATION_AMOUNT',
                        'ACCOUNT_CUR_AMOUNT', 'ACCOUNT_CUR_AMOUNT_LCY', 'PROD_TYPE', 'IS_INTERNET_TRANS'], axis=1)

g_trans = transactions.groupby('CR_SID', as_index=False).agg({'SOU_SERVICE': 'count', 'ACCOUNT_CUR_AMOUNT_LCY': 'sum'})
g_trans = g_trans.rename_axis({"SOU_SERVICE": "trans_count", "ACCOUNT_CUR_AMOUNT_LCY": "sum_amount"}, axis=1)
crsids = crsids.join(g_trans.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_gtrans')

restor = transactions[transactions['TRANSACTION_DETAILS'] == "cafe-bar-restoran"]
restor = restor.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
restor = restor.rename_axis({"TRANSACTION_DETAILS": "use_cafe-bar-restoran"}, axis=1)
crsids = crsids.join(restor.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_restor')
crsids['use_cafe-bar-restoran'] = crsids['use_cafe-bar-restoran'].fillna(0)

shops = transactions[transactions['TRANSACTION_DETAILS'] == "shops"]
shops = shops.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
shops = shops.rename_axis({"TRANSACTION_DETAILS": "use_shops"}, axis=1)
crsids = crsids.join(shops.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_restor')
crsids['use_shops'] = crsids['use_shops'].fillna(0)

azs = transactions[transactions['TRANSACTION_DETAILS'] == "azs"]
azs = azs.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
azs = azs.rename_axis({"TRANSACTION_DETAILS": "use_azs"}, axis=1)
crsids = crsids.join(azs.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_restor')
crsids['use_azs'] = crsids['use_azs'].fillna(0)

atm = transactions[transactions['TRANSACTION_DETAILS'] == "ATM"]
atm = atm.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
atm = atm.rename_axis({"TRANSACTION_DETAILS": "use_atm"}, axis=1)
crsids = crsids.join(atm.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['use_atm'] = crsids['use_atm'].fillna(0)

aliexpress = transactions[transactions['TRANSACTION_DETAILS'] == "aliexpress"]
aliexpress = aliexpress.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
aliexpress = aliexpress.rename_axis({"TRANSACTION_DETAILS": "use_aliexpress"}, axis=1)
crsids = crsids.join(aliexpress.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['use_aliexpress'] = crsids['use_aliexpress'].fillna(0)

UBER = transactions[transactions['TRANSACTION_DETAILS'] == "UBER"]
UBER = UBER.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
UBER = UBER.rename_axis({"TRANSACTION_DETAILS": "use_UBER"}, axis=1)
crsids = crsids.join(UBER.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['use_UBER'] = crsids['use_UBER'].fillna(0)

ITUNES = transactions[transactions['TRANSACTION_DETAILS'] == "ITUNES"]
ITUNES = ITUNES.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
ITUNES = ITUNES.rename_axis({"TRANSACTION_DETAILS": "use_ITUNES"}, axis=1)
crsids = crsids.join(ITUNES.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['use_ITUNES'] = crsids['use_ITUNES'].fillna(0)

metro = transactions[transactions['TRANSACTION_DETAILS'] == "metro"]
metro = metro.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
metro = metro.rename_axis({"TRANSACTION_DETAILS": "use_metro"}, axis=1)
crsids = crsids.join(metro.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['use_metro'] = crsids['use_metro'].fillna(0)

pets_zoo = transactions[transactions['TRANSACTION_DETAILS'] == "pets zoo"]
pets_zoo = pets_zoo.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
pets_zoo = pets_zoo.rename_axis({"TRANSACTION_DETAILS": "use_pets_zoo"}, axis=1)
crsids = crsids.join(pets_zoo.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['use_pets_zoo'] = crsids['use_pets_zoo'].fillna(0)

fitness = transactions[transactions['TRANSACTION_DETAILS'] == "fitness"]
fitness = fitness.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
fitness = fitness.rename_axis({"TRANSACTION_DETAILS": "use_fitness"}, axis=1)
crsids = crsids.join(fitness.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['use_fitness'] = crsids['use_fitness'].fillna(0)

no_card_out_belarus = transactions[transactions['COUNTRY_NAME_FULL'] != "BELARUS"]
no_card_out_belarus = no_card_out_belarus[no_card_out_belarus['IS_INTERNET_TRANS'] == 0]
no_card_out_belarus = no_card_out_belarus.groupby('CR_SID', as_index=False).agg({'TRANSACTION_DETAILS': 'count'})
no_card_out_belarus = no_card_out_belarus.rename_axis({"TRANSACTION_DETAILS": "no_card_out_belarus"}, axis=1)
crsids = crsids.join(no_card_out_belarus.set_index('CR_SID'), on="CR_SID", lsuffix='', rsuffix='_atm')
crsids['no_card_out_belarus'] = crsids['no_card_out_belarus'].fillna(0)

# print(crsids)
crsids.to_csv("transactions.csv")