# coding=utf-8
# !/usr/bin/python
import sys
sys.path.append('..')
from base.spider import Spider
import json
import re
import urllib

class Spider(Spider):  # 元类 默认的元类 type
    def getName(self):
        return "Alist"

    def init(self, extend=""):
        print("============{0}============".format(extend))
        pass

    def isVideoFormat(self, url):
        pass

    def manualVideoCheck(self):
        pass

    def homeContent(self, filter):
        result = {}
        cateManual = {
          "菊花盘": "https://pan.142856.xyz/OneDrive",
		"课程搜索": "https://www.souke365.cn",
		"博哥": "www.top12345.top",
          "🔮嗨翻":"https://pan.hikerfans.com",
		  "帅仔1":"http://43.139.29.179:5244/",
                  "帅仔2":"http://43.139.29.179:5245/",
		  "🦀9T(Adult)":"https://drive.9t.ee",
		  "🐱梓澪の妙妙屋":"https://xn--i0v44m.xyz",
		  "🚆资源小站":"https://pan.142856.xyz",
		  "🌤晴园的宝藏库":"https://alist.52qy.repl.co",
		  "🐭米奇妙妙屋":"https://anime.mqmmw.ga",
		  "💂小兵组网盘影视":"https://6vv.app",
		  "📀小光盘":"https://alist.xiaoguanxiaocheng.life",
		  "🐋一只鱼":"https://alist.youte.ml",
		  "🌊七米蓝":"https://al.chirmyram.com",
		  "🌴非盘":"http://www.feifwp.top",
		  "🥼帅盘":"https://hi.shuaipeng.wang",
		  "🐉神族九帝":"https://alist.shenzjd.com",
		  "☃姬路白雪":"https://pan.jlbx.xyz",
		  "🎧听闻网盘":"https://wangpan.sangxuesheng.com",
		  "💾DISK":"http://124.222.140.243:8080",
		  "🌨云播放":"https://quanzi.laoxianghuijia.cn",
		  "✨星梦":"https://pan.bashroot.top",
		  "🌊小江":"https://dyj.me",
		  "💫触光":"https://pan.ichuguang.com",
		  "🕵好汉吧":"https://8023.haohanba.cn",
		  "🥗AUNEY":"http://121.227.25.116:8008",
		  "🎡资源小站":"https://960303.xyz/",
		  "🐝神器云": "https://quanzi.laoxianghuijia.cn",
		  "🏝fenwe":"http://www.fenwe.tk:5244",
		  "🎢轻弹浅唱":"https://g.xiang.lol"
        }
        classes = []
        for k in cateManual:
            classes.append({
                'type_name': k,
				"type_flag": "1",
                'type_id': cateManual[k]
            })
        result['class'] = classes
        if (filter):
            result['filters'] = self.config['filter']
        return result

    def homeVideoContent(self):
        result = {
            'list': []
        }
        return result

    ver = ''
    baseurl = ''
    def getVersion(self, gtid):
        param = {
            "path": '/'
        }
        if gtid.count('/') == 2:
            gtid = gtid + '/'
        baseurl = re.findall(r"http.*://.*?/", gtid)[0]
        ver = self.fetch(baseurl + 'api/public/settings', param)
        vjo = json.loads(ver.text)['data']
        if type(vjo) is dict:
            ver = 3
        else:
            ver = 2
        self.ver = ver
        self.baseurl = baseurl

    def categoryContent(self, tid, pg, filter, extend):
        result = {}
        if tid.count('/') == 2:
            tid = tid + '/'
        nurl = re.findall(r"http.*://.*?/", tid)[0]
        if self.ver == '' or self.baseurl != nurl:
            self.getVersion(tid)
        ver = self.ver
        baseurl = self.baseurl
        if tid.count('/') == 2:
            tid = tid + '/'
        pat = tid.replace(baseurl,"")
        param = {
            "path": '/' + pat
        }
        if ver == 2:
            rsp = self.postJson(baseurl + 'api/public/path', param)
            jo = json.loads(rsp.text)
            vodList = jo['data']['files']
        elif ver == 3:
            rsp = self.postJson(baseurl + 'api/fs/list', param)
            jo = json.loads(rsp.text)
            vodList = jo['data']['content']

        videos = []
        cid = ''
        purl = ''
        svodList = str(vodList)
        lenvodList = len(vodList)
        substr = str(re.findall(r"\'name\': \'(.*?)\'", svodList))
        foldernum = svodList.count('\'type\': 1')
        filenum = lenvodList - foldernum
        for vod in vodList:
            if ver == 2:
                img = vod['thumbnail']
            elif ver == 3:
                img = vod['thumb']
            if len(img) == 0:
                if vod['type'] == 1:
                    img = "http://img1.3png.com/281e284a670865a71d91515866552b5f172b.png"
            if pat != '':
                aid = pat + '/'
            else:
                aid = pat
            if vod['type'] == 1:
                tag = "folder"
                remark = "文件夹"
                cid = baseurl + aid + vod['name']
            #计算文件大小
            else:
                size = vod['size']
                if size > 1024 * 1024 * 1024 * 1024.0:
                    fs = "TB"
                    sz = round(size / (1024 * 1024 * 1024 * 1024.0), 2)
                elif size > 1024 * 1024 * 1024.0:
                    fs = "GB"
                    sz = round(size / (1024 * 1024 * 1024.0), 2)
                elif size > 1024 * 1024.0:
                    fs = "MB"
                    sz = round(size / (1024 * 1024.0), 2)
                elif size > 1024.0:
                    fs = "KB"
                    sz = round(size / (1024.0), 2)
                else:
                    fs = "KB"
                    sz = round(size / (1024.0), 2)
                tag = "file"
                remark = str(sz) + fs
                cid = baseurl + aid + vod['name']
                # 开始爬视频与字幕
                if filenum < 150:
                    if 'mp4' in vod['name'] or 'mkv' in vod['name'] or 'TS' in vod['name'] or 'flv' in vod['name'] or 'rmvb' in vod['name'] or 'mp3' in vod['name'] or 'flac' in vod['name'] or 'wav' in vod['name'] or 'wma' in vod['name'] or 'dff' in vod['name']:
                        vodurl = vod['name']
                        # 开始爬字幕
                        cid = '###'
                        subname = re.findall(r"(.*)\.", vod['name'])[0]
                        if filenum == 2:
                            if '.ass' in substr:
                                sub = re.findall(r"'(.*).ass", substr)[0]
                                subt = '@@@' + sub + '.ass'
                                if ',' in sub:
                                    sub = re.findall(r"', '(.*).ass", substr)[0]
                                    subt = '@@@' + sub + '.ass'
                            if '.srt' in substr:
                                sub = re.findall(r"'(.*).srt", substr)[0]
                                subt = '@@@' + sub + '.srt'
                                if ',' in sub:
                                    sub = re.findall(r"', '(.*).srt", substr)[0]
                                    subt = '@@@' + sub + '.srt'
                        else:
                            if subname + '.ass' in substr:
                                subt = '@@@' + subname + '.ass'
                            elif subname + '.srt' in substr:
                                subt = '@@@' + subname + '.srt'
                        # 合并链接
                        if 'subt' in locals().keys():
                            purl = purl + '{0}{1}#'.format(vodurl, subt)
                        else:
                            purl = purl + '{0}#'.format(vodurl)
                else:
                    subname = re.findall(r"(.*)\.", vod['name'])[0]
                    if subname + '.ass' in substr:
                        subt = '@@@' + subname + '.ass'
                        cid = cid + subt
                    elif subname + '.srt' in substr:
                        subt = '@@@' + subname + '.srt'
                        cid = cid + subt
            videos.append({
                "vod_id":  cid,
                "vod_name": vod['name'],
                "vod_pic": img,
                "vod_tag": tag,
                "vod_remarks": remark
            })
        if 'purl' in locals().keys():
            purl = baseurl + aid + '+++' + purl
            for i in range(foldernum, lenvodList):
                if videos[i]['vod_id'] == '###':
                    videos[i]['vod_id'] = purl
        result['list'] = videos
        result['page'] = 1
        result['pagecount'] = 1
        result['limit'] = 999
        result['total'] = 999999
        return result

    def detailContent(self, array):
        id = array[0]
        if '+++' in id:
            ids = id.split('+++')
            durl = ids[0]
            vsList = ids[1].strip('#').split('#')
            vsurl = ''
            for vs in vsList:
                if '@@@' in vs:
                    dvs = vs.split('@@@')
                    vname = dvs[0]
                    vurl = durl + dvs[0]
                    surl = durl + dvs[1]
                    vsurl = vsurl + '{0}${1}@@@{2}#'.format(vname, vurl, surl)
                else:
                    vurl = durl + vs
                    vsurl = vsurl + '{0}${1}#'.format(vs, vurl)
            url = vsurl
        else:
            durl = id
        if self.ver == '' or self.baseurl == '':
            self.getVersion(durl)
        baseurl = self.baseurl
        if '+++' in id:
            vid = durl.replace(baseurl, "").strip('/')
        else:
            vid = durl.replace(re.findall(r".*/", durl)[0], "")
            url = vid + '$' + id
        vod = {
            "vod_id": vid,
            "vod_name": vid,
            "vod_pic": '',
            "vod_tag": '',
            "vod_play_from": "播放",
            "vod_play_url": url
        }
        result = {
            'list': [
                vod
            ]
        }
        return result

    def searchContent(self, key, quick):
        result = {
            'list': []
        }
        return result

    def playerContent(self, flag, id, vipFlags):
        result = {}
        url = ''
        subturl = ''
        ifsub = '@@@' in id
        if ifsub is True:
            ids = id.split('@@@')
            if self.ver == '' or self.baseurl == '':
                self.getVersion(ids[1])
            ver = self.ver
            baseurl = self.baseurl
            fileName = ids[1].replace(baseurl, "")
            vfileName = ids[0].replace(baseurl, "")
            param = {
                "path": '/' + fileName,
                "password": "",
                "page_num": 1,
                "page_size": 100
            }
            vparam = {
                "path": '/' + vfileName,
                "password": "",
                "page_num": 1,
                "page_size": 100
            }
            if ver == 2:
                rsp = self.postJson(baseurl + 'api/public/path', param)
                jo = json.loads(rsp.text)
                vodList = jo['data']['files'][0]
                subturl = vodList['url']
                vrsp = self.postJson(baseurl + 'api/public/path', vparam)
                vjo = json.loads(vrsp.text)
                vList = vjo['data']['files'][0]
                url = vList['url']
            elif ver == 3:
                rsp = self.postJson(baseurl + 'api/fs/get', param)
                jo = json.loads(rsp.text)
                vodList = jo['data']
                subturl = vodList['raw_url']
                vrsp = self.postJson(baseurl + 'api/fs/get', vparam)
                vjo = json.loads(vrsp.text)
                vList = vjo['data']
                url = vList['raw_url']
            if subturl.startswith('http') is False:
                head = re.findall(r"h.*?:", baseurl)[0]
                subturl = head + subturl
            if url.startswith('http') is False:
                head = re.findall(r"h.*?:", baseurl)[0]
                url = head + url
            urlfileName = urllib.parse.quote(fileName)
            subturl = subturl.replace(fileName, urlfileName)
            urlvfileName = urllib.parse.quote(vfileName)
            url = url.replace(vfileName, urlvfileName)
            result['subt'] = subturl
        else:
            if self.ver == '' or self.baseurl == '':
                self.getVersion(id)
            ver = self.ver
            baseurl = self.baseurl
            vfileName = id.replace(baseurl, "")
            vparam = {
                "path": '/' + vfileName,
                "password": "",
                "page_num": 1,
                "page_size": 100
            }
            if ver == 2:
                vrsp = self.postJson(baseurl + 'api/public/path', vparam)
                vjo = json.loads(vrsp.text)
                vList = vjo['data']['files'][0]
                driver = vList['driver']
                url = vList['url']
            elif ver == 3:
                vrsp = self.postJson(baseurl + 'api/fs/get', vparam)
                vjo = json.loads(vrsp.text)
                vList = vjo['data']
                url = vList['raw_url']
                driver = vList['provider']
            if url.startswith('http') is False:
                head = re.findall(r"h.*?:", baseurl)[0]
                url = head + url
            urlvfileName = urllib.parse.quote(vfileName)
            url = url.replace(vfileName, urlvfileName)
            if driver == 'Baidu.Disk':
                result["header"] = {"User-Agent": "pan.baidu.com"}
        result["parse"] = 0
        result["playUrl"] = ''
        result["url"] = url

        return result

    config = {
        "player": {},
        "filter": {}
    }
    header = {}

    def localProxy(self, param):
        return [200, "video/MP2T", action, ""]


import json
import yaml
import datetime

def filter_merge_img_json(txt_paths: list, json_paths: list, save_path: str, full: bool=False):
    """
    Create new json file (coco_true format) with image_paths from correspondence txt file
    Input:
        txt_paths: list of txt path
        json_path: list of json path
        save path: path to save output json file
        full: if full is True, merge full json files, not get paths from txt_paths
    """

    print(f'Number txt paths: {len(txt_paths)}', '\n', f'Number json paths: {len(json_paths)}')
    assert len(txt_paths) == len(json_paths)

    write_json_context = dict()
    write_json_context['info'] = {
        'description': '', 'url': '', 'version': '', 'year': 2021,
        'contributor': '', 'date_created': f'{datetime.datetime.now()}'
    }
    write_json_context['licenses'] = [{'id': 1, 'name': None, 'url': None}]
    write_json_context['categories'] = [
        {'id': 0,'name': 'face','supercategory': 'face'},
        {'id': 1,'name': 'plate','supercategory': 'plate'}
    ]
    write_json_context['images'] = []
    write_json_context['annotations'] = []
    start_img_id = 0
    # debug
    num_imgs = 0
    num_bboxes = 0

    for txt_p, js_p in zip(txt_paths, json_paths):
        with open(txt_p) as txt_file:
            img_path_list = txt_file.read().strip().split('\n')

        with open(js_p) as json_file:
            image_data = json.load(json_file)

        old_img_ids = []
        new_img_ids = []
        for img in image_data['images']:
#             if img['file_name'] in img_path_list:
            if not full and img['file_name'] not in img_path_list:
                continue
            old_img_ids.append(img['id'])
            img['id'] = start_img_id + 1
            new_img_ids.append(img['id'])
            write_json_context['images'].append(img)
            start_img_id += 1
            num_imgs += 1

        for anno in image_data['annotations']:
#             if anno['image_id'] in old_img_ids:
            if not full and anno['image_id'] not in old_img_ids:
                continue
            anno['image_id'] = new_img_ids[old_img_ids.index(anno['image_id'])]
            write_json_context['annotations'].append(anno)
            num_bboxes += 1

    with open(save_path, 'w') as write_f:
        json.dump(write_json_context, write_f, indent=4)

    print('num_imgs:', num_imgs)
    print('num_bboxes', num_bboxes)

if __name__ == "__main__":

    with open("utils/configs/merge_json.yaml", 'r') as stream:
        try:
            config_dict = yaml.safe_load(stream)
        except yaml.YAMLError as exc:
            print(exc)

    filter_merge_img_json(
        config_dict['txt_paths'],
        config_dict['json_paths'],
        config_dict['save_path'],
        config_dict['full']
    )


import os
import tempfile

from google.cloud import storage, vision
from wand.image import Image

storage_client = storage.Client()
client = vision.ImageAnnotatorClient()

def main(event, context):
    print("Evento-contexto", context)

    #client = vision.ImageAnnotatorClient()

    image_to_open = 'cara.jpg'

    with open(image_to_open, 'rb') as image_file:
        content = image_file.read()

    image = vision.Image(content=content)

    face_response = client.face_detection(image=image)

    #face_data = []

    for face_detection in face_response.face_annotations:
        d={
            'confidence': face_detection.detection_confidence,
            'joy': face_detection.joy_likelihood
        }
        print(d)


"""
Unit and regression test for the tica module.
"""

# Import package, test suite, and other packages as needed
import pytest
import torch
import numpy as np
import pandas as pd
from mlcvs.utils.io import load_dataframe
from mlcvs.tica import TICA_CV

# set global variables
torch.set_default_tensor_type(torch.DoubleTensor)
torch.set_printoptions(precision=8)

def test_tica_train_2d_model():
    """Perform TICA on 2d_model data folder."""

    # Load dataset
    data = load_dataframe("mlcvs/tests/data/2d_model/COLVAR_md", stride=50)

    X = data.filter(regex='p.')

    # Define model
    n_features = X.shape[1]
    tica = TICA_CV(n_features)

    # Fit TICA
    tica.fit(data, lag=10)

    # Project
    y_test = tica(X.iloc[0])

    # assert
    print(X.iloc[0])
    print(y_test)
    y_test_expected = torch.tensor(
                        [-0.83703607,  0.90135466]
                      )

    assert torch.abs(y_test_expected - y_test).sum() < 1e-6


import argparse, pyramda, logging, sys, os
from dotenv import load_dotenv, find_dotenv
from app import actions

load_dotenv(find_dotenv())
logging.basicConfig( level=getattr(logging, os.environ.get( 'LOG_LEVEL' ) or 'DEBUG' ) )

def run( ):
    parser = argparse.ArgumentParser()

    parser.add_argument( 'action', type = str )
    parser.add_argument( 'resource', type = str )

    args = parser.parse_args( )

    getattr(actions, "%s_%s" % ( args.action, args.resource ) )( )

run()

# with open('db/json/favorite_playlists.json', 'r') as favorite_playlists_json:
#     playlists = json.loads(favorite_playlists_json.read())
#     # for playlist in playlists:
#     print playlists[0]['playlist']['permalink_url']


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#pd.set_option('display.expand_frame_repr', False)
#pd.set_option('display.max_rows', None)
#I am using python 3.9 IDLE so these are needed to show all columns and rows instead of only 5

# 2) Read the dataset located here 'https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data'
data = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data', header=None)

# 3) Assign new headers to the DataFrame
data.columns = ['Sample code number', 'Clump Thickness', 'Uniformity of Cell Size', 'Uniformity of Cell Shape',
                'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bare Nuclei', 'Bland Chromatin',
                'Normal Nucleoli', 'Mitoses','Class']

# 4) Drop the 'Sample code number' attribute
data = data.drop(['Sample code number'],axis=1)

### Missing Values ###

# 5)Convert the '?' to NaN
data = data.replace('?', np.nan)

# 6) Count the number of missing values in each attribute of the data.
print('Number of missing values:')
for col in data.columns:
    print('\t%s: %d' % (col,data[col].isna().sum()))

# 7) Discard the data points that contain missing values
data = data.dropna()

### Outliers ###

# 8)Draw a boxplot to identify the columns in the table that contain outliers

data['Bare Nuclei'] = data['Bare Nuclei'].astype(str).astype(int)
#need to convert Bare Nuclei column to int in order to draw the boxplot
plot1 = plt.figure(1)
data.boxplot(column = ['Clump Thickness', 'Uniformity of Cell Size', 'Uniformity of Cell Shape',
                'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bare Nuclei', 'Bland Chromatin',
               'Normal Nucleoli', 'Mitoses','Class'])

#The attributes with outliers are: 'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bland Chromatin',
#'Normal Nucleoli', 'Mitoses'

### Duplicate Data ###

# 9) Check for duplicate instances.
dups = data.duplicated()
print('Number of duplicate rows = %d' % (dups.sum()))

# 10) Drop row duplicates
print('Number of rows before discarding duplicates = %d' % (data.shape[0]))
data = data.drop_duplicates()
print('Number of rows after discarding duplicates = %d' % (data.shape[0]))

### Discretization ###

# 11) Plot a 10-bin histogram of the attribute values 'Clump Thickness' distribution
plot2 = plt.figure(2)
data['Clump Thickness'].hist(bins=10)
plt.show()

# 12)Discretize the Clump Thickness' attribute into 4 bins of equal width.
data['Clump Thickness'] = pd.cut(data['Clump Thickness'], 4)
data['Clump Thickness'].value_counts(sort=False)

#print(data['Clump Thickness'].value_counts(sort=False))
#Range of Values and number of records of each category:
#(0.991, 3.25]    131
#(3.25, 5.5]      140
#(5.5, 7.75]       52
#(7.75, 10.0]     126

### Sampling ###
# 13) Randomly select 1% of the data without replacement. The random_state argument of the function specifies the seed value of the random number generator.
sample = data.sample(frac=0.01, replace=False, random_state=1)
sample
#print(sample)


from django import forms
from . import models

class CreateArticle(forms.ModelForm):
    class Meta:
        model = models.Article
        fields = ['title','body','slug','thumb','publish']

class AddComment(forms.ModelForm):
    def __init__( self, *args, **kwargs ):
        super(AddComment, self).__init__( *args, **kwargs )
        self.fields['body'].label = ''

    class Meta:
        model = models.Comment
        fields = ['body']


import random
from random import choice
from string import ascii_lowercase, digits
import logging
import json
import time

from django.contrib.auth.models import User
from django.db import transaction

from django.shortcuts import render
from django.http import HttpResponse, HttpResponseServerError, JsonResponse, HttpResponseRedirect
from django.views.decorators.csrf import csrf_exempt
from polls.tasks import task_process_notification, task_subscribe, task_send_welcome_email, \
    task_transaction_test, task_add_subscribe, task_sync_user
from polls.forms import SubscribeForm

import requests

logger = logging.getLogger(__name__)

def random_username():
    username = ''.join([choice(ascii_lowercase) for i in range(5)])
    return username

@csrf_exempt
def webhook_test(request):
    if random.choice([0, 1]):
        # mimic the error
        raise Exception()

    # this would block the I/O
    requests.post('https://httpbin.org/delay/5')
    return HttpResponse('pong')

@csrf_exempt
def webhook_test2(request):
    """
    Use celery worker to handle the notification
    """
    task = task_process_notification.delay()
    logger.info(task.id)
    return HttpResponse('pong')

def subscribe_user(email):
    # code here can let us test what happen if the api call fail
    if random.choice([0, 1]):
        raise Exception('random processing error')

    # here we call 3-party api to subscribe our user
    requests.post('https://httpbin.org/delay/5')

def subscribe(request):
    if request.method == 'POST':
        form = SubscribeForm(request.POST)
        if form.is_valid():
            subscribe_user(form.cleaned_data['email'])
            return HttpResponse('thanks')
    else:
        form = SubscribeForm()

    return render(request, 'subscribe.html', {'form': form})

def subscribe2(request):
    if request.is_ajax() and request.method == "POST":
        raw_json = json.dumps(request.POST)
        task = task_subscribe.delay(raw_json)

        # return task_id so the js can poll the state
        return JsonResponse({
            'task_id': task.task_id,
        })
    else:
        form = SubscribeForm()
        return render(request, 'subscribe2.html', {'form': form})

def task_status(request):
    from celery.result import AsyncResult

    task_id = request.GET.get('task_id')
    if task_id:
        task = AsyncResult(task_id)
        if task.state == 'FAILURE':
            error = str(task.result)
            response = {
                'state': task.state,
                'error': error,
            }
        else:
            response = {
                'state': task.state,
            }
        return JsonResponse(response)

def transaction_test(request):
    with transaction.atomic():
        user = User.objects.create_user('john1', 'lennon@thebeatles.com', 'johnpassword')
        logger.info('create user {instance.pk}'.format(instance=user))
        raise Exception('force transaction to rollback')

@transaction.atomic
def transaction_test2(request):
    user = User.objects.create_user('john2', 'lennon@thebeatles.com', 'johnpassword')
    logger.info('create user {instance.pk}'.format(instance=user))
    raise Exception('force transaction to rollback')

@transaction.atomic
def transaction_celery(request):
    """
    Why you get DoesNotExist error in Celery worker
    """
    username = random_username()
    user = User.objects.create_user(username, 'lennon@thebeatles.com', 'johnpassword')
    logger.info('create user {instance.pk}'.format(instance=user))
    task_send_welcome_email.delay(user.pk)

    time.sleep(1)
    return HttpResponse('test')

@transaction.atomic
def transaction_celery2(request):
    """
    Solve DoesNotExist error in Celery worker
    """
    username = random_username()
    user = User.objects.create_user(username, 'lennon@thebeatles.com', 'johnpassword')
    logger.info('create user {instance.pk}'.format(instance=user))
    # we register callback functions and they would be call after transaction commit
    transaction.on_commit(lambda: task_send_welcome_email.delay(user.pk))
    transaction.on_commit(lambda: task_sync_user.delay(user.pk))

    time.sleep(1)
    return HttpResponse('test')

@transaction.atomic
def transaction_celery3(request):
    """
    How to use DB transaction in Celery task
    """
    transaction.on_commit(lambda: task_transaction_test.delay())

    return HttpResponse('test')

@transaction.atomic
def subscribe3(request):
    """
    This Django view save user email to db and send task to Celery worker
    to subscribe
    """
    if request.method == 'POST':
        form = SubscribeForm(request.POST)
        if form.is_valid():
            instance, flag = User.objects.get_or_create(
                username=form.cleaned_data['name'],
                email=form.cleaned_data['email'],
            )
            transaction.on_commit(
                lambda: task_add_subscribe.delay(instance.pk)
            )
            return HttpResponseRedirect('')
    else:
        form = SubscribeForm()

    return render(request, 'subscribe.html', {'form': form})


# Leetcode 191 [Easy]
# Number of 1 Bits

"""
Write a function that takes an unsigned integer and returns the number of '1'
bits it has (also known as the Hamming weight).

Note:
 - Note that in some languages, such as Java, there is no unsigned integer
    type. In this case, the input will be given as a signed integer type. It
    should not affect your implementation, as the integer's internal binary
    representation is the same, whether it is signed or unsigned.
 - In Java, the compiler represents the signed integers using 2's complement
    notation. Therefore, in Example 3, the input represents the signed integer.
    -3.

Example 1:
Input: n = 00000000000000000000000000001011
Output: 3
"""

"""
Approach 1:
If we take the binary integer and mod the number by 2, we will get 0 or 1
depending on if the last bit is 0 or 1, respectively. Then bitshifting
the binary number by 1 and repeating until there are no values left and
returning the sum of mod by 2 == 1.
"""
# Time complexity: O(32) => O(1)
# Space complexity: O(1)
class Solution:
    def hammingWeight(self, n):
        res = 0
        while n > 0:
            res += n % 2
            n >> 1

        return res

# downside: have to look at every bit, even the 0s
# what if we have a number like 1000000000001

"""
Approach 2:
We can come up with a bit manipulation technique that allows us to run the loop
for as many number of 1 bits are in the number. Using n = n & (n - 1) basically
removes the last 1 bit from the binary number, and we can add 1 to our res count
for each loop.
"""
# Time Complexity: O(32) => O(1).
# Space complexity: O(1)
class Solution2:
    def hammingWeight(self, n):
        res = 0
        while n > 0:
            n &= (n-1)
            res += 1
        return res


class Viewer:
    from PIL import Image
    from examples.Event_camera_sim import Event_simulator
    import matplotlib.pyplot as plt
    import numpy as np
    import cv2
    import time
    event_sim=Event_simulator()
    def __init__(self,height=64,width=64):
        # self.observation=observeation
        # self.dimensions = (100, 100)
        self.height=height
        self.width=width
        self.RGB=self.np.zeros((self.height, self.width, 3),dtype=self.np.uint8)
        self.event_figure=self.plt.figure(num='event_viewer')

    def RGB_view(self,observeation):

        resized = self.Image.fromarray(observeation['0']['rgb'].astype('uint8'), 'RGBA')
        self.plt.imshow(resized)
        self.plt.show(block=False)
        self.plt.pause(0.01)

    def Event_view(self,pos, neg,event_frame):
        # self.plt.figure(num="event viewer")
        img = self.np.zeros((self.height,self.width))
        # img = img + pos * 0.5 - neg * 0.5
        # img =  pos * 0.5 - neg * 0.5
        img=event_frame
        # img[pos]=1
        # img[neg]=-1
        # img=img+pos*255+neg*(-255)
        img=self.cv2.resize(img, (400, 400))
        self.cv2.imshow('ev frame', img)
        self.time.sleep(0.01)

        # self.plt.imshow(img)
        # # self.plt.show(block=False)
        # self.plt.pause(0.1)

    def RGB_get(self,observeation):
        self.RGB=self.Image.fromarray(observeation['0']['rgb'].astype('uint8'), 'RGBA')
        return  self.RGB


"""
Desenvolva um programa que leia o nome, idade e sexo de 4 pessoas. No final do programa, mostre:
    — a média de idade do grupo;
    — qual é o nome do homem mais velho;
    — quantas mulheres têm menos de 20 anos.
"""

soma = 0
velho = 0
Hvelho = ''
cont = 0
for c in range(1, 5):
    print(f'{f" {c}° pessoa ":-^25}')
    nome = str(input(f'Nome: ')).strip().title()
    idade = int(input(f'Idade: '))
    sexo = str(input(f'Sexo [M/F]: ')).strip().upper()
    soma += idade
    if sexo == 'M':
        if c == 1:
            velho = idade
            Hvelho = nome
        else:
            if idade > velho:
                velho = idade
                Hvelho = nome
    elif sexo == 'F':
        if idade < 20:
            cont += 1
    else:
        print('Entrada INVALIDA! Tente novamente.')
        exit()
print('\n')
print(f'A média de idades do grupo é de {soma / 4:.1f} anos')
print(f'O homen mais velho tem {velho} anos e se chama {Hvelho}')
print(f'Ao todo são {cont} mulheres abaixo dos 20 anos')

# A estrutura de repetição acima pode ser rescrita da seguinte forma:
somaidade = 0
maioridadehomem = 0
nomevelho = ''
totmulher20 = 0
for c in range(1, 5):
    print(f'----- {c}° pessoa -----')
    nome = str(input('Nome: ')).strip().title()
    idade = int(input('Idade: '))
    sexo = str(input('Sexo [M/F}: ')).strip()
    somaidade += idade
    if c == 1 and sexo in 'Mm':
        maioridadehomem = idade
        nomevelho = nome
    if sexo in 'Mm' and idade > maioridadehomem:
        maioridadehomem = idade
        nomevelho = nome
    if sexo in 'Ff' and idade < 20:
        totmulher20 += 1
print('\n')
print(f'A média de idades do grupo é de {somaidade / 4} anos')
print(f'O homen mais velho tem {maioridadehomem} anos e se chama {nomevelho}')
print(f'Ao todo são {totmulher20} mulheres abaixo dos 20 anos')


import random

def jogar():

    print("###############################")
    print("Bem vindo ao jogo de advinhação")
    print("###############################")

    numero_secreto = random.randrange(1,101)
    total_tentativas = 3
    pontos = 100

    print("Qual nível de dificuldade?")
    print("(1) Facíl (2) Médio (3) Difícil")

    nivel = int(input("Define o nível"))

    if(nivel == 1):
        total_tentativas = 20
    elif(nivel ==2):
        total_tentativas = 10
    else:
        total_tentativas = 5

    for rodada in range(1, total_tentativas + 1):
        print("Tentativa {} de {}". format(rodada,total_tentativas))
        chute = int(input("Digite um número entre 1 e 100 numero: "))

        acertou = chute == numero_secreto
        maior = chute >= numero_secreto
        menor = chute <= numero_secreto

        if(chute<1 or chute > 100):
            print("Você deve digitar um número entre 1 e 100!")
            continue

        if(acertou):
            print("Você acertou e fez {} pontos".format(pontos))
        else:
            if(maior):
                print("Você errou, bobão cara de mamão, um numero maior que o numero secreto")
            elif(menor):
                print("Você errou, bobão cara de mamão, um numero menor que o numero secreto")
            pontos_perdidos = abs(numero_secreto - chute)
            pontos = pontos - pontos_perdidos
    print("Fim do jogo")


import json
import matplotlib.pyplot as plt
import matplotlib
from matplotlib.patches import Ellipse
import numpy as np

font = {'size':18}
matplotlib.rc('font', **font)

are_4 = []
are_6 = []
are_8 = []
are_10 = []
are_12 = []
are_14 = []
f1score_4 = []
f1score_6 = []
f1score_8 = []
f1score_10 = []
f1score_12 = []
f1score_14 = []

阈值 = "100"

def 提取(gamma, 数据包数, 平均相对误差, f1score):
	文件名 = ".".join(["ehf", "gamma", str(gamma), "npkts", str(数据包数), "caida.130000.json"])
	with open(文件名, "r") as f:
		数据 = json.load(f)
	平均相对误差.append(数据["are"][阈值])
	f1score.append(数据["f1score"][阈值])
	return 平均相对误差, f1score

if __name__ == "__main__":
	for 数据包数 in range(5, 26, 2):
		数据包数 = 数据包数*(10**6)
		are_4, f1score_4 = 提取(4, 数据包数, are_4, f1score_4)
		are_6, f1score_6 = 提取(6, 数据包数, are_6, f1score_6)
		are_8, f1score_8 = 提取(8, 数据包数, are_8, f1score_8)
		are_10, f1score_10 = 提取(10, 数据包数, are_10, f1score_10)
		are_12, f1score_12 = 提取(12, 数据包数, are_12, f1score_12)
		are_14, f1score_14 = 提取(14, 数据包数, are_14, f1score_14)
	print("are_14:")
	for 项 in are_14:
		print("%.3f" % 项)
	print("f1score_14:")
	for 项 in f1score_14:
		print("%.3f" % 项)

	数据包数 = range(5, 26, 2)
	plt.figure(1)
	plt.ylim(0, 0.5)
	plt.plot(数据包数, are_4, label=r"$\gamma=4$", marker = "x", mfc="none")
	plt.plot(数据包数, are_6, label=r"$\gamma=6$", marker = "s", mfc="none")
	plt.plot(数据包数, are_8, label=r"$\gamma=8$", marker = "o", mfc="none")
	plt.plot(数据包数, are_10, label=r"$\gamma=10$", marker = "1", mfc="none")
	plt.plot(数据包数, are_12, label=r"$\gamma=12$", marker = "p", mfc="none")
	plt.plot(数据包数, are_14, label=r"$\gamma=14$", marker = "d", mfc="none")
	plt.legend(loc = 1, ncol=2)
	plt.xlabel(r"Num. of Packets($\times 10^6$)")
	plt.ylabel("ARE")
	plt.savefig("are.pdf", bbox_inches="tight")
	plt.savefig("are.png", bbox_inches="tight")

	plt.figure(2)
	plt.ylim(0.5, 1.0)
	plt.plot(数据包数, f1score_4, label=r"$\gamma=4$", marker = "x", mfc="none")
	plt.plot(数据包数, f1score_6, label=r"$\gamma=6$", marker = "s", mfc="none")
	plt.plot(数据包数, f1score_8, label=r"$\gamma=8$", marker = "o", mfc="none")
	plt.plot(数据包数, f1score_10, label=r"$\gamma=10$", marker = "1", mfc="none")
	plt.plot(数据包数, f1score_12, label=r"$\gamma=12$", marker = "p", mfc="none")
	plt.plot(数据包数, f1score_14, label=r"$\gamma=14$", marker = "d", mfc="none")
	plt.legend(loc = 4, ncol=2)
	plt.xlabel(r"Num. of Packets($\times 10^6$)")
	plt.ylabel("F1 Score")
	plt.savefig("f1score.pdf", bbox_inches="tight")
	plt.savefig("f1score.png", bbox_inches="tight")


from helper import Helper
from crawler import crawler
from bs4 import BeautifulSoup
from DbOps import DbOperations
import re
import hashlib

class xinyiglass(object):
    def __init__(self,url,body=None):
        self.url = url
        self.body = body
    def crawler_news(self):
        loop = True
        page = 7
        while loop:
            response = crawler.MakeRequest(self.url.format(page=page),'Get',postData=self.body)
            soup = BeautifulSoup(response.content, 'html.parser')
            bulk_obj = DbOperations.Get_object_for_bulkop(False,'xinyiglass_news')
            news_list = soup.find('div',{'class':'NewsList'})
            if news_list:
                news_data = news_list.find_all('li')
                if news_data:
                    for news in news_data:
                        news_dict = Helper.get_news_dict()

                        title_data = news.find('div',{'class':'title'})
                        title = title_data.text if title_data else ""

                        url_data = news.find('a', {'href': True})
                        url = "https://www.xinyiglass.com/"+str(url_data['href']) if url_data else ''

                        regex = re.compile(r'[\n\r\t]')
                        description_data = news.find('div',{'class':'info'})
                        description = regex.sub("", description_data.text) if description_data else ''

                        date = news.find('span')
                        year_month = news.find('em')
                        publish_date = Helper.parse_date((year_month.text)+"-"+str(date.text)) if date and year_month else ''

                        news_dict.update(
                            {"title": title, "news_title_uid": hashlib.md5(title.encode()).hexdigest(),
                             "url": url, "link": url, "news_url_uid": hashlib.md5(url.encode()).hexdigest(),
                             "description": description.strip(), "text": description.strip(),
                             "publishedAt": publish_date, 'date': publish_date, "publishedAt_scrapped": publish_date,
                             "company_id": "xinyiglass", "ticker": "xinyiglass_scrapped", "industry_name": "xinyiglass",
                             "news_provider": "xinyiglass" })

                        bulk_obj.insert(news_dict)

                        if len(bulk_obj._BulkOperationBuilder__bulk.__dict__['ops']) >100:
                            bulk_obj.execute()
                            bulk_obj = DbOperations.Get_object_for_bulkop(False,'xinyiglass_news')

                    if len(bulk_obj._BulkOperationBuilder__bulk.__dict__['ops']) > 0:
                        bulk_obj.execute()

                    page += 1
                else:
                    print("News Not Found")
                    loop = False
            else:
                print("News Not Found")
                loop = False

url = "https://www.xinyiglass.com/en/companynews/list.aspx?page={page}"
news_obj = xinyiglass(url)
news_obj.crawler_news()


from datasets import GeneratorBasedBuilder, BuilderConfig, Version, DatasetInfo, Features, Value, \
    Sequence, ClassLabel, DownloadManager, SplitGenerator, Split
import os
import textwrap

_CITATION = """
@inproceedings{DBLP:conf/tsd/SimonV21,
  author    = {Eszter Simon and
               No{\'{e}}mi Vad{\'{a}}sz},
  editor    = {Kamil Ekstein and
               Frantisek P{\'{a}}rtl and
               Miloslav Konop{\'{\i}}k},
  title     = {Introducing {NYTK-NerKor}, A Gold Standard {Hungarian} Named Entity
               Annotated Corpus},
  booktitle = {Text, Speech, and Dialogue - 24th International Conference, {TSD}
               2021, Olomouc, Czech Republic, September 6-9, 2021, Proceedings},
  series    = {Lecture Notes in Computer Science},
  volume    = {12848},
  pages     = {222--234},
  publisher = {Springer},
  year      = {2021},
  doi       = {10.1007/978-3-030-83527-9\_19},
}
"""
_SOURCE = "https://github.com/nytud/NYTK-NerKor/archive/36a9aefd37e1a77fb8671375def0a5ad343d5dc3.zip"
_PATHS = {
    "train": "data/train-devel-test/train/",
    "validation": "data/train-devel-test/devel/",
    "test": "data/train-devel-test/test/",
}

_SUBS = ["fiction", "legal", "news", "web", "wikipedia", "all"]

_NER = Sequence(
    ClassLabel(
        names=[
            "O",
            "B-PER",
            "I-PER",
            "B-ORG",
            "I-ORG",
            "B-LOC",
            "I-LOC",
            "B-MISC",
            "I-MISC",
        ]
    )
)

_UPOS = Sequence(
    ClassLabel(
        names=[
            "ADJ",
            "ADP",
            "ADV",
            "AUX",
            "CCONJ",
            "DET",
            "INTJ",
            "NOUN",
            "NUM",
            "PART",
            "PRON",
            "PROPN",
            "PUNCT",
            "SCONJ",
            "SYM",
            "VERB",
            "X"
        ]
    )
)

def get_repo_url(x):
    if x.startswith("https://huggingface"):
        return "/".join(x.split("/")[:6])
    return "/".join(x.split("/")[:5])

class NerKorConfig(BuilderConfig):
    """BuilderConfig for Hulu."""

    def __init__(
            self,
            text_features,
            data_url,
            data_dir,
            url,
            **kwargs,
    ):
        super(NerKorConfig, self).__init__(version=Version("1.0.0", ""), **kwargs)
        self.text_features = text_features
        self.data_url = data_url
        self.data_dir = data_dir
        self.url = url

class NerKor(GeneratorBasedBuilder):
    """NerKor datasets."""
    # FORM LEMMA UPOS XPOS FEATS CONLL:NER
    BUILDER_CONFIGS = [
        NerKorConfig(
            name=name,
            description=textwrap.dedent(""""""),
            text_features=["form", "lemma", "upos", "xpos", "feats", "ner"],
            data_url=_SOURCE,
            data_dir=f"nytud-nerkor/",
            url=get_repo_url(_SOURCE)
        ) for name in _SUBS
    ]

    def _info(self):
        features = {
            "idx": Value("int32"),
            "text": Value("string"),
            "tokens": Sequence(Value("string")),
            "lemmas": Sequence(Value("string")),
            "upos": _UPOS,
            "xpos": Sequence(Value("string")),
            "feats": Sequence(Value("string")),
            "ner": _NER,
            "file_name": Value("string"),
            "sentence_id": Value("int32"),
            "morph_tagged": Value("bool_"),
        }
        return DatasetInfo(
            description=self.config.description,
            features=Features(features),
            homepage=self.config.url,
            citation=textwrap.dedent(_CITATION),
        )

    def _split_generators(self, dl_manager: DownloadManager):
        path = dl_manager.download_and_extract(self.config.data_url)
        return [
            SplitGenerator(
                name=Split.TRAIN,
                gen_kwargs={"split_key": "train", "data_file": path},
            ),
            SplitGenerator(
                name=Split.VALIDATION,
                gen_kwargs={"split_key": "validation", "data_file": path},
            ),
            SplitGenerator(
                name=Split.TEST,
                gen_kwargs={"split_key": "test", "data_file": path},
            )
        ]

    def _generate_examples(self, data_file, split_key, **kwargs):
        if self.config.name == "all":
            base_path = os.path.join(data_file, os.listdir(data_file)[0], _PATHS[split_key])
            base_path = [os.path.join(base_path, p) for p in os.listdir(base_path)]
            train_pointers = [os.path.join(b, p) for b in base_path for p in os.listdir(b)]
        else:
            base_path = os.path.join(data_file, os.listdir(data_file)[0], _PATHS[split_key], self.config.name)
            train_pointers = [os.path.join(base_path, x) for x in os.listdir(base_path)]
        n = 0
        for annotation in train_pointers:
            for file in os.listdir(annotation):
                pointer_file_path = os.path.join(annotation, file)
                with open(pointer_file_path, mode="r") as f:
                    data_file_path = f.readlines()[0]
                for data in self._process_files(os.path.join(annotation, data_file_path), n):
                    yield data
                    n += 1

    @staticmethod
    def _process_files(data_file_path: str, n: int):
        with open(data_file_path, mode="r", encoding="utf8") as f:
            content = f.readlines()

        tokens, lemmas, upos, xpos, feats, ner = [], [], [], [], [], []

        sentence_id = 0
        for i, row in enumerate(content[1:]):
            if row == '\n':
                p = data_file_path.split("/")[-3:]
                yield n, {
                    "idx": n,
                    "text": " ".join(tokens),
                    "tokens": tokens,
                    "lemmas": lemmas,
                    "upos": upos,
                    "xpos": xpos,
                    "feats": feats,
                    "ner": ner,
                    "file_name": "/".join(p),
                    "sentence_id": sentence_id,
                    "morph_tagged": p[1] == 'morph'
                }
                tokens, lemmas, upos, xpos, feats, ner = [], [], [], [], [], []
                sentence_id += 1
                n += 1
                continue

            data = row.split("\t")
            _upos = data[2] if data[2] != "_" else -1
            _xpos = data[3] if data[3] != "_" else -1
            _feats = data[4] if data[4] != "_" else -1
            _ner = data[5] if data[5] != "_" else -1

            tokens.append(data[0])
            lemmas.append(data[1])
            upos.append(_upos)
            xpos.append(_xpos)
            feats.append(_feats)
            ner.append(_ner)


from time import time

import numpy as np
import pandas as pd

# utilities
from sklearn.grid_search import RandomizedSearchCV

# models
from sklearn.linear_model import Perceptron, LogisticRegression, BayesianRidge, SGDClassifier, \
    PassiveAggressiveClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

# more information about sklearn's incremental models
# http://scikit-learn.org/stable/auto_examples/linear_model/plot_sgd_comparison.html
# http://scikit-learn.org/stable/modules/scaling_strategies.html
# TODO add xgboost, keras
models_linear = [Perceptron, LogisticRegression, BayesianRidge, LinearSVC]
models_online = [Perceptron, MultinomialNB, SGDClassifier, PassiveAggressiveClassifier]
models_nonlinear_cheap = [DecisionTreeClassifier]
models_nonlinear_expensive = [RandomForestClassifier, SVC, GradientBoostingClassifier]

hyperparameters = {
    LogisticRegression: {'C': [1, 10, 50, 100, 1000, 10000]},
  # [{'C': [1, 10, 100, 1000], 'kernel': ['linear']},
  # {'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},]
    RandomForestClassifier: {"max_depth": [3, None],
              "max_features": [1, 3, 10],
              "min_samples_split": [1, 3, 10],
              "min_samples_leaf": [1, 3, 10],
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]},
}

# parameters
random_state = 0
random_search_iterations_max = 10

def requirements_bare_minimum(y_train):
    # by default model has to beat:
    # 1- random guessing (AUC must be higher than 0.5), and
    # 2- majority vote classification, important for imbalanced datasets, and
    # 3- cross-validation variation should not be too high (less than 0.1)

    return {'auc': 0.5,
            'accuracy': pd.Series(y_train).value_counts(normalize=True)[0],
            'accuracy_std': 0.1}

def check_requirements(model, requirements):
    if model_insights['accuracy'] > requirements['accuracy'] and \
                        model_variation < requirements['accuracy_std'] and \
                        model_insights['auc'] > requirements['auc'] and \
                        model_insights['accuracy_test'] > requirements['accuracy']:
        return True
    else:
        return False

def tune(insights, x_train, y_train, x_test, y_test, models='all', requirements=None, maximize=False):
    if requirements is None:
        requirements = requirements_bare_minimum(y_train)

    # do vanilla models satisfy the requirements?
    # assuming decision tree is the most intuitive, then logistic regression and then random forest
    # TODO: extend this to metrics other than accuracy using the confusion matrix
    for model_name in ['dt', 'lr', 'rf']:
        model_insights = insights[model_name]
        model_variation = np.std(model_insights['accuracy_folds'])

        if check_requirements(model_insights, requirements) and not maximize:
            pass
            # TODO: turn this back on
            # return model_name

    # model selection and tuning loop
    models_to_train = []

    if models == 'all':
        models_to_train += models_linear + models_nonlinear_cheap + models_nonlinear_expensive
    elif models == 'linear':
        models_to_train += models_online
    elif models_to_train == 'cheap':
        models_to_train += models_linear + models_nonlinear_cheap

    # TODO: using all of the training data, need to use less data if runtime for insights models is large (how large?)
    for model in models_to_train:
        # TODO: add the looping logic
        if model == LogisticRegression:
            number_configurations = np.prod(np.array([len(_) for _ in hyperparameters[model]]))
            random_search_iterations = np.min([random_search_iterations_max, number_configurations])
            random_search = RandomizedSearchCV(model(n_jobs=-1, random_state=random_state),
                param_distributions=hyperparameters[model], n_iter=random_search_iterations, n_jobs=-1, random_state=0)
            runtime = time()
            random_search.fit(x_train, y_train)
            runtime = time() - runtime

            info = dict()
            info['runtime'] = runtime
            # info['accuracy'] = min(scores)
            # info['accuracy_test'] = accuracy_score(y_test, y_test_predicted)
            # info['accuracy_folds'] = scores
            # info['confusion_matrix'] = confusion_matrix(y_test, y_test_predicted)
            # clf.fit(x_train, y_train)
            # fpr, tpr, _ = roc_curve(y_test, clf_predict_proba(clf, x_test))
            # info['fpr'] = fpr
            # info['tpr'] = tpr
            # info['auc'] = auc(fpr, tpr)

            return random_search

    return None


import datetime
from packages.connection import *
from app.models.WhatTable import WhatTable
from app.models.WhereTable import WhereTable
from app.controllers.WhatUploader import WhatUploader
from app.controllers.WhereUploader import WhereUploader
from app.controllers.WhatController import WhatController
from app.controllers.WhereController import WhereController
from app.controllers.ResultFile import ResultFile

def showTime():
    now = datetime.datetime.now()
    result = now.strftime("%d.%m.%Y %H:%M:%S")
    return result

def showInfo(text):
    result = showTime() + ' - ' + text
    return result

def printInfo(text):
    info_text = showInfo(text)
    print(info_text)

if(__name__ == "__main__"):
    printInfo('Программа запущена!!!')
    try:
        db.connect()

        # Удаляем таблицы, если вдруг решили поменять структуру БД
        WhatTable.drop_table()
        WhereTable.drop_table()
        # Удаляем таблицы, если вдруг решили поменять структуру БД

        # Создаём таблицы
        WhatTable.create_table()
        WhereTable.create_table()
        # Создаём таблицы

        # Очищаем таблицы
        WhatTable.truncate_table()
        WhereTable.truncate_table()
        # Очищаем таблицы
    except:
        printInfo('Ошибка!!! Не удалось подключиться к базе данных!')
        exit()

    # Загружаем данные в базу
    printInfo('Данные из xlsx таблиц загружаются в базу')
    WhatUploader().run()
    WhereUploader().run()
    printInfo('Данные из xlsx таблиц успешно загружены')
    # Загружаем данные в базу

    # Создаём объект файла с результатом
    resultObj = ResultFile()
    resultObj.headerContentPut()
    # Создаём объект файла с результатом

    # Вытаскиваем все данные из БД
    what_rows = WhatController().getData()
    count_rows = len(what_rows)
    # Вытаскиваем все данные из БД

    # Перебираем содержимое БД
    printInfo('Начинаем сравнение таблиц')
    this_row = 1
    for what_row in what_rows:
        printInfo(what_row.file_name + ': Обаботана ' + str(this_row) + ' строка из ' + str(count_rows))
        this_row+=1

        where_has = WhereController().hasSnils(what_row.snils)
        if where_has['status']:
            info = {
                "what_file": what_row.file_name,
                "what_row": what_row.num_row,
                "where_file": where_has['result'].file_name,
                "where_row": where_has['result'].num_row,
                "value": what_row.snils,
                "request_date": what_row.request_date,
                "serviceid": what_row.serviceid
            }
            resultObj.contentRowPut(info)
        else:
            info = {
                "what_file": what_row.file_name,
                "what_row": what_row.num_row,
                "where_file": 'None',
                "where_row": 'None',
                "value": what_row.snils,
                "request_date": what_row.request_date,
                "serviceid": what_row.serviceid
            }
            resultObj.contentRowPut(info)
    printInfo('Сравнение таблиц успешно завершено')

    # Сохраняем файл результата
    printInfo('Сохраняем результа в файл')
    resultObj.saveFile()
    # Сохраняем файл результата

    printInfo('Программа выполнена!!!')
    print('')
    print('***')
    print('Название программы: checkSnils v3.0')
    print('Разработка: Министерства труда и социального развития РД')
    print('Разработчик: Ахмедов Мурад Алилович')
    print('***')
    print('')
    input('Нажмите любую клавишу, для завершения работы программы')


import maya.OpenMaya as om
import maya.cmds as cmds

def zbw_FK2IKSnap(*args):
    """
    select FK wrist joint(or control), fk elbow joint (or ctrl), FK shoulder joint (or ctrl), IK wrist ctl and IK pole vector in that order
    """

    sel = cmds.ls(sl=True)
    fkEndC = cmds.listConnections((sel[0]+".fkEndCtrl"))
    fkMidC = cmds.listConnections((sel[0]+".fkMidCtrl"))
    fkTopC = cmds.listConnections((sel[0]+".fkTopCtrl"))
    ikTopJ = cmds.listConnections((sel[0]+".ikTopJnt"))
    ikMidJ = cmds.listConnections((sel[0]+".ikMidJnt"))
    ikEndJ = cmds.listConnections((sel[0]+".ikEndJnt"))

    #get FK wrist joint position & rot
    ikEndPos = cmds.xform(ikEndJ, q=True, ws=True, t=True)
    iep = om.MVector(ikEndPos[0], ikEndPos[1], ikEndPos[2])
    ikEndRot = cmds.xform(ikEndJ, q=True, ro=True)
    ier = om.MVector(ikEndRot[0], ikEndRot[1], ikEndRot[2])

    #get FK shoulder position & rot
    ikTopPos = cmds.xform(ikTopJ, q=True, ws=True, t=True)
    itp = om.MVector(ikTopPos[0], ikTopPos[1], ikTopPos[2])
    ikTopRot = cmds.xform(ikTopJ, q=True, ro=True)
    itr = om.MVector(ikTopRot[0], ikTopRot[1], ikTopRot[2])

    #get midJnt pos & rot
    ikMidPos = cmds.xform(ikMidJ, q=True, ws=True, t=True)
    imp = om.MVector(ikMidPos[0], ikMidPos[1], ikMidPos[2])
    ikMidRot = cmds.xform(ikMidJ, q=True, ro=True)
    imr = om.MVector(ikMidRot[0], ikMidRot[1], ikMidRot[2])

    #rotate joints to rotations
    cmds.xform(fkEndC, ro=(ier.x, ier.y, ier.z))
    cmds.xform(fkMidC, ro=(imr.x, imr.y, imr.z))
    cmds.xform(fkTopC, ro=(itr.x, itr.y, itr.z))

    #snap FK ctrl to positions
#-------try:except this part . . . Will only work if the channels aren't locked . . .
    cmds.move(iep.x, iep.y, iep.z, fkEndC, ws=True)
    cmds.move(imp.x, imp.y, imp.z, fkMidC, ws=True)
    cmds.move(iep.x, iep.y, iep.z, fkEndC, ws=True)


import io
import json

from fastapi import APIRouter,Depends,HTTPException,FastAPI, UploadFile, File,Body,Response
import os
from starlette.responses import FileResponse,StreamingResponse
import io
from loguru import logger
import pandas as pd
import pickle as pickle

from public_function import process_file
from public_function.response_message import Message
from sql_app import models as app_models

dirRoutes = APIRouter()

@dirRoutes.get('/dirlist/')
def dirs_list(dir_path:app_models.Path=Body(...,embed=False)):
    """获服务器文件夹目录"""
    path = dir_path.dict()['path']
    path = json.loads(path)
    if isinstance(path, str):
        if not os.path.exists(path):
            return Message.folder_not_exist(path)
        else:
            if os.path.isdir(path):
                return os.listdir(path)
            else:
                return Message.error_msg(f'{path} not a folder.')
    elif isinstance(path,(set,list)):
        return {one_path:os.listdir(one_path) if (os.path.exists(one_path)) & (os.path.isdir(one_path)) else None for one_path in path}

@dirRoutes.get('/file_exist/')
def file_exist(file_path:app_models.Path=Body(...,embed=False)):
    """判断文件是否存在"""
    path = file_path.dict()['path']
    path = json.loads(path)
    if isinstance(path,str):
        return os.path.exists(path)
    else:
        return {one_path:os.path.exists(one_path) for one_path in path}

@dirRoutes.get('/is_dir/')
def is_dir(dir_path:app_models.Path=Body(...,embed=False)):
    """获服务器文件夹目录"""
    path = dir_path.dict()['path']
    path = json.loads(path)
    if isinstance(path, str):
        if not os.path.isdir(path):
            return False
        else:
            return True
    elif isinstance(path,(set,list)):
        return {one_path:True if os.path.isdir(one_path) else False for one_path in path}

@dirRoutes.get('/make_dir/')
def make_dir(path:app_models.Path=Body(...,embed=False)):
    path = path.dict()['path']
    path = json.loads(path)
    if isinstance(path,str):
        if not os.path.exists(path):
            os.mkdir(path)
        return Message.success()
    if isinstance(path,(list,set)):
        {os.mkdir(one_path) if not os.path.exists(one_path) else None for one_path in path}
        return Message.success()

@dirRoutes.get('/download_file/')
def download_file(file_path):
    """下载文件"""
    if not os.path.exists(file_path):
        return Message.folder_not_exist(file_path)
    else:
        return FileResponse(file_path)

@dirRoutes.get('/read_file/')
def read_file(file_path,type='r'):
    """下载文件"""
    if not os.path.exists(file_path):
        return Message.folder_not_exist(file_path)
    else:
        if type == 'rb':
            return Response(process_file.read_file_as_bytes(file_path))
        elif type == 'r':
            for encodingType in ['gbk','utf-8']:
                try:
                    with open(file_path,'r+', encoding=encodingType) as f:
                        data = f.read()
                    return data
                except:
                    continue
            return Message.error_msg(f'cant read {file_path}')

@dirRoutes.get('/read_excel/')
def read_excel(path:app_models.Path=Body(...,embed=False)):
    path = path.dict()['path']
    path = json.loads(path)
    if isinstance(path, str):
        if os.path.exists(path):
            return Response(pickle.dumps(process_file.read_excel(path)))
    if isinstance(path, (list, set)):
        pathDf = {onePath:process_file.read_excel(onePath) for onePath in path}
        pathDfJson = {path:pathDf.to_json() if isinstance(pathDf,pd.DataFrame) else json.dumps({'error':'excel read error'}) for path,pathDf in pathDf.items()}
        return Response(json.dumps(pathDfJson).encode())

@dirRoutes.post('/upload_file/')
def upload_file(savePath,fileBytes:bytes=File(...)):
    saveFolder = os.path.dirname(savePath)
    if not os.path.exists(saveFolder):
        os.mkdir(saveFolder)
    with open(savePath,'wb') as f:
        f.write(fileBytes)
    logger.info(f'销售上传{os.path.basename(savePath)}')

@dirRoutes.post('/upload_folder/')
def upload_folder(folderPath,fileBytes:bytes=File(...)):
    savepath = os.path.join(os.path.dirname(folderPath),os.path.basename(folderPath)+'.zip')
    with open(savepath,'wb') as f:
        f.write(fileBytes)

@dirRoutes.get('/delete_file/')
def delete_file(path:app_models.Path=Body(...,embed=False)):
    path = path.dict()['path']
    path = json.loads(path)
    if isinstance(path, str):
        if os.path.exists(path):
            os.remove(path)
        return Message.success()
    if isinstance(path, (list, set)):
        {os.remove(one_path) if os.path.exists(one_path) else None for one_path in path}
        return Message.success()

@dirRoutes.get('/fileTime/')
def file_time(path:app_models.Path=Body(...,embed=False)):
    path = path.dict()['path']
    path = json.loads(path)
    if isinstance(path,str):
        if os.path.exists(path):
            return process_file.file_modify_time(path)
        else:
            return
    if isinstance(path,(list,set)):
        return {one_path:process_file.file_modify_time(one_path) if os.path.exists(one_path) else None for one_path in path}


import os

import jax.numpy as jnp
import numpy as np
from experiment_defs import *

def split_merge_vary_num_dataset(dir_path, group_id, configs):
    experiment_name = 'hpl_bo_split_group_id_{}_merge'.format(group_id)
    results = {}

    results['experiment_name'] = experiment_name

    # configs
    results['configs'] = configs
    results['configs']['hpl_lengthscale_mlp_features'] = HPL_LENGTHSCALE_MLP_FEATURES
    results['configs']['vary_num_dataset_method_name_list'] = VARY_NUM_DATASET_METHOD_NAME_LIST

    setup_b_id_list = configs['setup_b_id_list']
    method_name_list = configs['method_name_list']
    ac_func_type_list = configs['ac_func_type_list']
    n_bo_runs = configs['n_bo_runs']
    eval_nll_n_batches = configs['eval_nll_n_batches']

    results['setup_b'] = {}

    for num_dataset in VARY_NUM_DATASET_NUM_LIST:
        for seed in range(VARY_NUM_DATASET_NUM_SEEDS):
            extra_suffix = '_n{}seed{}'.format(num_dataset, seed)
            # setup b
            results['setup_b'][extra_suffix] = {}
            results_b = results['setup_b'][extra_suffix]

            # fit gp parameters
            fit_single_gp_list = setup_b_id_list
            results_b['fit_base_gp_params'] = {}
            for dataset_id in fit_single_gp_list:
                results_b['fit_base_gp_params'][dataset_id] = np.load(
                    os.path.join(dir_path,
                                 'split_fit_gp_params_setup_b_id_{}.npy'.format(dataset_id)), allow_pickle=True).item()

            # read fit direct hgp params
            if 'fit_direct_hgp' in VARY_NUM_DATASET_METHOD_NAME_LIST:
                results_b['fit_direct_hgp_params'] = np.load(
                    os.path.join(dir_path, 'split_fit_direct_hgp_two_step_setup_b{}.npy'.format(extra_suffix)),
                    allow_pickle=True).item()['gp_distribution_params']

            # read fit hpl hgp params
            if 'hpl_hgp_end_to_end' in VARY_NUM_DATASET_METHOD_NAME_LIST:
                results_b['hpl_hgp_end_to_end_params'] = np.load(
                    os.path.join(dir_path, 'split_fit_hpl_hgp_end_to_end_setup_b{}.npy'.format(extra_suffix)),
                    allow_pickle=True).item()
            if 'hpl_hgp_two_step' in VARY_NUM_DATASET_METHOD_NAME_LIST:
                results_b['hpl_hgp_two_step_params'] = np.load(
                    os.path.join(dir_path, 'split_fit_hpl_hgp_two_step_setup_b{}.npy'.format(extra_suffix)),
                    allow_pickle=True).item()

            # run BO and compute NLL
            results_b['bo_results'] = {}
            results_b['bo_results_total'] = {}
            results_b['nll_results'] = {}
            results_b['nll_results_total'] = {}
            results_b['kl_eval'] = {}

            for method_name in VARY_NUM_DATASET_METHOD_NAME_LIST:
                # BO results
                results_b['bo_results'][method_name] = {}
                results_b['bo_results_total'][method_name] = {}

                for ac_func_type in ac_func_type_list:
                    results_b['bo_results'][method_name][ac_func_type] = {}

                    regrets_all_list = []

                    for test_id in TEST_ID_LIST:
                        results_b['bo_results'][method_name][ac_func_type][test_id] = np.load(
                            os.path.join(dir_path,
                                         'split_test_bo_setup_b_id_{}_{}_{}{}.npy'.format(test_id, method_name,
                                                                                ac_func_type,
                                                                                extra_suffix)),
                            allow_pickle=True,
                        ).item()

                    for i in range(n_bo_runs):
                        regrets_all_list_i = []

                        for test_id in TEST_ID_LIST:
                            regrets_ij = results_b['bo_results'][method_name][ac_func_type][test_id]['regrets_list'][i]
                            regrets_all_list_i.append(regrets_ij)

                        regrets_all_list.append(jnp.mean(jnp.array(regrets_all_list_i), axis=0))

                    regrets_all_list = jnp.array(regrets_all_list)
                    regrets_mean_total = jnp.mean(regrets_all_list, axis=0)
                    regrets_std_total = jnp.std(regrets_all_list, axis=0)

                    results_b['bo_results_total'][method_name][ac_func_type] = {
                        'regrets_all_list': regrets_all_list,
                        'regrets_mean': regrets_mean_total,
                        'regrets_std': regrets_std_total,
                    }

                # KL divergence evaluation
                if method_name in EVAL_KL_METHOD_NAME_LIST:
                    results_b['kl_eval'][method_name] = np.load(os.path.join(
                        dir_path, 'split_eval_kl_with_gt_{}{}.npy'.format(method_name, extra_suffix)),
                        allow_pickle=True).item()

    # save all results
    merge_path = os.path.join(dir_path, 'merge')
    if not os.path.exists(merge_path):
        os.mkdir(merge_path)
    np.save(os.path.join(merge_path, 'results.npy'), results)

    # write part of results to text file
    with open(os.path.join(merge_path, 'results.txt'), 'w') as f:
        f.write(str(results))

    print('done.')


import model_zoo
import torch

# ======================================================================
# brief description of this experiment
# ======================================================================
# ...

# ======================================================================
# Model settings
# ======================================================================
model_handle = model_zoo.UNet
use_adaptive_batch_norm = True # If True set train_with_Bern to False

train_with_bern = False
defrozen_conv_blocks = False

# If cut_z is True, we cut the first and last 3 slices of the z axis
cut_z = False
cut_z_saved = 3
run_number = 1
da_ratio = 0.0
nchannels = 4 # [intensity, vx, vy, vz]
#note = f'_bern_{train_with_bern}_adaptive_{use_adaptive_batch_norm}'
note = f'_full_run'
extra_info = "_only_w_labels_adaptive_batch_norm_e80_lr_1e-3_AdamW_val_40"

use_saved_model = True
with_validation = True
train_file_name = 'only_w_labels_bern_images_and_labels_from_101_to_127' # bern_images_and_labels_from_101_to_116, only_w_labels_bern_images_and_labels_from_101_to_116, size_32_bern_images_and_labels_from_101_to_104
# If with_validation is True, we use the following file for validation
# else it's loaded but not used
val_file_name = 'size_40_bern_images_and_labels_from_122_to_127'
add_pixels_weight = 100
# ======================================================================
# data settings
# ======================================================================
data_mode = '3D'
image_size = [144, 112, 48] # [x, y, time]
nlabels = 2 # [background, foreground]

# ======================================================================
# training settings
# ======================================================================
#max_steps = 10000
#steps = 1000
epochs = 2
batch_size = 8
learning_rate = 1e-3
optimizer_handle = torch.optim.AdamW
betas = (0.9, 0.98)
loss_type = 'dice'  # crossentropy/dice
summary_writing_frequency = 20 # 4 times in each epoch (if n_tr_images=20, batch_size=8)
train_eval_frequency = 500 # every 4 epochs (if n_tr_images=20, batch_size=8)
val_eval_frequency = 500 # every 4 epochs (if n_tr_images=20, batch_size=8)
save_frequency = 200 # every 10 epochs (if n_tr_images=20, batch_size=8)

continue_run = False
augment_data = False
experiment_name_saved_model = 'unet3d_da_' + str(da_ratio) + 'nchannels' + str(nchannels) + '_r' + str(run_number) + '_loss_' + loss_type + '_cut_z_' + str(cut_z_saved) + note
experiment_name = 'unet3d_da_' + str(da_ratio) + 'nchannels' + str(nchannels) + '_r' + str(run_number) + '_loss_' + loss_type + '_cut_z_' + str(cut_z) + note + extra_info


import math

def func(arg1, arg2):
    """Summary line of func.

    Extended description of function. This is a Sample
    Function to show how the function documentation should be done.
    This is the numpy style documentation,
    will be formatted beautifully by sphinx-napoleon extension

    Parameters
    ----------
    arg1 : int
        Description of arg1
    arg2 : str
        Description of arg2

    Returns
    -------
    bool
        Description of return value

    """
    return True

def degcos(theta):
    """return cos(theta) where theta is in degrees

    Parameters
    ----------
    theta : float
        angle in degrees

    Returns
    -------
    float
        cosine value of the angle theta (theta is in degrees)

    """
    theta = math.radians(theta)
    return math.cos(theta)

def degsin(theta):
    """return sin(theta) where theta is in degrees

    Parameters
    ----------
    theta : float
        angle in degrees

    Returns
    -------
    float
        sine value of the angle theta (theta is in degrees)

    """
    theta = math.radians(theta)
    return math.sin(theta)

def ETradiation(daynum=None, thedate=None):
    """Calculates Extraterrestial Radiation for a specific day

    This is equation (4) from
    ASHRAE Fundamentals 2009, page 14.7.

    The equation is:

    - E0 = Esc (1 + 0.033 * cos(360 * (n - 3) / 365))
        - where n = daynum

    ETradiation takes *daynum* OR *thedate* as an input
        - daynum is the day of the year
            - (1 for January 1, 32 for February 1, etc)
        - thedate is a datetime.datetime object
            - ETradiation will calculate the daynum from the date

    Parameters
    ----------
    daynum : int
        daynum is the day of the year
    thedate : datetime.datetime()
        daynum is calculated from the datetime object

    Returns
    -------
    float
        Extraterrestial Radiation for a specific day

"""
    if daynum:
        pass
    else:
        daynum = thedate.timetuple().tm_yday
    Esc = 1367.
    E0 = Esc * (1 + 0.033 * degcos(360. * (daynum - 3) / 365.))
    return E0

def airmass(alt):
    """Calculates the airmass

    This is equation (16) from
    ASHRAE Fundamentals 2009, page 14.9.

    The equation is:

    -  m = 1 / [sin(b) + 0.50572 * power(6.07995 + b, -1.6364)]
        - where b = solar altitude

    Parameters
    ----------
    alt : float
        solar altitude

    Returns
    -------
    float
        Air Mass

"""
    b = alt
    m = 1 / (degsin(b) + 0.50572 * math.pow(6.07995 + b, -1.6364))
    return m

def tau(fhandle):
    """return tau_b and tau_d for each month of the year

    These values are in the weather file with extension .stat

    Parameters
    ----------
    fhandle : filehandle
        fhandle is a file open for reading

    Returns
    -------
    taub : list
    taud : list
        return two lists with values of taub and taud

"""
    for line in fhandle:
        line = line.strip()
        if line.startswith("taub"):
            words = line.split()
            taub = words[-12:]
            taub = [float(word) for word in taub]
            break
    for line in fhandle:
        line = line.strip()
        if line.startswith("taud"):
            words = line.split()
            taud = words[-12:]
            taud = [float(word) for word in taud]
            break
    return taub, taud

def getab(taub, taud):
    """calculates value of ab in equation (19)

    Equation (19) in ASHRAE Fundamentals 2009 pg 14.9

    Parameters
    ----------
    taub : float
        value of taub in equation (19)
    taud : float
        value of taud in equation (19)

    Returns
    -------
    float
        the value of ab

    """
    v1 = 1.219
    v2 = 0.043 * taub
    v3 = 0.151 * taud
    v4 = 0.204 * taub * taud
    return v1 - v2 - v3 - v4

def getad(taub, taud):
    """calculates value of ad in equation (20)

    Equation (20) in ASHRAE Fundamentals 2009 pg 14.9

    Parameters
    ----------
    taub : float
        value of taub in equation (20)
    taud : float
        value of taud in equation (20)

    Returns
    -------
    float
        the value of ad

    """
    # """return ad in equation 20
    # """
    v1 = 0.202
    v2 = 0.852 * taub
    v3 = 0.007 * taud
    v4 = 0.357 * taub * taud
    # print v1, v2, v3, v4
    # return v1 - v2 - v3 - v4 # typo in the equation. Example is correct.
    return v1 + v2 - v3 - v4

def directnormal_inner(E0, taub, m, ab):
    """Calculates value of Eb in equation (17)

    Equation (17) in ASHRAE Fundamentals 2009 page 14.9

    The equation is
        - Eb = E0 * exp( - taub * power(m, ab) )

    Parameters
    ----------
    E0 : float
        Extraterrestrial normal irradiance [Equation (4)]
    taub : float
        beam optical depth
    m : float
        Air Mass Equation (16)
    ab : float
        beam air mass exponents

    Returns
    -------
    float
        Beam normal irradiance

    """
    # """return the direct normal radiation
    # equation 17
    # Eb = E0 * exp( - taub * power(m, ab) )
    # """
    Eb = E0 * math.exp(-taub * math.pow(m, ab))
    return Eb

def diffhoriz_inner(E0, taud, m, ad):
    """Calculates value of Ed in equation (18)

    Equation (18) in ASHRAE Fundamentals 2009 page 14.9

    The equation is
        - Ed = E0 * exp(-taud * power(m, ad))

    Parameters
    ----------
    E0 : float
        Extraterrestrial normal irradiance [Equation (4)]
    taud : float
        diffuse optical depth
    m : float
        Air Mass Equation (16)
    ad : float
        diffuse air mass exponents

    Returns
    -------
    float
        Diffuse horizontal irradiance

    """
    # """return diffuse horizontal radiation
    # equation 18
    # Ed = E0 * exp(-taud * power(m, ad))"""
    # print math.exp(-taud * math.pow(m, ad))
    Ed = E0 * math.exp(-taud * math.pow(m, ad))
    return Ed

def directnormal(taub, taud, alt, daynum=None, thedate=None):
    """calculate direct normal irradiance from a clear sky

    - calculations for a specific day and specific sun altitude
    - taub and taud can be got from the .stat file
    - the .stat comes with the weather files (.epw, .ddy, .stat)
    - use either daynum or thedate
    - Calls directnormal_inner() that calculates equation (17)

    Parameters
    ----------
    taub : float
        Clear Sky Optical Depth for Beam Irradiance
    taud : float
        Clear Sky Optical Depth for Diffuse Irradiance
    alt : float
        Altitude of the sun
    daynum : int
        daynum is the day of the year
    thedate : datetime.datetime()
        daynum is calculated from the datetime object

    Returns
    -------
    float
        Direct Normal irradiance from a clear sky

    """
    E0 = ETradiation(daynum=daynum, thedate=thedate)
    m = airmass(alt)
    ab = getab(taub, taud)
    Eb = directnormal_inner(E0, taub, m, ab)
    return Eb

def diffusehorizontal(taub, taud, alt, daynum=None, thedate=None):
    """calculate diffuse horizontal irradiance from a clear sky

    - calculations for a specific day and specific sun altitude
    - taub and taud can be got from the .stat file
    - the .stat comes with the weather files (.epw, .ddy, .stat)
    - use either daynum or thedate
    - Calls diffhoriz_inner() that calculates equation (18)

    Parameters
    ----------
    taub : float
        Clear Sky Optical Depth for Beam Irradiance
    taud : float
        Clear Sky Optical Depth for Diffuse Irradiance
    alt : float
        Altitude of the sun
    daynum : int
        daynum is the day of the year
    thedate : datetime.datetime()
        daynum is calculated from the datetime object

    Returns
    -------
    float
        Direct Normal irradiance from a clear sky

    """
    E0 = ETradiation(daynum=daynum, thedate=thedate)
    m = airmass(alt)
    ad = getad(taub, taud)
    # print E0, taud, m, ad
    Ed = diffhoriz_inner(E0, taud, m, ad)
    return Ed


# -*- coding: utf-8 -*-
"""
Created on Sat May  8 19:19:13 2021

@author: cyclo
"""

import requests
import numpy as np
from time import sleep

nums = [i for i in range(0, 6501, 250)]
d1 = [f"{i+1:0>4}" for i in nums[0:-1]]
d2 = [f"{j:0>4}" for j in nums[1:]]
dz = [item for item in zip(d1, ["-"]*len(nums[1:]), d2)]
dirs = [d[0] + d[1] + d[2] for d in dz]
urls = [f"http://img.files-ds-scene.net/boxarts/{d}/" for d in dirs]

reqs = []
for i, u in enumerate(urls):
  rng = eval("range(" + str(int(dz[i][0])) + ", " + str(int(dz[i][2])+1) + ")")
  rng = [f"{n:0>4}.png" for n in rng]
  whole = [u + r for r in rng]
  for req in whole:
    reqs.append(req)
reqs = reqs[:6464]

saves = 0
for url in reqs:
  ID = url[48:52]
  sleep(1)
  print(f"Fetching banner from {url} now...")
  response = requests.get(f"{url}")
  if response.status_code != 200:
    if response.status_code == 404:
      print(f"File not found ({ID})")
      continue
    else:
      print(f"Something went wrong (code {response.status_code})", "Aborting!!")
      break

  print(f"Saving banner {ID} now...")
  with open(f"banners/banner_{ID}.png", "wb") as f:
    f.write(response.content)

  print("...success")
  saves += 1

print(f"Saved {saves} files")


from sqlite_helpers import SQLUtil
from pathlib import Path

def make_tables(db_path):
    sql_util = SQLUtil(db_path)
    db_tables = ['buoy_data', 'buoys', 'snow_data', 'snow_stations']
    for table in db_tables:
        sql_util.make_table(table)
        print(f'{table} table created')


import io
import unittest.mock as mock

import doxhooks.dataio as dataio
from doxhooks.errors import DoxhooksDataError, DoxhooksDataFileError
from pytest import fixture, mark

from doxhooks_pytest import withraises

class ExampleNonRestorableRepr:
    def __repr__(self):
        return "0"

class BaseTestDataIO:
    dummy_path = None

class TestLoading(BaseTestDataIO):
    @withraises
    def when_loading_literal_data(self, data_string):
        fake_input_file = io.StringIO(data_string)
        with mock.patch(
                "doxhooks.fileio.open_input", autospec=True,
                return_value=fake_input_file):
            return dataio.load_literals(self.dummy_path)

    @mark.parametrize(
        "data_string, data", [
            ("{}", {}),
        ])
    def test_literal_data_is_loaded_from_a_text_file(self, data_string, data):
        loaded_data = self.when_loading_literal_data(data_string)

        # then the data is returned.
        assert loaded_data == data

    @mark.parametrize("bad_data_string", ["{", "set()"])
    def test_loading_bad_literal_data_is_an_error(self, bad_data_string):
        self.when_loading_literal_data(
            bad_data_string, raises=DoxhooksDataFileError)

        assert self.error

class TestSaving(BaseTestDataIO):
    @fixture(autouse=True)
    def _setup_output_file(self, fake_output_file):
        self.file = fake_output_file

    @withraises
    def when_saving_literal_data(self, data):
        with mock.patch(
                "doxhooks.fileio.open_output", autospec=True,
                return_value=self.file):
            dataio.save_literals(self.dummy_path, data)

    @mark.parametrize(
        "data, data_string", [
            ({}, "{}"),
        ])
    def test_literal_data_is_saved_in_a_text_file(self, data, data_string):
        self.when_saving_literal_data(data)

        # then the data is saved.
        assert self.file.contents == data_string

    @mark.parametrize(
        "non_restorable_data", [set(), ExampleNonRestorableRepr()])
    def test_saving_non_restorable_data_is_an_error(self, non_restorable_data):
        self.when_saving_literal_data(
            non_restorable_data, raises=DoxhooksDataError)

        assert self.error


import random
import networkx as nx
import matplotlib.pyplot as plt

# Initialize the graph
G = nx.Graph()

# Initialize the arrays
a1 = [i for i in range(1, 99)]
a2 = []
a3 = ['s', 'd']
a4 = []
a5 = []
a6 = []

# Repeat until all numbers are in a6
while a1 or a2 or a3 or a4 or a5:
    # Generate two random numbers from the current array
    if a1:
        message = 'a1'
        array = a1
    elif a2:
        message = 'a2'
        array = a2
    elif a3:
        message = 'a3'
        array = a3
    elif a4:
        message = 'a4'
        plt.show()
        array = a4
    else:
        message = 'a5'
        array = a5
    if not array:
        continue
    index1 = random.randint(0, len(array) - 1)
    index2 = random.randint(0, len(array) - 1)
    if index1 == index2:
        continue
    # Add the two numbers to the graph
    num1 = array[index1]
    num2 = array[index2]
    num1_weight = random.randint(6, 9)
    num2_weight = random.randint(6, 9)
    if G.has_edge(num1, num2):
        # Skip adding the edge if it already exists
        continue
    elif (num1 == 's' and num2 == 'd') or (num1 == 'd' and num2 == 's'):
        # Skip adding the edge if it is between s and d
        continue
    # elif (num1 == 's' or num2 == 's') and G.has_node('d') and G.has_node('s'):
    #     path_length = nx.shortest_path_length(G, 's', 'd', weight='weight', method='dijkstra')
    #     if path_length < 70:
    #         continue
    # elif (num1 == 'd' or num2 == 'd') and G.has_node('s') and G.has_node('d'):
    #     path_length = nx.shortest_path_length(G, 's', 'd', weight='weight', method='dijkstra')
    #     if path_length < 70:
    #         continue
    # to Add the nodes and edges to the graph
    G.add_node(num1, connections=5, weight=num1_weight)
    G.add_node(num2, connections=5, weight=num2_weight)
    G.add_edge(num1, num2, weight=(num1_weight + num2_weight) / 2)

    # Remove the numbers from the current array and add them to the next array

    if a1:
        a2.append(num1)
        a2.append(num2)
    elif a2:
        a3.append(num1)
        a3.append(num2)
    elif a3:
        a4.append(num1)
        a4.append(num2)
    elif a4:
        a5.append(num1)
        a5.append(num2)
    else:
        a6.append(num1)
        a6.append(num2)
    array.pop(index1)
    if index1 < index2:
        index2 -= 1
    array.pop(index2)

    # print(message, array)
    # nx.draw(G, with_labels=True)
    # plt.show()
# Initialize the counts dictionary
counts = {}

# Create a list with the node colors
node_colors = []

# Iterate over the nodes of the graph
for node in G.nodes():

    if node in ['s', 'd']:
        node_colors.append('red')
    else:
        node_colors.append('lightblue')

    # Get the degree of the node
    degree = G.degree(node)

    # Increment the count for the degree
    if degree not in counts:
        counts[degree] = 0
    counts[degree] += 1
print(counts)

# Calculate all possible paths
# paths = nx.all_simple_paths(G, 's', 'd')
# print(list(paths))
# print(paths)
Shortest = nx.shortest_path(G, 's', 'd', weight='weight', method='dijkstra')
edge_list = [(Shortest[i], Shortest[i + 1]) for i in range(len(Shortest) - 1)]
layout = nx.spring_layout(G, pos={'s': (0, 0.5), 'd': (1, 0.5)}, fixed=['s', 'd'])
path_length = nx.shortest_path_length(G, 's', 'd', weight='weight', method='dijkstra')
print("path_length: ", path_length)

nx.draw(G, with_labels=True, node_color=node_colors, edge_color='lightgreen', width=1, linewidths=1, pos=layout)
nx.draw_networkx_edges(G, pos=layout, edgelist=edge_list, edge_color='r', width=2)
nx.draw_networkx_nodes(G, pos=layout, nodelist=Shortest, node_color='r')

# also draw weights of edges on the graph
edge_labels = nx.get_edge_attributes(G, 'weight')
nx.draw_networkx_edge_labels(G, pos=layout, edge_labels=edge_labels, font_size=5)
plt.show()


import tkinter as tk
from tkinter import filedialog
import customtkinter as ctk
from PIL import Image
import re
import os,glob
import numpy as NP
WIDTH = 600
HEIGHT = 800
ctk.set_appearance_mode("dark")
ctk.set_default_color_theme("dark-blue")

class referenceFinder(ctk.CTk):
    def __init__(self):
        super().__init__()

        self.filename = None
        self.rege = re.compile('<xref\swpid="[a-zA-z]\d{4,5}-\d{1,2}-\d{3,4}-\d{3}"\/>|<extref\sdocno=".*?"/>')
        self.wpp2 = re.compile('[A-z]\d{4,5}-\d{1,2}-\d{3,4}-\d{3}')
        self.title("References")
        self.geometry(f"{1100}x{800}")

        self.grid_columnconfigure(1,weight=1)
        self.grid_columnconfigure((2,3),weight=0)
        self.grid_rowconfigure((0,1,2), weight=1)

        #self.open_folder_image = ctk.CTkImage(Image.open("noun-new-folder-1965786.png"),size=(20,25))

        self.sidebar = ctk.CTkFrame(self,width=140,corner_radius=1)
        self.sidebar.grid(row=0,column=0,rowspan=4,sticky="nsew")
        self.sidebar.grid_rowconfigure(4,weight=1)

        self.textBox = ctk.CTkTextbox(self,width=700,height=550)
        self.textBox.grid(row=0,column=1,sticky="nsew",padx=20,pady=20)

        self.button1 = ctk.CTkButton(self.sidebar,text="Open WIP Folder", font =('times',13),border_width=2,border_color=("#8a7f7f"),command=self.openWIPFolder)
        self.button1.grid(row=0,column=0, padx=20,pady=60,ipady=20)
        self.radio_var = tk.IntVar(value=0)

        self.radioButton1 = ctk.CTkRadioButton(self.sidebar,font=('times',12),text="All References",variable=self.radio_var,value=0)
        self.radioButton1.grid(row=1,column=0, padx=20,pady=10,sticky="we")

        self.radioButton2 = ctk.CTkRadioButton(self.sidebar,font=('times', 12),text="Missing References",variable=self.radio_var,value=1)
        self.radioButton2.grid(row=2,column=0,padx=20,pady=10,sticky="we")

        # self.checkBox = ctk.CTkCheckBox(self.sidebar,font=('times', 12),text="Write to XML")
        # self.checkBox.grid(row=3,column=0,padx=20,pady=10,sticky="we")

        self.button2 = ctk.CTkButton(self.sidebar,text="Find",font =('times',13),border_width=2,border_color=("#8a7f7f"),command=self.findRef,state="disabled")
        self.button2.grid(row=4, column=0,padx=20,pady=60,ipady=20)

        #self.button3 = ctk.CTkButton(self.sidebar,text="Exit",font=('times',13),border_width=2,border_color=("#8a7f7f"),command=self.destroy)
        #self.button3.grid(row = 3, column = 0,padx=20,pady=60,ipady=20)
        #self.grab_set()

    def close_window(self):

        self.destroy()

    def openWIPFolder(self):

        self.filename = filedialog.askdirectory(initialdir="~",title="Select Folder")
        if self.filename:
            self.button2.configure(state="normal")
    def findRef(self):
        self.textBox.configure(state="normal")
        self.textBox.delete("1.0",tk.END)
        order = 0
        dict = {}
        matches = []
        rrrr = []
        present = []
        x = ""
        eqp = []
        # self.button3 = ctk.CTkButton(self,text="Write to XML",font =('times',13),border_width=2,border_color=("#8a7f7f"))
        # self.button3.grid(row=1,column=1,padx=20,pady=60,ipady=20,sticky="w")

        self.button4 = ctk.CTkButton(self,text="Save Output",font=('times',13),border_width=2,border_color=("#8a7f7f"),command=self.saveOutput)
        self.button4.grid(row=1,column=1,padx=20,pady=60,ipady=20,sticky="w")

        for filename1 in glob.glob(os.path.join(self.filename, '*.txt')):
            # for i, line in enumerate(open(os.path.join(os.getcwd(),filename),'r')):
            #numbers = re.search(r'(?!\\)[A-z]\d{4,5}', filename1)
            with open(filename1,'r',encoding="utf-8") as f5:
                for line in f5:
                    if '</wpidinfo>' in line:
                        break
                    wpp = re.search('[A-z]\d{4,5}-\d{1,2}-\d{3,4}-\d{3}',line)
                    if wpp:
                        wpp = wpp.group(0)
                        dict[wpp] = order
                    order += 1

        for filename1 in glob.glob(os.path.join(self.filename, '*.txt')):
            # for i, line in enumerate(open(os.path.join(os.getcwd(),filename),'r')):

            with open(filename1, 'r',encoding="utf-8") as f:
                for line in f:
                    if '<initial_setup>' in line:

                        for line3 in f:
                            text2 = line3
                            present += self.rege.findall(text2)
                            if '</ref>' in line3:
                                break
                    if '<eqpconds>' in line:
                        for line4 in f:
                            text3 = line4
                            eqp += self.rege.findall(text3)
                            if '</eqpconds>' in line4:
                                break
                    if '</initial_setup>' in line:

                        for line2 in f:
                            text = line2
                            matches += self.rege.findall(text)

            unique = set(matches)
            eqp = set(eqp)
            unique = NP.subtract(unique,eqp)

            present = set(present)
            miss = NP.subtract(unique,present)
            if self.radio_var.get() == 0:
                x = "All"

                for xx in unique:
                    reg = re.search(r'(?!\\)[A-z]\d{4,5}-\d{1,2}-\d{3,4}-\d{3}|<extref\sdocno=".*?"/>', xx)
                    if reg:
                        reg = reg.group(0)
                        rrrr.append(reg)
            elif self.radio_var.get() == 1:
                x = "Missing"
                for xx in miss:
                    reg = re.search(r'(?!\\)[A-z]\d{4,5}-\d{1,2}-\d{3,4}-\d{3}|<extref\sdocno=".*?"/>', xx)
                    if reg:
                        reg = reg.group(0)
                        rrrr.append(reg)

            try:
                rrrr = sorted(rrrr,key= lambda x: dict[x])
                #print("TRYING")
            except Exception as e:
                print("KEYERROR %s" % str(e))

            if rrrr:
                self.textBox.insert(tk.END,f'\n\n{x} REFERENCES IN: ' + os.path.basename(filename1))
            for x in rrrr:
                if 'extref' in x:
                    xmlS = f'\n<ref-setup-item>{x}</ref-setup-item>'
                else:
                    xmlS = f'\n<ref-setup-item><xref wpid="{x}"/></ref-setup-item>'

                self.textBox.insert(tk.END,xmlS)

            rrrr = []
            matches=[]
            present =[]
            eqp=[]
        self.textBox.configure(state="disabled")
    def saveOutput(self):
        output = self.textBox.get("1.0","end-1c")
        savefile = filedialog.asksaveasfilename(initialdir="/",title = "Select file",filetypes = (("txt files","*.txt"),("all files","*.*")))
        with open(savefile,'a+') as f1:
            f1.write(output)

def main():

    app =  referenceFinder()
    app.mainloop()

if __name__ == '__main__':
    main()


from collections import deque

class Graph:
    def __init__(self):
        self.edges={}
        self.neighbs={}

    def add_edge(self, a, b, x):
        if (a,b) not in self.edges:
            self.edges[(a,b)]=0
        if (b,a) not in self.edges:
            self.edges[(b,a)]=0
        self.edges[(a,b)]+=x
        if not a in self.neighbs:
            self.neighbs[a]=[]
        if not b in self.neighbs:
            self.neighbs[b]=[a]
        self.neighbs[a].append(b)
        self.neighbs[b].append(a)

    def bfs(self, x, z, parents):
        visited={}
        visited[x]=True
        queue=deque([x])
        found=False
        while len(queue)>0:
            vertex=queue.pop()
            if vertex==z:
                return True
            if not vertex in self.neighbs:
                continue
            for neighb in self.neighbs[vertex]:
                if self.edges[(vertex, neighb)]==0:
                    continue
                if neighb in visited:
                    continue
                visited[neighb]=True
                queue.append(neighb)
                parents[neighb]=vertex
        return False

    def maxflow(self, x, z):
        parents={}
        flow=0
        while self.bfs(x, z, parents):
            pathedges=[]
            pathflow=10**9
            node=z
            while node!=x:
                pathedges.append((parents[node], node))
                pathflow=min(pathflow, self.edges[(parents[node],node)])
                node=parents[node]
            flow+=pathflow
            pathedges=pathedges[::-1]
            for edge in pathedges:
                self.edges[(edge[0],edge[1])]-=1
                self.edges[(edge[1],edge[0])]+=1
##            print(pathedges)
        return flow
def pad(r):
    n=len(r)
    r_padded=["#"*(n+2)]
    for row in r:
        r_padded.append("#"+row+"#")
    r_padded.append("#"*(n+2))
    return r_padded

def count(r):
    r_padded=pad(r)
    n=len(r)
    graph=Graph()
    rowcounts=[0]*(n+1)
    colcounts=[0]*(n+1)
    for i in range(1, n+1):
        for j in range(1, n+1):
            if r_padded[i][j]=="X":
                graph.add_edge((i,0),(i,j),1)
                graph.add_edge((i,j),(0,j),1)
                rowcounts[i]+=1
                colcounts[j]+=1
    for i in range(1, n+1):
        graph.add_edge((0,0),(i,0),1)
        graph.add_edge((0,i),(n+1,n+1), 1)
##        graph.add_edge((0,0),(i,0),rowcounts[i])
##        graph.add_edge((0,i),(n+1,n+1), colcounts[i])
    return graph.maxflow((0,0),(n+1,n+1))
##    for i in range(1, n+1)

if __name__ == "__main__":
    r =["........",
        "........",
        "...X..X.",
        "........",
        "....X...",
        "..X.X..X",
        "........",
        "....X..."]
    print(count(r)) # 3


acoes = int(input())
letra = input()

pos = 0

if letra == "A":
    pos = 1
if letra == "B":
    pos = 2
if letra == "C":
    pos = 3

lista_acoes = []

for i in range(acoes):
    lista_acoes.append(int(input()))

for i in range(acoes):
    if lista_acoes[i] == 1:
        if pos == 1:
            pos = 2
        elif pos == 2:
            pos = 1
    if lista_acoes[i] == 2:
        if pos == 2:
            pos = 3
        elif pos == 3:
            pos = 2
    if lista_acoes[i] == 3:
        if pos == 3:
            pos = 1
        elif pos == 1:
            pos = 3

if pos == 1:
    print("A")
if pos == 2:
    print("B")
if pos == 3:
    print("C")


#Import Packages
import numpy as np
from PIL import Image
import torch
from torch.nn import functional as F
from torchvision import models, transforms
import torch.nn as nn
import streamlit as st
import warnings
warnings.filterwarnings('ignore')

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

icon = Image.open('icon.jpg')
st.set_page_config(page_title='Classifier', page_icon = icon)
st.header('Cat And Dog Classifier')

#Load Model
def Resnet50():
    model = models.resnet50(pretrained=False).to(device)

    model.fc = nn.Sequential(
                nn.Linear(2048, 128),
                nn.ReLU(inplace=True),
                nn.Linear(128, 2)).to(device)

    model.load_state_dict(torch.load('weights.h5' , map_location=torch.device('cpu')) )
    model.eval()
    return model

#Calculating Prediction
def Predict(img):
    Mod = Resnet50()
    pred_logits_tensor = Mod(img)

    pred_probs = F.sigmoid(pred_logits_tensor).cpu().data.numpy()
    return pred_probs

#Get Image
file_up = st.file_uploader('Upload an Image', type = "jpg")

#Normalizing
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])

#Transforming the Image
data_transform = transforms.Compose([
                transforms.Resize((224, 224)),
                transforms.ToTensor(),
                normalize
            ])

if file_up is not None:
    # display image that user uploaded
    image = Image.open(file_up)
    st.image(image, caption = 'Uploaded Image.', use_column_width = True)
    st.write("")
    st.write("Just a second ...")
    img = data_transform(image)
    img = torch.reshape(img , (1, 3, 224, 224))
    prob = Predict(img)

    if round(prob[0][0]) == 0:
        st.write("Its a Dog")
    if round(prob[0][0]) == 1:
        st.write("Its a Cat")


import random
#metin="qwertyuıopğüasdfghjklşizxcvbnmöç"
#upper=metin.upper()
#metin2=random.choices(metin,k=50)
#metin2="".join(metin2)
liste=[]
print("Çekilişi başlatmak İçin 'başlat',\n İptal Etmek İçin 'iptal' Yaz.")

while True:
  try:

    giris=input("Çekilişe katılacak kişilerin isimlerini yazın: ")
    if giris=="iptal":
      sor=input("Kişiyi mi iptal edeceksiniz (kisi),Yoksa çekilişi mi? (cekilis) \n :")
      if sor=="kisi":
        sor1=input("Kimi iptal etmek istiyorsun?\n ")
        if sor1 in liste:
          yer=liste.index(sor1)
          liste.pop(yer)
          print("Kişiyi çekilişten İptal Ettik.")
          print(liste)
        else:
          print("Böyle biri çekilişte Yok.")
          print(liste)

      elif sor=="cekilis":
        try:
          while True:
            liste.pop()
        except:
          print("Çekiliş İptal edildi.")
    elif giris=="başlat":
      sor3=input("Kazanacak kişi sayısını yazınız :")

      sor2=int(sor3)
      sonuc=random.sample(liste, sor2)

      if len(liste)<=sor2 :
       print("Kazanacak Kişi sayısı üye sayısından sayısından fazla olamaz! ")
       print("Tekrar dene!")

      else:
       print("Kazananlar:",sonuc)
    else:
      liste.append(giris)
  except:

    if len(liste)<sor2:
        print("kişi sayısından fazla değer girdin tekrar dene")
    else:
     print("hatalı yazdınız")


'''
Created on Sep 12, 2017

@author: qiankunhuang
'''

###bfs 广度优先，添加自己这一层的数值，下一层的放入queue，逐步去除

def bfs(graph, start):
    visited, queue = set(), [start]
    visited_path = []
    while queue:
        vertex = queue.pop()
        if vertex not in visited:
            visited.add(vertex)
            visited_path.append(vertex)
            queue.extend(graph[vertex])
    return visited_path

def bfs2(graph, start):
    visited = set()
    queue = [start]
    visited_path = []
    while queue:
        vertex = queue.pop(0)
        if vertex not in visited:
            visited.add(vertex)
            visited_path.append(vertex)
            for j in graph[vertex]:
                if j not in visited:
                    queue.append(j)
    return visited_path

graph = {0: {1, 4}, 1: {0, 3, 2}, 2: {1, 3}, 3: {1, 4, 2}, 4: {3, 0}}
print(bfs(graph, 0))
graph = {0: {1, 4}, 1: {0, 3, 2}, 2: {1, 3}, 3: {1, 4, 2}, 4: {3, 0}}
print(bfs(graph, 0))