from __future__ import print_function, absolute_import, division
from ..utils import check_equilibrium
import numpy as np
from scipy.linalg import inv, pinv, norm
from scipy.optimize import basinhopping, fmin
from copy import deepcopy

class PCCAPlus(object):
    def __init__(self, n_states=2, pi=None, psi=None,
                 ncv=None, optimize=True,
                 objective='crisp_metastability'):
        '''
        Parameters:
        -----------
        n_states: int
            Number of states to coarse-grain the model.
        objective_function: str
            Objective functions to minimize the PCCA+ solution. Can be one of
            the following: crispness, crisp_metastability, fuzzy_metastability.
            Additionally, a function can be passed directly.
        '''
        if n_states < 2:
            raise ValueError('''Coarse-graining with PCCA+ requires at least
                                two states for valid decomposition.''')
        self.n_states = n_states
        self.objective = objective
        self.optimize = optimize
        self.ncv = ncv
        self.psi = psi
        self.pi = pi

    def fit(self, model):
        if self.n_states > model._T.shape[0] - 1:
            raise ValueError('''Number of states to coarse-grain must be one
                                less than states in initial model.''')

        # Solve or check left (pi) and right (psi) eigenvectors
        if self.psi is None:
            if self.pi is None:
                mu, psi = model.eigenvectors('both', k=self.n_states,
                                             ncv=self.ncv)
                self.pi = mu[:, 0]
                self.psi = psi
            else:
                self.pi = check_equilibrium(self.pi, model._T.shape[0])
                self.psi = model.eigenvectors('right', k=self.n_states,
                                              ncv=self.ncv)
        else:
            assert self.psi.shape[0] == model._T.shape[0], 'Invalid right eigenvectors'
            assert self.psi.shape[1] == self.n_states, 'Invalid number of vectors'
            if self.pi is None:
                self.pi = model.equilibrium
            else:
                self.pi = check_equilibrium(self.pi, model._T.shape[0])

        # Get transition matrix to coarsen
        self.T = model.transition_matrix

        # Guess A-matrix
        index = _index_search(psi)
        A = self.psi[index, :]
        try:
            A = inv(A)
        except:
            A = pinv(A)
        A = _fill_A(A, self.psi)

        # Optimize A Matrix
        if self.optimize:
            A = self._optimize_A(A)
        self.A = _fill_A(A, self.psi)
        return self

    def transform(self, model):
        # Build new coarse-grained model
        new_model = deepcopy(model)
        new_model.n_states = self.n_states
        new_model.fuzzy_membership = self.chi = np.dot(self.psi, self.A)
        new_model.crisp_membership = np.argmax(self.chi, axis=1)

        # Estimate coarse-grained transition matrix
        w = inv(np.dot(self.chi.T, self.chi))
        a = np.dot(np.dot(self.chi.T, model._T), self.chi)
        T_coarse = np.dot(w, a)

        pi_coarse = np.dot(self.chi.T, self.pi)
        X_coarse = np.dot(np.diag(pi_coarse), T_coarse)
        T_coarse = X_coarse / X_coarse.sum(axis=1)[:, None]
        new_model._T = T_coarse
        return new_model

    def fit_transform(self, model):
        self.fit(model)
        return self.transform(model)

    def _optimize_A(self, A):
        def f(alpha):
            if self.objective == 'crispness':
                return -1 * _crispness(alpha, self.T, self.pi, self.psi)
            elif self.objective == 'roeblitz':
                return -1 * _roeblitz(alpha, self.T, self.pi, self.psi)
            elif self.objective == 'crisp_metastability':
                return -1 * _crisp_metastability(alpha, self.T, self.pi, self.psi)
            elif self.objective == 'fuzzy_metastability':
                return -1 * _fuzzy_metastability(alpha, self.T, self.pi, self.psi)

        shape = A.shape
        alpha = A.ravel()
        alpha = basinhopping(f, alpha, niter_success=1000)['x']
        alpha = fmin(f, alpha,
                     xtol=1e-4, ftol=1e-4,
                     maxfun=1e4, maxiter=1e4,
                     disp=False, full_output=False)

        if np.isneginf(f(alpha)):
            raise ValueError('Minimization has failed to find good solution')
        return alpha.reshape(shape)

def _index_search(psi):
    n_micro, n_macro = psi.shape
    index = np.zeros(n_macro, dtype=int)

    # Find the outermost point in eigenvector space
    index[0] = np.argmax([norm(psi[mi]) for mi in range(n_micro)])

    # Change orthogonal reference frame with outermost point the origin
    ortho = psi - psi[index[0], np.newaxis]

    # Gram-Schmidt orthogonalization
    for ma in range(1, n_macro):
        temp = ortho[index[ma - 1]].copy()
        for mi in range(n_micro):
            ortho[mi] -= temp * np.dot(ortho[mi], temp)
        dist = np.array([norm(ortho[mi]) for mi in range(n_micro)])
        index[ma] = np.argmax(dist)
        ortho /= dist.max()
    return index

def _fill_A(A, psi):
    n_micro, n_macro = psi.shape
    A = A.copy()

    A[1:, 0] = -1 * A[1:, 1:].sum(1)
    A[0] = -1 * np.dot(psi[:, 1:].real, A[1:]).min(0)
    return A / A[0].sum()

def _crispness(alpha, T, pi, psi):
    '''
    Parameters:
    -----------
    A: numpy.ndarray
        The assignment (or transformation) matrix which is the output of the
        constrained optimization problem.
    psi: numpy.ndarray (n_micro, n_macro)
        Right eigenvectors of the transition matrix. This should contain the
        second through n_macro eigenvectors. The first eigenvector should be
        excluded as these are normalized to one.
    Returns:
    --------
    objective: float
        The objective function
    References:
    -----------
    ..  Kube S.,  "Statistical Error Estimation and Grid-free
    Hierarchical Refinement in Conformation Dynamics," Doctoral Thesis.
    2008
    '''
    n_micro, n_macro = psi.shape
    A = alpha.reshape(n_macro, n_macro)
    A = _fill_A(A, psi)

    lhs = 1 - A[0, 1:].sum()
    rhs = np.dot(psi[:, 1:], A[1:, 0])
    rhs = -rhs.min()

    if abs(lhs - rhs) > 1e-8:
        return -np.inf
    score = np.trace(np.dot(np.diag(1 / A[0]), np.dot(A.T, A)))
    return score

def _crisp_metastability(alpha, T, pi, psi):
    n_micro, n_macro = psi.shape
    A = alpha.reshape(n_macro, n_macro)
    A = _fill_A(A, psi)

    fuzzy_membership = np.dot(psi, A)
    crisp_membership = np.argmax(fuzzy_membership, axis=1)
    chi = 0.0 * fuzzy_membership
    chi[np.arange(n_micro), crisp_membership] = 1

    lhs = 1 - A[0, 1:].sum()
    rhs = np.dot(psi[:, 1:], A[1:, 0])
    rhs = -rhs.min()
    if abs(lhs - rhs) > 1e-8:
        return -np.inf

    score = 0.0
    for ma in range(n_macro):
        score += np.dot(T.dot(chi[:, ma]),
                        pi * chi[:, ma]) / np.dot(chi[:, ma], pi)
    return score

def _fuzzy_metastability(alpha, T, pi, psi):
    n_micro, n_macro = psi.shape
    A = alpha.reshape(n_macro, n_macro)
    A = _fill_A(A, psi)
    chi = np.dot(psi, A)

    lhs = 1 - A[0, 1:].sum()
    rhs = np.dot(psi[:, 1:], A[1:, 0])
    rhs = -rhs.min()
    if abs(lhs - rhs) > 1e-8:
        return -np.inf

    score = 0.0
    for ma in range(n_macro):
        score += np.dot(T.dot(chi[:, ma]),
                        pi * chi[:, ma]) / np.dot(chi[:, ma], pi)
    return score

def _roeblitz(alpha, T, pi, psi):
    n_micro, n_macro = psi.shape
    A = alpha.reshape(n_macro, n_macro)
    A = _fill_A(A, psi)

    score = 0
    for i in range(n_macro):
        for j in range(n_macro):
            score += A[j, i] ** 2 / A[0, i]
    return score


import os
import cv2
import matplotlib.pyplot as plt
import numpy as np

# Set the input and output directories
input_dir = "image-processing-files/test_images/"
output_dir = "image-processing-files/results/"

for filename in os.listdir(input_dir):
    # Check if the file is an image
    if filename.endswith(".jpg"):
        image_path = os.path.join(input_dir, filename)
        img = cv2.imread(image_path)

        # Image processing

        processed_img = img

        ###INPAINTING
        # Using (similar) coordinates to inpaint
        mask = np.zeros_like(img[:,:,0])
        centre = (188, 212)
        radius = 22
        cv2.circle(mask, centre, radius, 255, -1)

        # Similar section
        section = img[120:160, 190:230]

        # Inpaint the corrupted region in the color image using the extracted section
        inpainted_color = cv2.inpaint(img, mask, 3, cv2.INPAINT_TELEA)

        # Resize the section to match the size of the inpainted region
        section_resized = cv2.resize(section, inpainted_color.shape[:2][::-1], interpolation=cv2.INTER_LANCZOS4)

        # Replace the inpainted region with the extracted section
        processed_img = inpainted_color.copy()
        processed_img[mask != 0] = section_resized[mask != 0]

        # Dewarping and removing borders
        gray = cv2.cvtColor(processed_img, cv2.COLOR_BGR2GRAY)
        ret, thresh = cv2.threshold(gray, 10, 255, 0)

        contours, _ = cv2.findContours(thresh, cv2.RETR_TREE,
                               cv2.CHAIN_APPROX_SIMPLE)

        # take the first contour
        cnt = contours[0]
        #https://www.geeksforgeeks.org/finding-minimum-enclosing-rectangle-in-opencv-python/
        rect = cv2.minAreaRect(cnt)
        box = cv2.boxPoints(rect)
        box = np.int0(box)
        box_img = cv2.drawContours(processed_img, [box], 0, (0, 0, 255), 2)

        extent = np.float32([[0,0], [255,0], [255,255], [0,255]])
        bounds = np.float32(box)

        M = cv2.getPerspectiveTransform(bounds, extent)

        # Apply the transformation to the image
        processed_img = cv2.warpPerspective(processed_img, M, (256, 256))

        #median (salt and pepper noise removal) (better after dewarping and stuff)
        # processed_img = cv2.medianBlur(processed_img, 5)
        kernel = np.ones((3,3), np.uint8)
        processed_img= cv2.dilate(processed_img, kernel, iterations=1)
        ###SHARPENING

        # Create a sharpening filter
        kernel = np.array([[0,-1,0], [-1,5,-1], [0,-1,0]])

        # Apply the sharpening filter to the blurred image
        processed_img= cv2.filter2D(processed_img, -1, kernel)
        processed_img = cv2.GaussianBlur(processed_img, (5,5), 30)

        #  #### REDUCING INTENSITY OF RED CHANNEL

        # b, g, r = cv2.split(processed_img)

        # # Reduce the intensity of the red channel
        # r = cv2.addWeighted(r, 0.8, 0, 0, 0)

        # # Merge the channels back together
        # processed_img = cv2.merge((b, g, r))

        ### Hist equalisation-  method 1- to equalise colour channels
        # using Lab (https://stackoverflow.com/questions/39308030/how-do-i-increase-the-contrast-of-an-image-in-python-opencv)
        # lab= cv2.cvtColor(processed_img, cv2.COLOR_BGR2LAB)
        # l_channel, a, b = cv2.split(lab)

        # # Applying CLAHE to L-channel
        # # feel free to try different values for the limit and grid size:
        # clahe = cv2.createCLAHE(clipLimit=1.5, tileGridSize=(8,8))
        # cl = clahe.apply(l_channel)

        # # merge the CLAHE enhanced L-channel with the a and b channel
        # limg = cv2.merge((cl,a,b))

        # # Converting image from LAB Color model to BGR color spcae
        # processed_img = cv2.cvtColor(limg, cv2.COLOR_LAB2BGR)

        ####Hist Equalisation- method 2- to equalise colour channels
        # img_yuv = cv2.cvtColor(processed_img, cv2.COLOR_RGB2YUV)

        # # Apply histogram equalization to the Y channel
        # img_yuv[:,:,0] = cv2.equalizeHist(img_yuv[:,:,0])

        # # Convert the image back to the original color space
        # processed_img = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2BGR)
        # processed_img = cv2.cvtColor(processed_img, cv2.COLOR_BGR2RGB)
        # # Exponential transform to increase contrast
        # gamma = 2.0
        # inv_gamma = 1.0 / gamma
        # table = np.array([((i / 255.0) ** inv_gamma) * 255 for i in np.arange(0, 256)]).astype('uint8')
        # # Apply the exponential transform to the image
        # processed_img = cv2.LUT(processed_img, table)

        ### BRIGHTNESS & CONTRAST

        # # Define the brightness and contrast adjustments
        # alpha = 2 # Contrast control (1.0-3.0)
        # beta = 50 # Brightness control (0-100)

        # # Apply the brightness and contrast adjustments
        # processed_img = cv2.convertScaleAbs(processed_img, alpha=alpha, beta=beta)

        ### Brightness and Contrast- but specific to image mean and std deviation
        # target_mean = 100
        # target_sd = 20
        # gray = cv2.cvtColor(processed_img, cv2.COLOR_BGR2GRAY)
        # mean_pixel = np.mean(gray)
        # sd_pixel = np.std(gray)
        # scale_factor = mean_pixel/target_mean
        # #scale_factor = target_sd / sd_pixel
        # difference = (target_mean- mean_pixel)*scale_factor
        # # Adjust the contrast and brightness
        # processed_img = cv2.convertScaleAbs(processed_img, alpha=scale_factor, beta=difference)

        # Save the processed image to the output directory
        output_path = os.path.join(output_dir, filename)
        cv2.imwrite(output_path, processed_img)

        print(f"Processed {filename}")


from bokeh.io import save, output_file
from bokeh.models.widgets import DataTable, TableColumn
from bokeh.models import ColumnDataSource
from bokeh.layouts import widgetbox
import pandas as pd
import modules.channel_module as channel_module

channel_list_file_path = "./list.txt"
channel_list = channel_module.get_channel_list(channel_list_file_path)

output_file("make_table_posting_interval.html")
df = pd.read_csv("./output/posting_interval.csv")
source = ColumnDataSource(df)

columns = [
    TableColumn(field="Channel", title="Channel"),
    TableColumn(field="ViewCount", title="ViewCount"),
    TableColumn(field="SubscriberCount", title="SubscriberCount"),
    TableColumn(field="VideCount", title="VideCount"),
    TableColumn(field="PostingInterval", title="PostingInterval"),
]

data_table = DataTable(
    source=source, columns=columns, fit_columns=True, width=1300, height=800
)
save(widgetbox([data_table], sizing_mode="scale_both"))


import re
import traceback
from io import BytesIO

import xlrd
from actions.backoffice.race.utils import get_menu
from pymongo import UpdateOne
from tornado.web import url
from xlsxwriter import Workbook

from commons.page_utils import Paging
from db import STATUS_UNIT_MANAGER_ACTIVE, STATUS_UNIT_MANAGER_INACTIVE
from db.models import Company
from enums import PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT
from logger import log_utils
from web import BaseHandler, decorators, datetime

logger = log_utils.get_logging()

class UnitManagerListViewHandler(BaseHandler):
    """
    单位管理的列表页
    """

    @decorators.render_template('backoffice/race/units_manager/list_view.html')
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def get(self):
        race_cid = self.get_argument('race_cid', '')
        menu_list = await get_menu(self, 'config', race_cid)
        query_params = {'record_flag': 1, 'race_cid': race_cid}
        title = self.get_argument('title', '')
        if title:
            query_params['title'] = {'$regex': title, '$options': 'i'}
        code = self.get_argument('code', '')
        if code:
            query_params['code'] = {'$regex': code, '$options': 'i'}

        per_page_quantity = int(self.get_argument('per_page_quantity', 10))
        to_page_num = int(self.get_argument('page', 1))
        page_url = '%s?race_cid=%s&page=$page&per_page_quantity=%s&title=%s&code=%s' % (
            self.reverse_url("backoffice_race_unit_manager_list"), race_cid, per_page_quantity, title, code)
        paging = Paging(page_url, Company, current_page=to_page_num,
                        items_per_page=per_page_quantity,
                        sort=['created_dt'], **query_params)
        await paging.pager()
        return locals()

class UnitManagerAddViewHandler(BaseHandler):
    """
    新增单位
    """

    @decorators.render_template('backoffice/race/units_manager/add_view.html')
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def get(self):
        race_cid = self.get_argument('race_cid', '')
        menu_list = await get_menu(self, 'config', race_cid)
        return locals()

    @decorators.render_json
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def post(self):
        r_dict = {'code': 0}
        race_cid = self.get_argument('race_cid', '')
        unit_id = None
        try:
            title = self.get_argument('title', None)
            status = self.get_argument('status', None)  # 状态
            code = self.get_argument('code', None)
            if title and code:
                unit_count = await Company.count(dict(record_flag=1, code=code, race_cid=race_cid))
                if unit_count > 0:
                    r_dict['code'] = -3
                else:
                    if status == 'on':
                        status = STATUS_UNIT_MANAGER_ACTIVE
                    else:
                        status = STATUS_UNIT_MANAGER_INACTIVE
                    unit = Company(title=title, code=code)
                    unit.race_cid = race_cid
                    unit.status = status
                    unit.created_dt = datetime.datetime.now()
                    unit.updated_id = self.current_user.oid
                    unit_id = await unit.save()
                    r_dict['code'] = 1
            else:
                if not title:
                    r_dict['code'] = -1
                if not code:
                    r_dict['code'] = -2
        except Exception:
            #  如果因为网络问题等其他问题导致前端展示添加不正确但是数据已经保存到数据库了,应该删除掉
            if unit_id:
                await Company.delete_by_ids([unit_id])
            logger.error(traceback.format_exc())
        return r_dict

class UnitManagerEditViewHandler(BaseHandler):
    """
    单位编辑
    """

    @decorators.render_template('backoffice/race/units_manager/edit_view.html')
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def get(self, unit_cid):
        race_cid = self.get_argument('race_cid', '')
        menu_list = await get_menu(self, 'config', race_cid)
        if race_cid and unit_cid:
            unit = await Company.find_one(
                filtered={'cid': unit_cid, 'record_flag': 1})
        return locals()

    @decorators.render_json
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def post(self, unit_cid):
        race_cid = self.get_argument('race_cid', '')
        r_dict = {'code': 0}
        try:
            if race_cid and unit_cid:
                unit = await Company.find_one(filtered={'cid': unit_cid, 'record_flag': 1})
                title = self.get_argument('title', None)
                status = self.get_argument('status', None)  # 状态
                code = self.get_argument('code', None)
                if unit and title and code:
                    unit_count = await Company.count(dict(code=code, race_cid=race_cid, cid={'$ne': unit_cid}))
                    if unit_count > 0:
                        r_dict['code'] = -3
                    else:
                        if status == 'on':
                            status = STATUS_UNIT_MANAGER_ACTIVE
                        else:
                            status = STATUS_UNIT_MANAGER_INACTIVE
                        unit.title = title
                        unit.code = code
                        unit.status = status
                        unit.updated_dt = datetime.datetime.now()
                        unit.updated_id = self.current_user.oid
                        await unit.save()
                        r_dict['code'] = 1
                else:
                    if not title:
                        r_dict['code'] = -1
                    if not code:
                        r_dict['code'] = -2
        except Exception:
            logger.error(traceback.format_exc())
        return r_dict

class UnitManagerStatusSwitchViewHandler(BaseHandler):
    """
    改变单位状态
    """

    @decorators.render_json
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def post(self, unit_cid):
        race_cid = self.get_argument('race_cid', '')
        r_dict = {'code': 0}
        try:
            if race_cid and unit_cid:
                status = self.get_argument('status', False)
                if status == 'true':
                    status = STATUS_UNIT_MANAGER_ACTIVE
                else:
                    status = STATUS_UNIT_MANAGER_INACTIVE
                unit = await Company.find_one(
                    filtered={'cid': unit_cid, 'record_flag': 1})
                if unit:
                    unit.status = status
                    unit.updated_dt = datetime.datetime.now()
                    unit.updated_id = self.current_user.oid
                    await unit.save()
                r_dict['code'] = 1
        except Exception:
            logger.error(traceback.format_exc())
        return r_dict

class UnitManagerBatchOperateViewHandler(BaseHandler):
    """
    批量改变单位状态
    """

    @decorators.render_json
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def post(self):
        race_cid = self.get_argument('race_cid', '')
        r_dict = {'code': 0}
        try:
            unit_cid_list = self.get_body_arguments('unit_cid_list[]', [])
            if race_cid and unit_cid_list:
                operate = self.get_argument('operate', None)
                if operate is not None:
                    if int(operate) == 1:
                        update_requests = []
                        for unit_cid in unit_cid_list:
                            update_requests.append(UpdateOne({'cid': unit_cid},
                                                             {'$set': {'status': STATUS_UNIT_MANAGER_ACTIVE,
                                                                       'updated_dt': datetime.datetime.now(),
                                                                       'updated_id': self.current_user.oid}}))
                        await Company.update_many(update_requests)
                    elif int(operate) == 0:
                        update_requests = []
                        for unit_cid in unit_cid_list:
                            update_requests.append(UpdateOne({'cid': unit_cid},
                                                             {'$set': {
                                                                 'status': STATUS_UNIT_MANAGER_INACTIVE,
                                                                 'updated_dt': datetime.datetime.now(),
                                                                 'updated_id': self.current_user.oid}}))
                        await Company.update_many(update_requests)
                    elif int(operate) == -1:
                        await Company.delete_many({'cid': {'$in': unit_cid_list}})
                    r_dict['code'] = 1
        except Exception:
            logger.error(traceback.format_exc())
        return r_dict

class UnitManagerDeleteViewHandler(BaseHandler):
    """
    删除单位
    """

    @decorators.render_json
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def post(self, unit_cid):
        race_cid = self.get_argument('race_cid', '')
        r_dict = {'code': 0}
        try:
            if race_cid and unit_cid:
                unit = await Company.find_one(
                    filtered={'cid': unit_cid, 'record_flag': 1})
                await unit.delete()
                r_dict['code'] = 1
        except Exception:
            logger.error(traceback.format_exc())
        return r_dict

class UnitImportViewHandler(BaseHandler):
    """
    单位导入
    """

    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    @decorators.render_template('backoffice/race/units_manager/upload_view.html')
    async def get(self):
        race_cid = self.get_argument('race_cid', '')
        menu_list = await get_menu(self, 'config', race_cid)
        return locals()

    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    @decorators.render_json
    async def post(self):
        r_dict = {'code': 0}
        race_cid = self.get_argument('race_cid', '')
        try:
            if race_cid and self.request.files:
                import_files_meta = self.request.files['file']
                if import_files_meta:
                    code = await self.__subject_import_excel(race_cid, import_files_meta[0]['body'])
                    r_dict['code'] = code
        except RuntimeError:
            logger.error(traceback.format_exc())
        return r_dict

    async def __subject_import_excel(self, race_cid, excel_file_content):
        result_code = 1
        #  所有单位的编号
        unit_code_list = await Company.distinct('code', {'race_cid': race_cid, 'record_flag': 1})
        book = xlrd.open_workbook(file_contents=excel_file_content)
        #  获得第一个表的信息
        sheet = book.sheet_by_index(0)
        unit_list = []
        row_list = []

        title_list = []
        for ind, col in enumerate(sheet.row_values(0)):
            if col:
                title_list.append(col)
        # 判断表头是否正确
        if len(title_list) != 2:
            result_code = 2
            return result_code
        # 拿到所有的行数据
        for rownum in range(1, sheet.nrows):
            row_list.append([col for col in sheet.row_values(rownum)])
        #  上传文件中有一些已经存在过的，需要删除，重新覆盖
        delete_unit_oid_list = []
        if row_list:
            for i, row_data in enumerate(row_list):
                code_repeat = False
                # 单位编码
                unit_code = str(self.__get_replace_data(row_data[0], 2))
                #  单位编码不能超过16位
                if not unit_code or len(unit_code) > 16:
                    continue
                else:
                    reg = re.compile(r'^[a-zA-Z0-9]*$')
                    if not bool(reg.match(unit_code)):
                        continue
                    if unit_code in unit_code_list:
                        unit = await Company.find_one(dict(code=unit_code))
                        #  上传文件中有编码重复的行，只添加一个
                        if unit:
                            delete_unit_oid_list.append(unit.oid)
                            await unit.delete()
                        else:
                            code_repeat = True
                    unit = Company()
                    unit.race_cid = race_cid
                    unit.code = unit_code
                    unit.title = str(row_data[1])[:-2] if not isinstance(row_data[1], str) else row_data[1]
                    unit.status = STATUS_UNIT_MANAGER_ACTIVE
                    unit_code_list.append(unit_code)
                if not code_repeat:
                    unit_list.append(unit)
                    if len(unit_list) == 500:
                        await  Company.insert_many(unit_list)
                        unit_list = []
            if unit_list:
                await Company.insert_many(unit_list)
                await Company.delete_by_ids(delete_unit_oid_list)
        return result_code

    def __get_replace_data(self, data, data_type, data_dict=None):
        if not data:
            return None
        if isinstance(data, float):
            data = int(data)
            data = str(data)
        if isinstance(data, str):
            data = data.replace('\n', '').replace(' ', '')
        if data_type == 1:
            if not data_dict.get(data):
                return None
            else:
                return data_dict.get(data)
        else:
            return data

class UnitDownloadTemplatetViewHandler(BaseHandler):
    @decorators.permission_required(PERMISSION_TYPE_MINIAPP_UNIT_MANAGER_MANAGEMENT)
    async def get(self):
        try:
            # 根据需求生成excel模板
            filename = 'unit_template_%s.xlsx' % datetime.datetime.now().strftime('%Y%m%d%H%M')
            sheet_name1 = '单位导入模板'
            options = {'sheet_name1': sheet_name1}
            first_column_list = self.__get_column_list()
            work_book = self.__create_common_excel(first_column_list, options=options)

            self.set_header('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
            self.set_header('Content-Disposition', 'attachment; filename=' + filename)
            self.write(work_book)
            self.finish()
        except Exception as e:
            logger.error(str(e))

    def __get_column_list(self):
        """
        拼接列名
        :return:
        """
        column_list_start = [u'单位编号', u'单位名称']
        return column_list_start

    def __create_common_excel(self, first_column_list, options={}):
        """
        生成excel 模板
        :param first_column_list: 单位列名
        :param options:
        :return:
        """
        output = BytesIO()
        workbook = Workbook(output, {'in_memory': True})
        # sheet 1名称
        worksheet_subject = workbook.add_worksheet(options['sheet_name1'] if 'sheet_name1' in options else None)
        line_index = 0  # 起始line
        first_column_num = len(first_column_list)  # 列总数
        worksheet_subject.set_column(0, first_column_num - 1, 14)
        column_format = workbook.add_format({'bold': '1', 'valign': 'vcenter', 'align': 'center',
                                             'font_name': 'Microsoft YaHei'})
        worksheet_subject.set_row(line_index, 20, column_format)
        for index, column in enumerate(first_column_list):
            worksheet_subject.write_string(line_index, index, column, column_format)
        workbook.close()
        return output.getvalue()

URL_MAPPING_LIST = [
    url(r'/backoffice/race/unit/manager/list/', UnitManagerListViewHandler,
        name='backoffice_race_unit_manager_list'),
    url(r'/backoffice/race/unit/manager/add/', UnitManagerAddViewHandler,
        name='backoffice_race_unit_manager_add'),
    url(r'/backoffice/race/unit/manager/edit/([0-9a-zA-Z_]+)/', UnitManagerEditViewHandler,
        name='backoffice_race_unit_manager_edit'),
    url(r'/backoffice/race/unit/manager/status/switch/([0-9a-zA-Z_]+)/', UnitManagerStatusSwitchViewHandler,
        name='backoffice_race_unit_manager_status_switch'),
    url(r'/backoffice/race/unit/manager/batch_operate/', UnitManagerBatchOperateViewHandler,
        name='backoffice_race_unit_manager_batch_operate'),
    url(r'/backoffice/race/unit/manager/delete/([0-9a-zA-Z_]+)/', UnitManagerDeleteViewHandler,
        name='backoffice_race_unit_manager_delete'),
    url(r'/backoffice/race/unit/manager/import/', UnitImportViewHandler,
        name='backoffice_race_unit_manager_import'),
    url(r'/backoffice/race/unit/manager/download/', UnitDownloadTemplatetViewHandler,
        name='backoffice_race_unit_manager_download'),
]


import cv2

# Load image
image_path = 'images/salt_pepper_noise.png'
image = cv2.imread(image_path)

# Gaussian blur
k = 5
blur = cv2.medianBlur(image, k)

# Show
cv2.imshow('Original', image)
cv2.imshow('Filtered', blur)
cv2.waitKey(0)
cv2.destroyAllWindows()


import numpy as np
import pandas as pd
import logging
from typing import List, Dict
from chinese_calendar.constants import holidays, workdays
from chinese_calendar.utils import is_workday, is_holiday
from src.feature_base import FeaturizerBase, WinOffsetFeaturizerBase
# from ..util.decorators import log_status

__all__ = ['DateTimeFeaturizer', 'DifferenceFeaturizer', 'RollingStatsFeaturizer']

class DateTimeFeaturizer(FeaturizerBase):
    DEFAULT_FEATURES = [
        'year',
        'index_15min',
        'day_of_year',
        'is_leap_year',
        'day_of_year_sin',
        'day_of_year_cos',
        'quarter',
        'month',
        'week',
        'day',
        'day_of_month_sin',
        'day_of_month_cos',
        'day_of_week',
        'is_weekend',
        'day_of_week_sin',
        'day_of_week_cos',
        'hour',
        'minute',
        'second',
        'second_of_day',
        'second_of_day_sin',
        'second_of_day_cos',
        'is_workday',
        'is_holiday',
        'is_special_workday',
        'is_special_holiday'
    ]

    dt_format = '%Y-%m-%d %H:%M:%S'

    def __init__(self, *, feature_col: List[str] = None, dt_col: str = None):
        """
        featurizer for making Datetime features, see class attribute DEFAULT_FEATURES for more information.

        Args:
            feature_col: the list of datetime features interested, must in DEFAULT_FEATURES,
                         default using all DEFAULT_FEATURES
            dt_col: datetime column, default using the index of the input dataframe
        """
        super().__init__()

        if feature_col is None:
            self.feature_cols = self.DEFAULT_FEATURES
        else:
            self.feature_cols = list(feature_col)

        self.dt_col = dt_col

#     @log_status()
    def transform(self, df: pd.DataFrame):
        """
        Transform input df to df with specified datetime features.

        Caution:
        1. The input df should have index that is a pandas datetime index, if not, should specify dt_col.
        2. The time gap between two samples should be 15mins.
        3. The output df will have a datetime index using original df.index or dt_col

        Args:
            df: input df

        Returns: df with Datetime features

        """
        if self.dt_col is None:
            dt_col = df.index
        else:
            dt_col = df[self.dt_col]

        dt_col = pd.to_datetime(dt_col, format=self.dt_format)
        dt_col = dt_col.to_series()
        out_df = pd.DataFrame(index=dt_col)

        year = dt_col.dt.year
        doy = dt_col.dt.dayofyear
        leap = np.asarray(dt_col.dt.is_leap_year, dtype=int)
        diy = 365 + leap
        day = dt_col.dt.day
        dim = dt_col.dt.days_in_month
        dow = dt_col.dt.dayofweek
        doy_angular = 2 * np.pi * doy / diy
        day_angular = 2 * np.pi * day / dim
        dow_angular = 2 * np.pi * dow / 7

        out_df['year'] = year - year.min()
        out_df['day_of_year'] = doy
        out_df['is_leap_year'] = leap
        out_df['day_of_year_sin'] = np.sin(doy_angular)
        out_df['day_of_year_cos'] = np.cos(doy_angular)

        out_df['quarter'] = dt_col.dt.quarter
        out_df['month'] = dt_col.dt.month
        out_df['week'] = dt_col.dt.isocalendar().week

        out_df['day'] = day
        out_df['day_of_month_sin'] = np.sin(day_angular)
        out_df['day_of_month_cos'] = np.cos(day_angular)

        out_df['day_of_week'] = dow
        out_df['is_weekend'] = np.asarray(dow >= 5, dtype=int)
        out_df['day_of_week_sin'] = np.sin(dow_angular)
        out_df['day_of_week_cos'] = np.cos(dow_angular)

        out_df['hour'] = dt_col.dt.hour
        out_df['minute'] = dt_col.dt.minute
        out_df['index_15min'] = out_df['hour'] * 4 + out_df['minute'] // 15
        out_df['second'] = dt_col.dt.second
        out_df['second_of_day'] = out_df['hour'] * 3600 + out_df['minute'] * 60 + out_df['second']
        sod_angular = 2 * np.pi * out_df['second_of_day'] / 86400
        out_df['second_of_day_sin'] = np.sin(sod_angular)
        out_df['second_of_day_cos'] = np.cos(sod_angular)

        out_df['is_workday'] = dt_col.dt.date.map(is_workday)
        out_df['is_holiday'] = dt_col.dt.date.map(is_holiday)
        out_df['is_special_workday'] = dt_col.dt.date.map(lambda d: d in workdays)
        out_df['is_special_holiday'] = dt_col.dt.date.map(lambda d: d in holidays)
        out_df['is_special_workday'] = out_df['is_special_workday'].map(lambda x: 1 if x else 0)
        out_df['is_special_holiday'] = out_df['is_special_holiday'].map(lambda x: 1 if x else 0)

        return pd.concat([df, out_df[self.feature_cols]], axis=1)

class DifferenceFeaturizer(WinOffsetFeaturizerBase):

    def __init__(self, *, offsets, feature_col, freq=None):
        """
        Building difference features.

        Args:
            offsets: list of lags, each element in the list corresponds to one lag.
            feature_col: list of input raw features. These features will be used to calculate stats and lag features.
            freq: The shift frequency.
        """
        super().__init__(offsets=offsets, feature_col=feature_col)

        self.freq = freq

#     @log_status()
    def transform(self, df: pd.DataFrame) -> pd.DataFrame:
        """
        Transform df to have difference features

        Args:
            df: input df

        Returns: output df with difference features.

        """
        feature_df = df[self.feature_col]
        df_lst = [df]

        for offset in self.offsets:
            diff_df = feature_df - feature_df.shift(offset, self.freq)
            diff_df.columns = [
                '{}_diff_offset_{}'.format(col, offset)
                for col in feature_df.columns.tolist()
            ]
            df_lst.append(diff_df)

        return pd.concat(df_lst, axis=1)

class RollingStatsFeaturizer(WinOffsetFeaturizerBase):
    DEFAULT_STATS = ['min', 'max', 'median', 'mean', 'std', 'skew']
    DEFAULT_ROLLING_KWARGS = {'min_periods': 1, 'center': False}

    def __init__(self,
                 *,
                 wins,
                 offsets,
                 feature_col,
                 is_interval: bool = False,
                 interval_key: str = 'index_15min',
                 freq: int = None,
                 stats: List[str] = None,
                 quantiles: List[float] = None,
                 rolling_kwargs: Dict = None):
        """
        Generate Rolling stats features, this is a class designed for three purpose:
        1. Generate Rolling window stats.
            1. subseries rolling window stats.
            2. ordinal rolling window stats.
        2. Generate Lag features.

        To generate pure Lag features, include window size 1 in wins, then this will generate pure lag according
        to offsets.

        To generate ordinal rolling window stats, set window size > 1, skew features will only be built when
        window size >= 3. The resulting ordinal rolling window stats will shift according to offsets. For example,
        for window size = 3 and the sample with timestamp 2020-01-02 08:30:00, the window will contain samples:

        ["2020-01-02 08:00:00", "2020-01-02 08:15:00", "2020-01-02 08:30:00"]

        To generate subseries rolling window stats, set is_interval=True, set window size > 1, skew features will
        only be built when window size >= 3. The data will first be grouped by using interval_key. The resulting ordinal
        rolling window stats will shift according to offsets. For example, for window size = 3 and the sample with
        timestamp 2020-01-05 08:30:00, the window will contain samples:

        ["2020-01-03 08:30:00", "2020-01-04 08:30:00", "2020-01-05 08:30:00"]

        Caution:
        if is_interval=True:
        1. Ensure that index is pd datetime index.
        2. Ensure that index gap is 15 mins.
        3. Ensure that index has no missing values or duplicate values.
        4. Ensure that the columns interested have no missing values.

        Args:
            wins: list of window sizes, each element in the list corresponds to one window size.
            offsets: list of lags, each element in the list corresponds to one lag.
            feature_col: list of input raw features. These features will be used to calculate stats and lag features.
            is_interval: If True, subseries rolling window stats else ordinal rolling stats features.
            interval_key: The interval key used to groupby df. used only if is_interval=True.
            freq: The shift frequency.
            stats: The stats features, if None, DEFAULT_STATS are used. skew is used only when window size >= 3, if
                   window size = 1, only mean will be used.
            quantiles: list of quantiles to build quantile features.
            rolling_kwargs: extra kwargs pass to df.rolling function.
        """
        super().__init__(wins=wins, offsets=offsets, feature_col=feature_col)

        self.freq = freq
        self.stats = stats
        self.offsets = offsets
        self.is_interval = is_interval
        self.interval_key = interval_key
        self.quantiles = quantiles
        self.rolling_kwargs = rolling_kwargs
        self.wins = wins

#     @log_status()
    def transform(self, df: pd.DataFrame) -> pd.DataFrame:
        """
        Transform df to have rolling stats features

        Args:
            df: input df

        Returns: output df with rolling stats features.

        """
        drop_interval_key = False

        if self.is_interval:
            if self.interval_key not in list(df.columns):
                logging.info('interval key {} not in df columns, '
                             'using default index_15min as interval key...'.format(self.interval_key))

                self.interval_key = 'index_15min'
                index_15min = df.index.hour * 4 + df.index.minute // 15
                df = df.assign(index_15min=index_15min)
                drop_interval_key = True  # drop index_15min after transformation

            if self.interval_key not in self.feature_col:
                self.feature_col.append(self.interval_key)
            else:
                logging.warning('interval_key {} in feature_col, '
                                'this feature will not be used to'
                                ' calculate the interval features ...'.format(self.interval_key))

        feature_df = df[self.feature_col]
        df_lst = [df]

        for win in self.wins:
            if win <= 1:
                logging.warning('window size <= 1, this will build lags only ...')
                stats = ['mean']
                win = 1
            elif win < 3 and 'skew' in self.stats:
                logging.warning('window size {} < 3, '
                                'skew requires at least 3 samples, '
                                'it will not be calculated !'.format(win))
                stats = self.stats[:]
                stats.remove('skew')
            else:
                stats = self.stats

            for offset in self.offsets:
                if self.is_interval:
                    curr_df = self._build_interval_stats(feature_df, win, offset, stats)
                else:
                    curr_df = self._build_rolling_stats(feature_df, win, offset, stats)

                df_lst.append(curr_df)

        out_df = pd.concat(df_lst, axis=1)
        if drop_interval_key:
            out_df.drop(columns=self.interval_key, inplace=True)

        return out_df

    def _build_interval_stats(self, df, win, offset, stats):
        df_lst = []
        df_groupby = df.groupby(self.interval_key)
        for col in df.columns.tolist():
            if col != self.interval_key:
                agg_dict = {
                    f'{self.interval_key}_{col}_win_{win}_offset_{offset}_{stat}': stat
                    for stat in stats
                }
                stats_df = df_groupby[col].apply(lambda x: x.rolling(win, **self.rolling_kwargs).agg(agg_dict))
                df_lst.append(stats_df)

                if self.quantiles is not None:
                    for qtl in self.quantiles:
                        pct_df = df_groupby[col].apply(lambda x: x.rolling(win, **self.rolling_kwargs).quantile(qtl))
                        pct_df.name = f'{self.interval_key}_{col}_win_{win}_offset_{offset}_q{qtl}'
                        df_lst.append(pct_df)

        return pd.concat(df_lst, axis=1).shift(offset, self.freq)

    def _build_rolling_stats(self, df, win, offset, stats):
        df_lst = []
        for col in df.columns.tolist():
            if col != self.interval_key:
                agg_dict = {
                    f'{col}_win_{win}_offset_{offset}_{stat}': stat
                    for stat in stats
                }
                stats_df = df[col].rolling(win, **self.rolling_kwargs).agg(agg_dict)
                df_lst.append(stats_df)

                if self.quantiles is not None:
                    for qtl in self.quantiles:
                        pct_df = df[col].rolling(win, **self.rolling_kwargs).quantile(qtl)
                        pct_df.name = f'{col}_win_{win}_offset_{offset}_q{qtl}'
                        df_lst.append(pct_df)

        return pd.concat(df_lst, axis=1).shift(offset, self.freq)

    @property
    def stats(self):
        return self._stats

    @stats.setter
    def stats(self, value):
        if value is None:
            logging.warning('using all default stats {}'.format(self.DEFAULT_STATS))
            self._stats = self.DEFAULT_STATS
        else:
            self._stats = list(value)

    @property
    def rolling_kwargs(self):
        return self._rolling_kwargs

    @rolling_kwargs.setter
    def rolling_kwargs(self, value):
        if value is None:
            self._rolling_kwargs = self.DEFAULT_ROLLING_KWARGS
        elif not isinstance(value, dict):
            raise ValueError('rolling_kwargs should be a dict!')
        else:
            self._rolling_kwargs = value

    @property
    def quantiles(self):
        return self._quantiles

    @quantiles.setter
    def quantiles(self, value):
        if value is not None:
            if isinstance(value, (float, int)):
                self._quantiles = [value]
            else:
                self._quantiles = list(value)

            for qtl in self._quantiles:
                if qtl > 1 or qtl < 0:
                    raise ValueError('Percentiles have to be float between 0 and 1!')

        else:
            self._quantiles = value


# This script computes the scaled change in tas, hus, hur and wap averaged over the Caribbean region for the whole atmospheric column (Figures 18 and 19)

import xarray as xr
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import os
import pandas as pd
import datetime
from cftime import DatetimeNoLeap
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import cmocean.cm as cmo
from matplotlib import cm
from matplotlib.colors import ListedColormap,LinearSegmentedColormap
from scipy import stats
import matplotlib.lines as mlines
import matplotlib.patches as mpatches

#------------------------INPUT------------------------#

# SETTINGS PROGRAM

# -------------Things to choose:----------------------
# Variables

var = 'hur' # hur hus ta wap
n = 58986
ssp = 'ssp585'

# ----------------------------------------------------

# list things to do
season_list = ['annual','dry','wet']
number_list = [0,1,2]
year_period_list = ['2005', '2085']

# Paths variables
paths = {'hus':'/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/hus/*'+ssp+'*.nc' ,
        'ta':'/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/ta/*'+ssp+'*.nc',
        'wap': '/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/wap/*'+ssp+'*.nc',
        'hur': '/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6_vertical/00_join_CMIP6_vertical_data/hur/*'+ssp+'*.nc'}

path_area_mean = {'tas_40N_40S': '/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/preprocessed_data/CMIP6/03_area_mean/40N_40S/tas/*'+ssp+'*.nc'}

# SETTINGS FIGURES

period_date = {'2005': ['1-1-1991','31-12-2020'], '2085' : ['1-1-2071','31-12-2100']}

xlabel = {'hus': '$\Delta$ Specific humidity', 'ta': r'$\Delta$ Temperature [$^{\circ}$C/$^{\circ}$C]', 'wap': '$\Delta$ Omega [Pa/s $^{\circ}$C]', 'hur': '$\Delta$ Relative humidity [%/ $^{\circ}$C]'}

# Titles

title_var = {'hus': 'Specific humidity','ta': 'Air temperature', 'wap': 'Vertical motion','hur': 'Relative humidity'}
season_title = {'annual':'Annual','dry': 'Dry season (December-April)','wet':'Wet season (May-November)'}
title_ssp = {'ssp126':'SSP1-2.6', 'ssp245': 'SSP2-4.5', 'ssp585': 'SSP5-8.5'}

# Functions

def time_period(da,period):
    """
    Select only the months inside the season of interest ('period') and takes running mean 30 yr window
    """
    if period == 'annual':
        da_time = da
        #da_time = da_time.rolling(time=30*12,center=False).mean(dim='time')
    elif period == 'wet':
        da_time = da.where((da.time.dt.month>=5) & (da.time.dt.month<=11),drop=True)#.dropna(dim='time')
        #da_time = da_time.rolling(time=30*7,center=False).mean(dim='time')
    elif period == 'dry':
        da_time = da.where((da.time.dt.month<=4) | (da.time.dt.month>=12),drop=True)#.dropna(dim='time')
        #da_time = da_time.rolling(time=30*5,center=False).mean(dim='time')
    return da_time #.dropna(dim='time')

def boxmean(da):
    """Compute spatial mean"""
    weights = np.cos(np.deg2rad(da.lat))
    weights.name = 'weights'
    boxmean = da.weighted(weights).mean(dim=('lat','lon'))
    return boxmean

def preprocessing_vertical(ds):
    """
    Function to call when importing netcdf files with xr.open_mfdataset.
    - Sets all the datasets to the same datetime and time axis
    - Deletes from the files inconsistent dimensions/coordinates and saves models name
    - Saves the name of the model and the rpf number
    """
    if ds.indexes['time'].inferred_type != "datetime64":
            ds['time'] = ds.indexes['time'].to_datetimeindex()
    ds['time'] = pd.date_range(start=str(ds.time[0].dt.year.values)+'-0'+str(ds.time[0].dt.month.values)+'-01',end='2100-12-16',periods=len(ds.time))
    if 'plev_bnds' in ds:
        ds =  ds.drop('plev_bnds')
    if 'plev' in ds:
        ds = ds.drop('plev')
    if 'height' in ds:
        ds = ds.drop('height')
    if 'file_qf' in ds:
        ds = ds.drop('file_qf')
    ds = ds.assign_coords(model_name = ds.source_id)
    ds = ds.assign_coords(variant_label = ds.variant_label)
    return ds

def preprocessing_area_mean(ds):
    """
    Function to call when importing netcdf files with xr.open_mfdataset.
    - Sets all the datasets to the same datetime and time axis
    - Deletes from the files inconsistent dimensions/coordinates and saves models name
    """
    if ds.indexes['time'].inferred_type != "datetime64":
            ds['time'] = ds.indexes['time'].to_datetimeindex()
    ds['time'] = pd.date_range(start=str(ds.time[0].dt.year.values)+'-0'+str(ds.time[0].dt.month.values)+'-01',end='2100-12-16',periods=len(ds.time))
    if 'plev_bnds' in ds:
        ds =  ds.drop('plev_bnds')
    if 'plev' in ds:
        ds = ds.drop('plev')
    if 'height' in ds:
        ds = ds.drop('height')
    if 'file_qf' in ds:
        ds = ds.drop('file_qf')
    return ds

#------------------------CALCULATIONS------------------------#

# Import variables
breakpoint()

tas_40 = xr.open_mfdataset(path_area_mean['tas_40N_40S'],concat_dim = 'model',combine = 'nested', preprocess = preprocessing_area_mean)
tas_40 = tas_40.load()
tas_40 = tas_40.tas
tas_40 = tas_40.sortby(tas_40.model_name)

ds = xr.open_mfdataset(paths[var],combine = 'nested', concat_dim = 'model',preprocess = preprocessing_vertical)
ds = ds.load()
ds = ds.sortby(ds.model_name)

if var == 'ta': ds = ds.ta[:,:,:,0,0]-273.15; color_line =  'darkred'; color_range = 'lightcoral'; x_0 = 1; x_limit = (-3.5,2.5)
elif var == 'hus': ds = ds.hus[:,:,:,0,0]; color_line = 'steelblue'; color_range = 'lightskyblue'; x_0 = 0
elif var == 'wap': ds = ds.wap[:,:,:,0,0]; color_line = 'chocolate'; color_range =   'sandybrown'; x_0 = 0
elif var == 'hur': ds = ds.hur[:,:,:,0,0]; color_line ='steelblue'; color_range = 'lightskyblue'; x_0 = 0

plevs = [100000.,  92500.,  85000.,  70000.,  60000.,  50000.,  40000.,  30000., 25000.,  20000.,  15000.,  10000.,   7000.,   5000.,   3000.,   2000., 1000.,    500.,    100.]
ds['plev'] = plevs

#----------------------------------#

# Start computations

breakpoint()

for season,number in zip(season_list,number_list):

    ds_season = time_period(ds,season)

    #fig , ax = plt.subplots(ncols=1, nrows=1,figsize=(10,7),sharex=True, sharey =True,subplot_kw=dict(projection = ccrs.PlateCarree(central_longitude=180)))
    plt.figure(n+number)

    ds_2005 = ds_season.sel(time=slice(period_date['2005'][0], period_date['2005'][1])).mean(dim = 'time')
    ds_2085 = ds_season.sel(time=slice(period_date['2085'][0], period_date['2085'][1])).mean(dim = 'time')
    if (var == 'hus') or (var == 'hur'): ds_2005 = ds_2005.where((ds_2005 >=0), drop = True); ds_2085 = ds_2085.where((ds_2085 >=0), drop = True)
    tas_40_2005 = tas_40.sel(time=slice(period_date['2005'][0], period_date['2005'][1])).mean(dim = 'time')
    tas_40_2085 = tas_40.sel(time=slice(period_date['2085'][0], period_date['2085'][1])).mean(dim = 'time')
    ds_toplot = (ds_2085.mean(dim = 'model') - ds_2005.mean(dim = 'model'))/(tas_40_2085.mean(dim = 'model') - tas_40_2005.mean(dim = 'model'))
    ds_toquant = (ds_2085 - ds_2005)/(tas_40_2085 - tas_40_2005)

    plt.gca().invert_yaxis()
    plt.plot(ds_toplot,ds_toplot.plev*1e-2,color = color_line )
    plt.fill_betweenx(ds_toplot.plev*1e-2,ds_toquant.quantile(0.05,dim='model'),ds_toquant.quantile(0.95,dim='model'),color=color_range,linewidth=0)
    plt.axvline(x= x_0, color='black', ls = '--', lw = 0.5)
    # plt.axvline(x= 0, color='grey', ls = '--', lw = 0.5)
    # plt.xlim(x_limit)
    plt.gca().invert_yaxis()

    plt.xlabel(xlabel[var],fontsize=12)
    plt.ylabel('Pressure level [hPa]',fontsize=12)
    plt.title(season_title[season],loc='right',fontsize=11)
    plt.title(title_ssp[ssp],loc='left',fontsize=11)

    # fig.suptitle('30 years mean vertical profile. '+title_var[var])

    percent =  mpatches.Patch( color=color_range, label=r'$90\%$ CMIP6')
    line = mlines.Line2D([], [], color=color_line, label=r'CMIP6 mmmean')
    plt.legend(handles=[percent,line])

    plt.gca().invert_yaxis()
    # plt.savefig('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/definitivos/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png')
    # print('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/definitivos/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png')

    plt.savefig('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/report/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png',dpi = 300)
    print('/home/bbrotonsbl/shared_data/volume_2/bbrotonsbl/resampling/Method-2014/output_plots/report/CMIP6_vertical_profile_2085-2005_'+var+'_'+season+'_season_'+ssp+'.png')


import logging
import os
import time
import warnings
from datetime import datetime, timedelta
import onnxruntime
import json
warnings.filterwarnings("ignore")
import cv2
import numpy as np
import torch

from clogging import LOGGING_PATH, setup_logger

time_run = datetime.now()
dt_string = time_run.strftime("%d-%m-%Y")
log_fname = '%s' % dt_string
logger = setup_logger(log_fname,LOGGING_PATH +
                       '/log_%s.log' % dt_string, level=logging.DEBUG)

class ObjectDetectionClassification:
    """
        Performs the Object Localization or extraction into important entities of poses from image and gives coorniates for detected bbox
        and also gives the classification if images into 2 classes Helathy,Disease

    Args:
        self: constructor,
        combined_json: configuration file json

    Returns:
        None

    """

    def __init__(self,img):
        self.file_dir = os.getcwd()
        self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
        logger.info(['FILE DIR',self.file_dir])
        self.img = img
        #Config file
        with open("config.json") as config_file:
            self.config_data = json.load(config_file)
        config_file.close()
        self.model_path = os.path.join(self.file_dir,self.config_data['models_location_path'])
        self.object_detection_model = os.path.join(self.model_path,self.config_data['obj_detection_model'])

    def load_model(self):
        """
            Loads Object Detection Model in ONNX runtime.

        Args:
            self: constructor

        Returns:
            ONNX Runtime inference session
        """
        try:
            cuda = False
            model_name = self.object_detection_model
            providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider']
            session = onnxruntime.InferenceSession(model_name, providers=providers)
            return session
        except Exception as e:
            print(['Exception occured in loading model',e])

    def scale_coords(self,img1_shape, coords, img0_shape, ratio_pad=None):
        '''
            Rescale coords (xyxy) from img1_shape to img0_shape
            Args:
            img1_shape: resized image shape
            img0_shape: original input image shape
            coords: predictions returned from object detection model

            Returns:
            ROI coordinates in original image
        '''
        try:
            if ratio_pad is None:  # calculate from img0_shape
                gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1])  # gain  = old / new
                pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2  # wh padding
            else:
                gain = ratio_pad[0][0]
                pad = ratio_pad[1]

            coords[ [0, 2]] -= pad[0]  # x padding
            coords[ [1, 3]] -= pad[1]  # y padding
            coords[ 0:4] /= gain
            return coords
        except Exception as e:
            print(['Exception occured in scale_coords function ',e])

    def letterbox(self,im: np.array, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleup=True, stride=32):
        """
            Creates Letter Box or gives depth,width,height of boxes.

        Args:
            self: constructor,
            im (array): image of input card,
            new_shape (filter): image resizing filter,
            color:  color in RGB channel,
            auto,scaleup,stride: other tuning parameters

        Returns:
            Image with ratio,width and height of boxes
        """
        try:
            # Resize and pad image while meeting stride-multiple constraints
            shape = im.shape[:2]  # current shape [height, width]

            if isinstance(new_shape, int):
                new_shape = (new_shape, new_shape)

            # Scale ratio (new / old)
            r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
            if not scaleup:  # only scale down, do not scale up (for better val mAP)
                r = min(r, 1.0)

            # Compute padding
            new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
            dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding

            if auto:  # minimum rectangle
                dw, dh = np.mod(dw, stride), np.mod(dh, stride)  # wh padding

            dw /= 2  # divide padding into 2 sides
            dh /= 2

            if shape[::-1] != new_unpad:  # resize
                im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_CUBIC)
            top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
            left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
            # The image returned below gets degraded
            im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)  # add border
            return im, r, (dw, dh)
        except Exception as e:
            print(['Exception occured in creating bounding box',e])

    def preprocess_image(self):
        """
            Preprocessing of input image.

        Args:
            self: constructor,
            img (array): image of input

        Returns:
            Image with ratio,width and height of boxes
        """
        try:
            # img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            logger.info(['Processing Images'])
            image = self.img.copy()
            image, ratio, dwdh = self.letterbox(image, auto=False)
            original_image = image
            image = image.transpose((2, 0, 1))
            image = np.expand_dims(image, 0)
            image = np.ascontiguousarray(image)
            im = image.astype(np.float32)
            im /= 255
            return im,original_image,ratio,dwdh
        except Exception as e:
                print(['Exception occured in preprocessing image',e])

    def object_model_predictions(self,img,ort_session):
        """
            Object Detection Model predictions using ONNX runtime.

        Args:
            self: constructor,
            img (array): image of input ,
            ort_session: onnx modelsession,

        Returns:
            Output predictions of different rois or entities with craft coordinates
        """

        try:
            outname = [i.name for i in ort_session.get_outputs()]
            inname = [i.name for i in ort_session.get_inputs()]
            inp = {inname[0]:img}
            # ONNX inference
            outputs = ort_session.run(outname, inp)[0]
            return outputs
        except Exception as e:
            print(['Exception occured in ONNX Inference session and predictions',e])

    def roi_extraction(self,img,outputs,im0):
        """
            Extracts ROIs from the Model outputs

        Args:
            self: constructor,
            img (array): preprocessed image,
            original_image: original input image ,
            ratio (float):  ratio, in which boxes are proportioned,
            dw,dh (float): width and height of bounding box,
            outputs (array): bounding boxes predictions for different entities
            im0: original input image

        Returns:
            Bounding box extracted for different entities clubbed in a Dictionary
        """

        try:
            # roi ext
            roi_dict = dict()
            pred = outputs
            names = ['disease','healthy']

            # Process detections
            for det in enumerate(pred):  # detections per image
                if len(det):
                    # Rescale boxes from img_size to im0 size
                    det1 = self.scale_coords(img.shape[2:], det[1][1:5], im0.shape).round()
                    logger.info(['Scaled Coordinates',det1])
                    cls = det[1][5]
                    score = det[1][6]
                    cls_id = int(cls)
                    name = names[cls_id]
                    logger.info(['Class Name',name])
                    score = round(float(score),3)
                    box_coordinates = [int(coord) for coord in det1]
                    logger.info(['Confidence Score',score])
                    im = cv2.rectangle(im0, (box_coordinates[0], box_coordinates[1]), (box_coordinates[2], box_coordinates[3]), (0, 255, 0), 2)
                    # Add text to the image
                    font = cv2.FONT_HERSHEY_SIMPLEX
                    cv2.putText(im, name, (box_coordinates[0], box_coordinates[1] - 10), font, 1, (0, 255, 0), 2, cv2.LINE_AA)
            return im
        except Exception as e:
            print(['Exception occured in roi_extraction from images',e])

    def main(self):
        """
            Integration of all above methods,gives coordinates of bbox and image classification results

        Args:
            self: constructor,
            image (array): original input image

        Returns:
            Dictionary with bbox img
        """
        try:
            logger.info(['Starting Main Function In Object Detection/Classifiation'])
            start_t = time.perf_counter()
            object_model = self.load_model()
            img,_,_,_ = self.preprocess_image()
            preds = self.object_model_predictions(img,object_model)
            im = self.roi_extraction(img,preds,self.img)
            end_t = time.perf_counter()
            print("Time Taken for execution:",end_t-start_t)
            logger.info(["Returning Output Image"])
            return im
        except Exception as e:
            logger.debug(['Exception occured in main function',e])


# coding: utf-8

# In[277]:

import pandas as pd
import matplotlib.pyplot as plt
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# In[248]:

#Edward J Smith - The captain goes doen with the ship
#we are trying to predict whether a passanger would survive using the train test
#then test it on the testing set

# In[280]:

test.head()
X = train
train.head() #feature matrix

# In[281]:

X.isnull().sum().astype(float) /X.shape[0]#check feature matrix for nas
#I have nas so I need to either fill them in with sume value or remove them from the dataset
X = X.drop(['Cabin'], axis = 1)#because there is so much missing data from cabin just remove it

# In[282]:

X['Embarked'].value_counts() #only 2 values from embarked are missing, options are S,C or Q so can just use S, most common
X['Embarked'] = X['Embarked'].fillna('S') #so can just replace with the most common value which is S

# In[283]:

import matplotlib.pyplot as plt
X.isnull().sum().astype(float) /X.shape[0] #only column with nas now is Age
 #get distribution of Age
X['Age'].hist()
X['Age'].mean() #one option is to fill in the nas with the mean (29)
X['Age'].median() #(28) so data is slightly skewed to the right

# In[284]:

#compare age versus pclass too see how age an pclass fit together
import seaborn as sns
for pclass in X['Pclass'].unique(): #for every unique pclass
    sns.kdeplot(X[X['Pclass'] == pclass]['Age'], label = pclass)#groupy pclass and density plot for age
    #we see a trend of the older the person the higher the class (1 = highest)

# In[285]:

#so to replace missing age nas values we might group by pclass and assign mean age for each class
X.groupby('Pclass')['Age'].mean()

for pclass in X['Pclass'].unique(): #for every unique pclass, locate the age column value, match to groupe by Pclass means
    X.loc[(X['Pclass'] == pclass) & (X['Age'].isnull()),'Age'] = X.groupby('Pclass')['Age'].mean().loc[pclass]

# In[287]:

X.isnull().sum() # wehave finally got rid of all the missing data
#X.head() #all age values are now means grouped by Pclass
#now we can assess which variables are important when considering passenger survival.
# [Feature Engineering and transformations]

# In[157]:

#visual on survival and Pclass to determine if Pclass is a good predictor
plt.figure(figsize= [16,8])
plt.subplot(121)
plt.hist(train[X['Survived'] == 0]['Pclass'])
plt.title("Perished")
plt.xlabel("Ticket Class")
plt.subplot(122)
plt.hist(train[X['Survived'] == 1]['Pclass'])
plt.title("Survived")
plt.xlabel("Ticket Class")

# In[158]:

#to get better understanding you might want the mean as well
X.groupby('Pclass')['Survived'].mean()
#so this is obviously going to be a pretty good predictor features of who would survive, PClass

# In[145]:

#visual on Age and Pclass to determine if Pclass is a good predictor
#turns out Age is not a great predictor of survival, although there is the spike for infants among survivors
#versus perished so knowing whether or not they are an infant would be helpful
print (X.groupby('Survived')['Age'].mean())
plt.figure(figsize= [8,4])
plt.subplot(121)
plt.hist(train[X['Survived'] == 0]['Age'])
plt.title("Perished")
plt.xlabel("Age of Passenger")
plt.subplot(122)
plt.hist(train[X['Survived'] == 1]['Age'])
plt.title("Survived")
plt.xlabel("Age of Passenger")

# In[129]:

#create an infant variable to use as a predictor
#an idea to try later, distribution if age older than 10 for survivor and perish groups looks quite similar
X['Infant'] = X['Age'] < 10
X.head()

# In[288]:

# How to do consider string data and categorical data
#you could make categorical data numeric, e.g Sex could be 1,0 or Embarked could be 1,2,3 but numerical data could interpret
# embarked 3 being much more different to 1 than it is from 2, the ML estimator will interpret these that way
#as feature matrix only can have numerical values for scikit learn then they must be changed

X['Sex'].replace({'male': 0, 'female':1}, inplace = True) #create dictionary to replace sex data

# In[289]:

#Ticket data appears random and not useful for any predictions so remove it
X.drop(['Ticket'], axis = 1, inplace = True)

# In[303]:

#consider embarked feature with one hot encoding
#this means that instead of having 1 embarked column we would have 3
X['Embarked_S'] = 0
X['Embarked_C'] = 0
X['Embarked_Q'] = 0
X.loc[X['Embarked'] == 'S', 'Embarked_S'] = 1
X.loc[X['Embarked'] == 'C', 'Embarked_C'] = 1
X.loc[X['Embarked'] == 'Q', 'Embarked_Q'] = 1
X.drop(['Embarked'], axis = 1, inplace = True)

# In[291]:

#Now to consider the name column
#Mrs,ms,Masters etc...is it a predictor versus Mr
#take name feature, split on , and get 1st value, split again on . and get 0th value, then get all unique values
X['Name'].str.split(',').apply(lambda x: x[1]).str.split('.').apply(lambda x: x[0]).unique()
#an idea for later to see if it is a good predictor when having a royal title etc

#for now
X.drop(['Name'], axis = 1, inplace = True)

# In[292]:

X.drop(['PassengerId'], axis = 1, inplace = True) #dont need passengerid as it is random
#so not a good predictor

# In[304]:

#y = X.pop('Survived') #this is my target label data
X.head() #is my feature matrix
#y.head()

# In[293]:

#building the model
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression

# In[305]:

model = Pipeline([
    ('scaler', StandardScaler()),
    ('logreg', LogisticRegression()),
    ])

#gridsearch for feature engineering, look at hyper-parameters of Logistic Regression
gs = GridSearchCV(
    model,
    {'logreg__penalty':['l1','l2'],
    'logreg__C':[0.01, 0.1, 1, 10, 100]},
    cv = 5,
    n_jobs = 4
)

# In[306]:

gs.fit(X,y)

# In[307]:

gs.best_params_ #tells us which hyper-paramters performed best
#so I can go back and just enter these parameters into the hyper parameters under logistic regression model(penalty = etc)

# In[308]:

test.head()

# In[309]:

#now need to do the same data munging to our test data (target label data)
test = test.drop(['Cabin'], axis = 1)#because there is so much missing data from cabin just remove it
test.drop(['Name'], axis = 1, inplace = True)
test.drop(['Ticket'], axis = 1, inplace = True)
passengertestID = test.pop('PassengerId') #pop passenger ID to add bacl into test df later, for predict probabilitys
#need the shape of test and train data to be the same

# In[310]:

test['Embarked'] = test['Embarked'].fillna('S')
for pclass in test['Pclass'].unique(): #for evertest unique pclass, locate the age column value, match to groupe btest Pclass means
    test.loc[(test['Pclass'] == pclass) & (test['Age'].isnull()),'Age'] = test.groupby('Pclass')['Age'].mean().loc[pclass]
test['Sex'].replace({'male': 0, 'female':1}, inplace = True) #create dictionartest to replace setest data

test['Embarked_S'] = 0
test['Embarked_C'] = 0
test['Embarked_Q'] = 0
test.loc[test['Embarked'] == 'S', 'Embarked_S'] = 1
test.loc[test['Embarked'] == 'C', 'Embarked_C'] = 1
test.loc[test['Embarked'] == 'Q', 'Embarked_Q'] = 1
test.drop(['Embarked'], axis = 1, inplace = True)

# In[313]:

test['Fare'] = test['Fare'].fillna('9.0')
test.isnull().sum().astype(float) /test.shape[0] #clear test nans

# In[314]:

score = gs.score(X, y)
print('Logistic Regression pipeline test accuracy: %.3f' % score)

# In[315]:

predictions = gs.predict_proba(test)
default_prob = [i[1] for i in predictions] #Probability for survival

# In[316]:

#create dataframe for predicitons
#dont have the correct passenger ids
prediction = gs.predict(test)
survived = pd.DataFrame({'Survived': prediction}) #prediction of survival generated by model for each test passenger row
survived['PassengerID'] = passengertestID #add correct passenger ids back from test

# In[318]:

survived.head()
survived.to_csv('results.csv', encoding='utf-8', index=False)


from keras import backend as K
K.set_image_data_format('channels_first')
from keras.models import model_from_json
from facenet import load_dataset, load_facenet, img_to_encoding
import cv2 as cv
import numpy as np
import pickle
import os
import sklearn
from sklearn.cluster import KMeans

def q1_c():
    model = load_facenet()
    files = os.listdir("saved_faces")
    data = []
    paths = []
    for path in files:
        image = cv.imread("saved_faces/" + path)
        v = img_to_encoding(image, model)
        data.append(v)
        paths.append(path)

    with open("saved_data.pickle", "wb") as handle:
        pickle.dump([data, paths], handle, protocol=pickle.HIGHEST_PROTOCOL)

def load_input_face():
    model = load_facenet()
    files = os.listdir("input_faces")
    data = []
    paths = []
    for path in files:
        image = cv.imread("input_faces/" + path)
        image = cv.resize(image, (96, 96))
        v = img_to_encoding(image, model)
        data.append(v)
        paths.append(path)

    with open("input_data.pickle", "wb") as handle:
        pickle.dump([data, paths], handle, protocol=pickle.HIGHEST_PROTOCOL)
#
# load_dataset()

def load_pickle_data_saved():
    with open("saved_data.pickle", "rb") as handle:
        Data = pickle.load(handle)
        names = list(Data[1])
        embeddings = Data[0]
        print(len(embeddings))
        print(embeddings)
    return names, embeddings

def load_pickle_data_input():
    with open("input_data.pickle", "rb") as handle:
        Data = pickle.load(handle)
        names = list(Data[1])
        embeddings = Data[0]
    return names, embeddings

def cluster():
    # use sklearn's kmeans to cluster
    # Q1 e
    names, embeddings = load_pickle_data_saved()
    pre_clustering = []
    for vector in embeddings:
        pre_clustering.append(vector[0])
    kmeans = KMeans(n_clusters=6, random_state=0).fit(pre_clustering)
    # print(len(kmeans.labels_))
    print(kmeans.labels_)
    return kmeans, names

def inverted_index():
    # Q1 f
    kmeans, names = cluster()
    labels_list = kmeans.labels_
    d = {1: [], 2: [], 3: [], 4: [], 5: [], 0: []}
    for i in range(len(names)):
        d[labels_list[i]].append(names[i])
    return d, kmeans.cluster_centers_

def match_image():
    # Q1 g and h
    inverted_index_dictionary, centers = inverted_index()
    names, embeddings = load_pickle_data_input()

    images_scores = {}
    for i in range(len(names)):
        # compute norm-dot_product
        q = embeddings[i]
        scores = []
        for center_index in range(len(centers)):
            t = centers[center_index]
            score = np.divide(np.dot(q, t), (np.linalg.norm(q) * np.linalg.norm(t)))
            scores.append([score, center_index])
        images_scores[names[i]] = max(scores)

    # use a threshold to keep only high-scored matchings
    threshold = 0.8
    for key in images_scores.keys():
        if images_scores[key][0] <= threshold:
            images_scores[key][1] = "None"
        else:
            images_scores[key][0] = list(inverted_index_dictionary[images_scores[key][1]])
    # for key in images_scores.keys():
    #     print("----image: " + str(key) + " belongs to class: " + str(images_scores[key][1]) + "----")
    print(images_scores)

match_image()


## Ex 5-1. QPushButton.
## https://wikidocs.net/21934

import sys
from PyQt5.QtWidgets import QApplication, QWidget, QPushButton, QVBoxLayout

class MyApp(QWidget):

    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        btn1 = QPushButton('&Button1', self)
        btn1.setCheckable(True)
        btn1.toggle()
        # btn1.toggled.connect(self.activateBtn)

        btn2 = QPushButton(self)
        btn2.setText('Button&2')

        btn3 = QPushButton('Button3', self)
        btn3.setEnabled(False)

        vbox = QVBoxLayout()
        vbox.addWidget(btn1)
        vbox.addWidget(btn2)
        vbox.addWidget(btn3)

        self.setLayout(vbox)
        self.setWindowTitle('QPushButton')
        self.setGeometry(300, 300, 300, 200)
        self.show()

    # def activateBtn(self, state):
    #     if self.btn1.isChecked == 1:
    #         self.btn3.setEnabled(True)
    #     else:
    #         self.btn3.setEnabled(False)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = MyApp()
    sys.exit(app.exec_())


import torch
import torch.nn as nn

class LatentEmbeddingModel(nn.Module):
    def __init__(self, n_user, n_item, n_embed):
        super(LatentEmbeddingModel, self).__init__()

        # embedding layer (B, 2) -> (B, 2, N_EMBED)
        self.embedding_user = nn.Embedding(n_user, n_embed)
        self.embedding_item = nn.Embedding(n_item, n_embed)

        # global avg, bias
        self.mu = nn.Embedding(1, 1)
        self.b_user = nn.Embedding(n_user, 1)
        self.b_item = nn.Embedding(n_item, 1)

    def forward(self, x):
        # x (B, 2)
        x_user = self.embedding_user(x[:, 0].long())                # (B, N_EMBED)
        x_item = self.embedding_item(x[:, 1].long())                # (B, N_EMBED)

        mu = self.mu(torch.tensor([0]*x.shape[0])).view(-1)         # (B)
        b_user = self.b_user(x[:, 0].long()).view(-1)               # (B)
        b_item = self.b_item(x[:, 1].long()).view(-1)               # (B)

        # x = torch.sum(x_user * x_item, dim=1)  # (B)
        x = mu + b_user + b_item + torch.sum(x_user*x_item, dim=1)  # (B)
        return x


# -*- coding: utf-8 -*-
"""Расчёт длины трубопроводной петли"""
__title__ = 'Длина\nпетли'
__author__ = 'SG'

import clr
clr.AddReference('System.Core')
from System.Collections.Generic import *

from Autodesk.Revit.DB import FilteredElementCollector, BuiltInCategory, Transaction, TransactionGroup, BuiltInParameter, ElementId

doc = __revit__.ActiveUIDocument.Document
uidoc = __revit__.ActiveUIDocument

k = 304.8

global handledElems
global counter
counter = 0
global OUT

def toList(iEnumerator):
    list = []
    while iEnumerator.MoveNext():
        list.append(iEnumerator.Current)
    return list

def getConnectors(elem):
    global tabs
    tabs += 1
    connectors = []
    if elem.Category.Name == 'Трубы':
        connectors = toList(elem.ConnectorManager.Connectors.GetEnumerator())
    elif elem.Category.Name == 'Соединительные детали трубопроводов'\
         and (elem.Name == 'Дренаж' or elem.Name == 'Прямая'):
        connectors = toList(
            elem.MEPModel.ConnectorManager.Connectors.GetEnumerator())
    tabs -= 1
    return connectors

def getSiblings(elem):
    global handledElems
    global counter
    global tabs
    connectorSet = getConnectors(elem)
    siblings = []
    if elem.Name == 'Дренаж' or 'Труба' in elem.Name or 'Прямая' in elem.Name:
        siblings = [elem]
    for i, connector in enumerate(connectorSet):
        counter += 1
        tabs += 1
        if connector.IsConnected:
            for con in connector.AllRefs:
                if con.Owner.Id.ToString() != connector.Owner.Id.ToString() and\
                   con.Owner.Category.Name != 'Трубопроводные системы':
                    linkedCon = con
        else:
            tabs -= 1
            continue
        owner = linkedCon.Owner
        if owner.Id.ToString() not in handledElems:
            handledElems.append(owner.Id.ToString())
            sibs = getSiblings(owner)
            if sibs:
                siblings.extend(sibs)
    tabs -= 1
    return siblings

def main():
    pipes = FilteredElementCollector(doc)\
        .OfCategory(BuiltInCategory.OST_PipeCurves)\
        .WhereElementIsNotElementType().ToElements()
    pipes = [pipe for pipe in pipes
             if '{}'.format(pipe.LookupParameter('Длина').AsDouble() * k) !=
             pipe.LookupParameter('Длина старая').AsString()
             ]

    tg = TransactionGroup(doc, "Длина петли")
    tg.Start()
    t = Transaction(doc, "blabla")
    t.Start()

    [pipe.LookupParameter('Длина старая').Set('{}'.format(pipe.LookupParameter('Длина').AsDouble() * k)) for pipe in pipes]

    global handledElems
    handledElems = []
    global tabs
    global OUT
    siblingGroups = []
    for pipe in pipes:
        siblings = []
        if pipe.Id.ToString() not in handledElems:
            handledElems.append(pipe.Id.ToString())
            tabs = 1
            siblings = getSiblings(pipe)
            siblingGroups.append(siblings)

    for group in siblingGroups:
        s = sum([e.LookupParameter('Длина').AsDouble() * k for e in group if e.LookupParameter('Длина')])
        for el in group:
            el.LookupParameter('Длина петли').Set('{:.0f}'.format(s / 1000).replace('.', ','))

    t.Commit()
    tg.Assimilate()

with open('C:\\Users\\sgrodnik\\Desktop\\so.txt', 'w', buffering=1) as o:
    # sys.stdout = o
    main()


# vi: set shiftwidth=4 tabstop=4 expandtab:
import datetime

def get_triangle_from_line(line):
    return tuple(int(v) for v in line.split())

def get_triangles_from_file(file_path="../../resources/year2016_day3_input.txt"):
    with open(file_path) as f:
        return [get_triangle_from_line(l.strip()) for l in f]

def triangle_is_possible(t):
    a, b, c = sorted(t)
    return a + b > c

def run_tests():
    assert not triangle_is_possible((5, 10, 25))
    assert triangle_is_possible((5, 10, 12))

def get_solutions():
    triangles = get_triangles_from_file()
    print(sum(triangle_is_possible(t) for t in triangles) == 993)

    s = 0
    triangles_iter = iter(triangles)
    while True:
        try:
            for t in zip(
                next(triangles_iter), next(triangles_iter), next(triangles_iter)
            ):
                s += triangle_is_possible(t)
        except StopIteration:
            break
    print(s == 1849)

if __name__ == "__main__":
    begin = datetime.datetime.now()
    run_tests()
    get_solutions()
    end = datetime.datetime.now()
    print(end - begin)


import requests
from rdflib import DCTERMS, RDF, SKOS, Graph, Literal, Namespace, URIRef

from dawe_vocabs.schemas import LUTSchema
from dawe_vocabs.vocabs.common import get_local_uuid_name

class NoDataInAPIException(Exception):
    """This is raised when no data is received from the API."""

    pass

class UnexpectedDataShapeException(Exception):
    """This is raised when the shape of the data (JSON) is not the expected shape."""

def create(
    base_uri: Namespace, g: Graph, config: LUTSchema, parent_collection_uri: str
) -> None:
    r = requests.get(config.endpoint_url)
    r.raise_for_status()

    data = r.json()

    if isinstance(data, list):
        # Strapi3
        rows = data
    else:
        if data.get("data"):
            # Strapi4
            rows = data["data"]
        else:
            rows = None
            # raise UnexpectedDataShapeException(
            #     "Expected the JSON object to have a 'data' key but none was found."
            # )

    if not rows:
        raise NoDataInAPIException(
            f"API endpoint {config.endpoint_url} returned no data."
        )

    concepts = list()

    for row in rows:
        if row.get("attributes") and row["attributes"].get("uuid"):
            local_name = row["attributes"]["uuid"]
        else:
            # Use this temporarily until the "uuid" field is available from the API.
            local_name = get_local_uuid_name(f"{config.uuid_namespace}-{row['id']}")
        concept_uri = base_uri[local_name]
        concepts.append(concept_uri)

        if row.get("attributes"):
            label = row["attributes"]["label"]
            symbol = row["attributes"]["symbol"]
            if row["attributes"].get("tree_description"):
                description = (
                    "Tree description: "
                    + row["attributes"]["tree_description"]
                    + ".  "
                    + row["attributes"]["shrub_description"]
                    + ". "
                )
            else:
                description = row["attributes"]["description"]
        else:
            label = row["label"]
            description = row["description"]
            symbol = row["symbol"]

        g.add((concept_uri, RDF.type, SKOS.Concept))
        g.add((concept_uri, SKOS.prefLabel, Literal(label)))
        g.add((concept_uri, SKOS.definition, Literal(description)))
        g.add((concept_uri, SKOS.notation, Literal(symbol)))
        g.add((concept_uri, DCTERMS.identifier, Literal(row["id"])))
        g.add(
            (
                concept_uri,
                DCTERMS.source,
                URIRef(config.endpoint_url),
            )
        )

    # Create the collection
    collection_uri = base_uri[config.collection_uuid]
    g.add((collection_uri, DCTERMS.source, URIRef(config.endpoint_url)))
    g.add((URIRef(parent_collection_uri), SKOS.member, collection_uri))
    g.add((collection_uri, RDF.type, SKOS.Collection))
    g.add((collection_uri, SKOS.prefLabel, Literal(config.label)))
    g.add(
        (
            collection_uri,
            DCTERMS.description,
            Literal(config.description),
        )
    )
    for concept in concepts:
        g.add((collection_uri, SKOS.member, concept))


import os.path
from PyQt5.QtWidgets import (
    QWizardPage, QWizard, QLabel, QVBoxLayout, QCheckBox, QGridLayout,
    QLineEdit, QProgressBar, QPushButton, QMessageBox, QHBoxLayout,
    QApplication, QSizePolicy, QFileDialog
)
from PyQt5.QtGui import QFont
from PyQt5.QtCore import Qt, pyqtSignal, QThread

class FinalPage(QWizardPage):
    """Final QWizard page, displays test result."""
    def __init__(self, threadlink, test_utility, report):
        self.system_font = QApplication.font().family()
        self.label_font = QFont(self.system_font, 12)

        super().__init__()

        self.threadlink = threadlink
        self.tu = test_utility
        self.report = report

    def initializePage(self):
        self.report.file_not_found_signal.connect(self.file_not_found)
        self.report.generic_error_signal.connect(self.generic_error)

        # Disable abort button
        self.threadlink.button(QWizard.CustomButton1).setEnabled(False)

        # Check test result
        report_dir_path = self.tu.settings.value("report_dir_path")
        self.report.set_file_location(report_dir_path)
        report_file_path = self.report.generate_report()

        test_result = self.report.test_result

        if test_result == "PASS":
            self.test_status = "Successful"
        else:
            self.test_status = "Failed"

        self.test_status_lbl = QLabel(f"Test {self.test_status}!")
        self.test_status_lbl.setFont(self.label_font)

        self.report_location_lbl = QLabel(
            f"Report available at: {report_file_path}.")
        self.report_location_lbl.setWordWrap(True)
        self.report_location_lbl.setFont(self.label_font)

        self.break_down_lbl = QLabel("Remove power and remove DUT test "
                                     "fixture.")
        self.break_down_lbl.setFont(self.label_font)

        self.layout = QVBoxLayout()
        self.layout.addSpacing(100)
        self.layout.addWidget(self.test_status_lbl)
        self.layout.addSpacing(25)
        self.layout.addWidget(self.break_down_lbl)
        self.layout.addSpacing(25)
        self.layout.addWidget(self.report_location_lbl)
        self.layout.addStretch()
        self.setLayout(self.layout)
        self.setTitle("Test Completed")

    def file_not_found(self):
        QMessageBox.warning(self, "Warning", "Report directory does not "
                            "exist!\n Please specify directory.")
        report_dir = QFileDialog.getExistingDirectory(self,
            "Select report save location.")
        self.tu.settings.setValue("report_dir_path", report_dir)
        self.initializePage()

    def generic_error(self, error):
        QMessageBox.warning(self, "Warning", error)


#!/usr/bin/python3
# encoding=utf-8

import os
import sys
import json
import time
from ffmpy import FFmpeg
import logging
import requests
from bs4 import BeautifulSoup

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

SAVE_PATH = "E://test"

def main():
    domain = "https://www.hifini.com/"
    page_idx = -1
    soup = BeautifulSoup('', features='html.parser')
    while True:
        page_idx += 1
        for idx in range(7, 1024, 2):
            music_data = soup.select(
                "#body > div > div > div.col-lg-9.main > div > div.card-body > ul > li:nth-child(%d)" % (idx))

            idx += 1
            if len(music_data) == 0:
                page_url = "https://www.hifini.com/forum-1-%d.htm?orderby=lastpid" % int(page_idx)
                logging.info("try request %s" % page_url)
                while True:
                    response = requests.get(page_url)  # Get方式获取网页数据
                    if response.status_code != 200:
                        logging.warning("request failed, code=%d" % response.status_code)
                        time.sleep(1)
                    else:
                        break
                soup = BeautifulSoup(response.text, features='html.parser')
                break
            elif len(music_data) != 1:
                continue

            data_href = music_data[0].get('data-href')
            url = domain + data_href
            play_html = requests.get(url)
            music_soup = BeautifulSoup(play_html.text, features="html.parser")
            author = None
            title = None
            music_url = None

            for script in music_soup.select("script"):
                script = str(script)
                if script.find("music") >= 0:
                    lines = script.split("\n")
                    for line in lines:
                        line = line.strip()
                        if line.split(":")[0] == "author":
                            author = line.split(":")[1].strip().strip(',')
                        elif line.split(":")[0] == "title":
                            title = line.split(":")[1].strip().strip(',')
                        elif line.split(":")[0] == "url":
                            music_url = line[len("url:"):].strip().strip(",")

            if author is not None and title is not None and music_url is not None:
                author = author.strip("'")
                title = title.strip("'")
                music_url = music_url.strip("'")
                download_url = domain + music_url

                response = requests.get(download_url)
                '''while response.status_code is None or response.status_code != 200:
                    response = requests.get(download_url)
                    #logging.info("request play page failed, code=%d" % response.status_code)
                    #time.sleep(1)
                '''
                if response.headers["Content-Type"] == "application/json;charset=UTF-8":
                    res_json = json.loads(response.text)
                    if res_json["code"] == -460:
                        logging.info("request play page failed, msg=%s, try later..." % res_json["message"])
                        continue
                elif response.status_code == 200:
                    suffix = response.headers["Content-Type"]
                    if suffix.find("/") > 0:
                        suffix = suffix.split("/")[1]
                    save_path = "%s/%s - %s.%s" % (SAVE_PATH, title, author, suffix)
                    with open(save_path, "wb") as fp:
                        fp.write(response.content)
                        logging.info("download success, saved as %s" % save_path)
                    if suffix.upper() != "MP3":
                        mp3_path = "%s/%s - %s.mp3" % (SAVE_PATH, title, author)
                        ff = FFmpeg(inputs={save_path: None}, outputs={mp3_path: None}, global_options="-y")
                        ff.run()
                        os.remove(save_path)

if __name__ == "__main__":
    main()


"""
Agent that creates symlinks to logfiles and embeds metadata in the
file/directory name of the Symlink.  Can be used in conjunction with a
log shipping agent (e.g. Fluentd) that watches the directory structure
containing the symlinks. Since all metadata is embedded in the
filename, there is no need to dynamically generate configuration for
the log shipping agent.
"""

import logging
import os
import pathlib
import re
import shutil

from kube_log_watcher.agents.base import BaseWatcher

logger = logging.getLogger(__name__)

def sanitize(s):
    return re.sub('[^a-zA-Z0-9_-]', '_', s)

class Symlinker(BaseWatcher):
    def __init__(self, configuration):
        symlink_dir = os.environ.get('WATCHER_SYMLINK_DIR', configuration.get('symlink_dir'))
        if not symlink_dir:
            raise RuntimeError(
                'Symlinker watcher agent initialization failed. Env variable WATCHER_SYMLINK_DIR must be set')
        self.symlink_dir = pathlib.Path(symlink_dir)
        if not self.symlink_dir.is_dir():
            raise RuntimeError(
                'Symlinker watcher agent initialization failed. Symlink base directory {} does not exist'
                .format(self.symlink_dir))
        logger.info('Symlinker watcher agent initialized')

    @property
    def name(self):
        return 'Symlinker'

    def add_log_target(self, target):
        logger.debug('Symlinker: add_log_target for %s called', target['id'])
        kw = target['kwargs']
        top_dir = self.symlink_dir / sanitize(kw['container_id'])
        link_dir = top_dir \
            / sanitize(kw['application'] or 'none') \
            / sanitize(kw['component'] or kw['application'] or 'none') \
            / sanitize(kw['namespace']) \
            / sanitize(kw['environment']) \
            / sanitize(kw['version'] or 'none') \
            / sanitize(kw['container_name'])
        link = (link_dir / sanitize(kw['pod_name'])).with_suffix('.log')

        if top_dir.exists():
            if link.is_symlink and link.samefile(kw['log_file_path']):
                logger.debug('Symlinker: link already exists for %s. Nothing to be done.', target['id'])
                return
            logger.info('Symlinker: metadata has changed for %s. Creating new symlink.', target['id'])
            shutil.rmtree(str(top_dir))
            logger.debug('Symlinker: Removed directory %s', top_dir)

        link_dir.mkdir(parents=True)
        link.symlink_to(kw['log_file_path'])
        logger.debug('Symlinker: Created symlink %s -> %s', link, kw['log_file_path'])

    def remove_log_target(self, container_id):
        logger.debug('Symlinker: remove_log_target for %s called', container_id)
        link_dir = str(self.symlink_dir / sanitize(container_id))
        try:
            shutil.rmtree(link_dir)
            logger.debug('Symlinker: Removed directory %s', link_dir)
        except Exception:
            logger.warning('%s watcher agent failed to remove link directory %s', self.name, link_dir)

    def flush(self):
        for container_dir in pathlib.Path(self.symlink_dir).iterdir():
            link = next(pathlib.Path(container_dir).glob('**/*.log'))
            if link and link.exists():
                continue
            else:
                shutil.rmtree(str(container_dir))


import openpyxl
import re
wb = openpyxl.load_workbook("question-choosing.xlsx").active
dataDict = {}
resultDict = {}

for i in range(4, 29):
    for j in range(2, 12, 4):
        key = wb.cell(row=i, column=j).value
        if (key != None):
            key = re.sub('\s+', '', key)
            key = key[-2:]
            dataDict[key] = wb.cell(row=i, column=j+1).value

groupResults = open("result.txt","w")
studentNames = open("../studentList.txt").readlines()

for i in range(1,26):
    groupResults.write("\n\nQuestion "+str(i)+"\n")
    result = [studentNames[int(k)-1] for k,v in dataDict.items() if v==i]
    groupResults.writelines(result)

for i in resultDict.items():
    print(i)