import time
from functools import wraps
from typing import Dict

import requests

from constants import GITHUB_ROOT, RENDER_ROOT
from logging_config import logger
from render_api.utils import get_headers, get_github_status

session = requests.Session()

# Decorator for logging and error handling
def log_and_handle_errors(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as exc:
            logger.error(f"Exception in {func.__name__}| {exc}")
            return None

    return wrapper

@log_and_handle_errors
def manage_deployment_status(data: Dict):
    pr = data["pull_request"]
    repo_data = data["repository"]
    state, merged = pr["state"], pr["merged"]
    user_repo, repo_url = repo_data["full_name"], repo_data["html_url"]
    owner, repo = repo_data["owner"]["login"], repo_data["name"]

    if not (merged and state == "closed"):
        return

    service_id = get_render_service_id(repo_url)
    if not service_id:
        logger.error("Render service ID is null")
        return

    deployment_status = get_render_deployment_status(service_id)
    if not deployment_status:
        return

    process_deployment_status(user_repo, repo, owner, deployment_status, service_id)

@log_and_handle_errors
def process_deployment_status(user_repo, repo, owner, deployment_status, service_id):
    github_status = get_github_status(deployment_status["status"])
    deployment_id = deployment_status["id"]
    github_deployment_id = create_github_deployment(user_repo, repo, owner)

    if not github_deployment_id:
        logger.error("Failed to create GitHub deployment")
        return

    update_github_deployment_status(
        owner, repo, github_status, deployment_id, user_repo, github_deployment_id, service_id
    )

@log_and_handle_errors
def update_github_deployment_status(
    owner, repo, status, deployment_id, user_repo, github_deployment_id, service_id
):
    create_github_deployment_status(
        owner, repo, status, deployment_id, user_repo, github_deployment_id
    )
    new_status = ""
    while new_status not in ["failure", "success"]:
        new_render_deployment_status = get_render_deployment_status(service_id)
        new_status = get_github_status(new_render_deployment_status["status"])
        time.sleep(
            10
        )  # You can remove it (but it's better to not spam the render API [400 GET request/minutes])
    create_github_deployment_status(
        owner, repo, new_status, deployment_id, user_repo, github_deployment_id
    )

@log_and_handle_errors
def get_render_deployment_status(service_id: str) -> Dict:
    url = f"{RENDER_ROOT}/services/{service_id}/deploys"
    response = session.get(url, headers=get_headers("render"))
    logger.info(f"GET: {url} executed with status_code: {response.status_code}")
    data = response.json()[0]["deploy"]
    return {"status": data["status"], "id": data["id"]}

@log_and_handle_errors
def get_render_service_id(repo: str) -> str:
    url = f"{RENDER_ROOT}/services"
    response = session.get(url, headers=get_headers("render"))
    logger.info(f"GET: {url} executed with status_code: {response.status_code}")
    for service in response.json():
        if service["service"]["repo"] == repo:
            return service["service"]["id"]

@log_and_handle_errors
def create_github_deployment(user_repo: str, repo: str, owner: str) -> str:
    url = f"{GITHUB_ROOT}/repos/{user_repo}/deployments"
    data = {
        "owner": owner,
        "repo": repo,
        "ref": "main",
        "environment": "Production",
        "production_environment": True,
        "description": "Deployment status from Render",
    }
    response = session.post(url, headers=get_headers("github"), json=data)
    logger.info(f"POST: {url} executed with status_code: {response.status_code}")
    return response.json().get("id")

@log_and_handle_errors
def create_github_deployment_status(
    owner: str,
    repo: str,
    status: str,
    render_deployment_id: str,
    user_repo: str,
    github_deployment_id: str,
):
    url = f"{GITHUB_ROOT}/repos/{user_repo}/deployments/{github_deployment_id}/statuses"
    data = {
        "owner": owner,
        "repo": repo,
        "state": status,
        "deployment_id": render_deployment_id,
        "environment": "Production",
        "description": "Deployment status from Render",
    }
    response = session.post(url, headers=get_headers("github"), json=data)
    logger.info(f"POST: {url} executed with status_code: {response.status_code}")


"""
This will fetch database data from database
"""
from typing import List
from copy import deepcopy
from codegen.table.python_free_connex_table import PythonFreeConnexTable
from codegen.database import DatabaseDriver
from os import path

class DataFetcher:
    def __init__(self, db_driver: DatabaseDriver):
        """
        Construct a db fetcher instance. It requires to have a db driver input,
        in order to fetch different files

        :param db_driver: A db_driver, can be postgres_db_driver
        """
        self.db_driver = db_driver

    def store_data(self, output_dir: str, tables: List[PythonFreeConnexTable], should_write=True) -> List[
        PythonFreeConnexTable]:
        """
        Perform a select on all tables and stored output data into the [output_dir].
        Will also return a new list of tables which has the dat_path and data_size set.

        :type should_write: object
        :param output_dir: Output dir
        :param tables: List of tables
        :return:
        """
        new_tables = deepcopy(tables)
        for i, table in enumerate(tables):
            if len(table.annotations) > 0:
                annotations = ""
                for index, annotation in enumerate(table.annotations):
                    annotations += f"{annotation} as {table.get_annotation_name(index)}"
                    if index < len(table.annotations) - 1:
                        annotations += ","

                sql = f"select *, {annotations} from {table._table_name};"
            else:
                sql = f"select * from {table._table_name};"
            output_path = path.join(output_dir, table.variable_table_name) + '.tbl'
            size = 0
            if should_write:
                size = self.db_driver.execute_save(sql=sql, output_filename=output_path)
            new_tables[i].data_paths = [output_path]
            new_tables[i].data_sizes = [size]
        return new_tables


import streamlit as st
import pandas as pd
import numpy as np
import umap
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster
from sklearn.decomposition import PCA
import webbrowser

# Set width mode to wide to display plots better
st.set_page_config(layout="wide")
# Streamlit Configuration
st.set_option('deprecation.showPyplotGlobalUse', False)

# Sidebar
st.sidebar.header("Schizophrenia Data Analysis")
uploaded_file = st.sidebar.file_uploader("Choose a CSV file", type="csv")

# Sliders for UMAP and KMeans parameters
st.sidebar.subheader("UMAP Parameters")
n_neighbors = st.sidebar.slider("Number of Neighbors", 2, 50, 5)
min_dist = st.sidebar.slider("Minimum Distance", 0.0, 1.0, 0.3, 0.1)

st.sidebar.subheader("Clustering Parameters")
n_clusters = st.sidebar.slider("Number of Clusters (KMeans)", 2, 20, 5)
n_dendro_clusters = st.sidebar.slider("Number of Clusters (Dendrogram)", 2, 20, 5)

# Add option to choose linkage method for dendrogram
linkage_methods = ["ward", "single", "complete", "average"]
selected_linkage_method = st.sidebar.selectbox("Linkage Method for Dendrogram", linkage_methods, 0)

# Checkbox to toggle PCA and UMAP visualization
show_pca = st.sidebar.checkbox("Show PCA Visualization", False)
show_umap = st.sidebar.checkbox("Show UMAP Visualization", False)

# Load the data
def load_data(uploaded_file):
    data = pd.read_csv(uploaded_file)
    return data

# Function to perform UMAP embedding and K-means clustering
def umap_and_kmeans(band_data, n_neighbors=n_neighbors, min_dist=min_dist, n_clusters=n_clusters):
    embedding = umap.UMAP(n_neighbors=n_neighbors, min_dist=min_dist, random_state=42).fit_transform(band_data)
    kmeans_labels = KMeans(n_init=4, n_clusters=n_clusters, random_state=42).fit(embedding).labels_
    return embedding, kmeans_labels

# Function to plot UMAP embedding results
def plot_umap_embedding(embedding, kmeans_labels, ax, title):
    ax.scatter(embedding[:, 0], embedding[:, 1], c=kmeans_labels, cmap='rainbow', s=20)
    # add a text with umap parameters and kmeans cluster number
    ax.text(0.99, 0.01, f"n_neighbors={n_neighbors}, min_dist={min_dist}, n_clusters={n_clusters}",
            transform=ax.transAxes, ha='right', va='bottom', size=10)
    ax.set_title(title)

def plot_dendrogram_colored_ticks(band_data, ax, title, method='ward'):
    """
    Plot the dendrogram with correctly colored tick numbers for the "All Subjects" group.
    """
    # Hierarchical clustering
    Z = linkage(band_data, method=method)

    # Plot the dendrogram
    ddata = dendrogram(Z, ax=ax, leaf_rotation=90)
    ax.set_title(title + " Dendrogram (" + method + " linkage)")
    ax.set_xlabel("Sample Index")
    ax.set_ylabel("Distance")

    # Color the tick numbers based on control and schizophrenia subjects
    control_indices = data_control.index.to_list()
    schizophrenia_indices = data_schizophrenia.index.to_list()

    # Get the x-tick labels (leaf labels) from the dendrogram
    leaf_labels = ddata['leaves']

    # Iterate through x-ticks and color them based on the group
    for idx, label in enumerate(ax.get_xticklabels()):
        label_idx = leaf_labels[idx]
        if label_idx in control_indices:
            label.set_color('black')
        elif label_idx in schizophrenia_indices:
            label.set_color('red')

def plot_dendrogram_and_pca_with_correct_colored_ticks(band_data, ax_dendro, title, color_ticks=False, method='ward'):
    """
    Plot the dendrogram with optionally colored tick numbers and PCA visualization on the given axes.
    """
    # Hierarchical clustering
    Z = linkage(band_data, method=method)

    # Plot the dendrogram
    ddata = dendrogram(Z, ax=ax_dendro, leaf_rotation=90)
    ax.set_title(str(title) + " Dendrogram (" + str(method) + " linkage)")
    ax_dendro.set_xlabel("Sample Index")
    ax_dendro.set_ylabel("Distance")

    if color_ticks:
        # Color the tick numbers based on control and schizophrenia subjects
        control_indices = data_control.index.to_list()
        schizophrenia_indices = data_schizophrenia.index.to_list()

        # Get the x-tick labels (leaf labels) from the dendrogram
        leaf_labels = ddata['leaves']

        # Iterate through x-ticks and color them based on the group
        for idx, label in enumerate(ax_dendro.get_xticklabels()):
            label_idx = leaf_labels[idx]
            if label_idx in control_indices:
                label.set_color('black')
            elif label_idx in schizophrenia_indices:
                label.set_color('red')
    return Z

def plot_band_pca(band_data, Z, ax_pca, title):

    # Cut the dendrogram to obtain 3 clusters
    labels = fcluster(Z, t=n_dendro_clusters, criterion='maxclust')
    band_data['Cluster'] = labels

    # Use PCA to reduce the data to 2D
    pca = PCA(n_components=2)
    band_pca = pca.fit_transform(band_data.drop('Cluster', axis=1))
    # return band_pca

    # Create a scatter plot for PCA reduced data
    ax_pca.scatter(band_pca[:, 0], band_pca[:, 1], c=band_data['Cluster'], cmap='rainbow')
    ax_pca.set_title(title + " 2D PCA")
    ax_pca.set_xlabel("Principal Component 1")
    ax_pca.set_ylabel("Principal Component 2")

# If a CSV file is uploaded
if uploaded_file:
    st.write("Dataset loaded successfully!")

    # Load the data
    data = load_data(uploaded_file)

    # Split data into control and schizophrenia groups
    data_control = data[data['Group'] == 0]
    data_schizophrenia = data[data['Group'] == 1]
    data_full = data

    # Combined dendrogram for "All Subjects"
    all_bands_data = pd.concat([
        data.loc[:, data.columns.str.startswith('avpp_delta')],
        data.loc[:, data.columns.str.startswith('avpp_theta')],
        data.loc[:, data.columns.str.startswith('avpp_alpha')],
        data.loc[:, data.columns.str.startswith('avpp_beta')],
        data.loc[:, data.columns.str.startswith('avpp_gamma')]
    ], axis=1)

    fig, ax = plt.subplots(figsize=(16, 8))
    plot_dendrogram_colored_ticks(all_bands_data, ax, "All Bands Combined", method=selected_linkage_method)
    plt.tight_layout()

    # Save the dendrogram plot to a PNG file
    dendrogram_filename = "Combined_Dendrogram_plot.png"
    fig.savefig(dendrogram_filename, dpi=300)

    # Provide a download button for the dendrogram PNG file
    with open(dendrogram_filename, "rb") as f:
        btn = st.download_button(
            label="Download Combined Dendrogram Plot",
            data=f,
            file_name=dendrogram_filename,
            mime="image/png"
        )

    st.pyplot(fig)
    st.write("EDA - Exploratory Data Analysis")
    # Detect available bands from column names
    bands_list = ['delta', 'theta', 'alpha', 'beta', 'gamma']
    available_bands = [band for band in bands_list if any(data.columns.str.startswith(f'avpp_{band}'))]

    # Note: Replace all `plt.show()` with `st.pyplot()`
    # Create the plots with dendrogram, PCA, and UMAP visualizations
    nrows = 3 if show_pca and show_umap else 2 if show_pca or show_umap else 1 # Number of rows in the plot
    hight = 15 if show_pca and show_umap else 10 if show_pca or show_umap else 5 # Height of the plot
    for data_group, title in zip([data_schizophrenia, data_control, data_full], ["Schizophrenia", "Control", "All Subjects"]):
        fig, axes = plt.subplots(nrows=nrows, ncols=len(available_bands), figsize=(36, hight))
        fig.suptitle(title, fontsize=25)

        # Ensure axes is 2D
        if nrows == 1:
            axes = axes.reshape(1, -1)

        # Create band data based on detected bands for the current data group
        bands = [(band.capitalize(), data_group.loc[:, data_group.columns.str.startswith(f'avpp_{band}')]) for band in available_bands]

        # Configure the axes based on the selected visualizations
        axes_mapping = [0]  # dendrogram axes index is always 0
        if show_pca:
            axes_mapping.append(len(axes_mapping))
        if show_umap:
            axes_mapping.append(len(axes_mapping))

        # Plot dendrogram, PCA, and UMAP visualizations for each band
        for col, (band_name, band_data) in enumerate(bands):
            ax_dendro = axes[axes_mapping[0]][col]
            ax_dendro.set_title(band_name)
            color_ticks = True if title == "All Subjects" else False
            # Dendrogram plots using previous functions
            Z = plot_dendrogram_and_pca_with_correct_colored_ticks(band_data.copy(), ax_dendro, band_name, color_ticks, method=selected_linkage_method)

            if show_pca:
                ax_pca = axes[axes_mapping[1]][col]
                plot_band_pca(band_data.copy(), Z, ax_pca, title)

            if show_umap:
                ax_umap = axes[axes_mapping[-1]][col]
                embedding, kmeans_labels = umap_and_kmeans(band_data)
                plot_umap_embedding(embedding, kmeans_labels, ax_umap, band_name + " 2D UMAP")

        plt.tight_layout()
        plt.subplots_adjust(top=0.85)
        # Save the plot to a PNG file
        plot_filename = f"{title.replace(' ', '_')}_plot.png"
        fig.savefig(plot_filename, dpi=600)
        # plt.show()
        # st.pyplot()
        # st.image(plot_filename, use_column_width=True, clamp=True)
        st.pyplot(fig)
        plt.close(fig)

        # Provide a download button for the PNG file
        with open(plot_filename, "rb") as f:
            btn = st.download_button(
                label=f"Download {title} Plot",
                data=f,
                file_name=plot_filename,
                mime="image/png"
            )


import tensorflow as tf
import re
import time, datetime
import os
import data_helper
TOWER_NAME = 'tower'

class CNNClassify(object):
    """CNN图像分类
    """
    def __init__(self, batch_size, num_classes, num_train_examples, initial_lr=0.1, lr_decay_factor=0.1,
                 moving_average_decay=0.9999, num_epochs_per_decay=300, log_frequency=10,
                 max_steps=200000, checkpoint_every=5000, num_gpus=4, session_conf=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True,
                                           gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.3))):
        self.batch_size = batch_size
        self.num_classes = num_classes
        self.moving_average_decay = moving_average_decay  # 用于移动平均的衰减
        self.initial_lr = initial_lr    # 最初的学习速率
        self.lr_decay_factor = lr_decay_factor  # 学习速率衰减因子
        self.num_epochs_per_decay = num_epochs_per_decay  # 多少轮衰减一次
        self.num_train_examples = num_train_examples  # 训练样本数量
        self.log_frequency = log_frequency  # 多少步控制台打印一次结果
        self.max_steps = max_steps
        self.checkpoint_every = checkpoint_every  # 多少步之后保存一次模型
        self.num_checkpoints = 5
        self.num_gpus = num_gpus
        self.session_conf = session_conf

    def _variable_on_cpu(self, name, shape, initializer):
        """帮助创建存储在CPU内存上的变量。"""
        with tf.device('/cpu:0'):
            dtype = tf.float32
            var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype)
        return var

    def _variable_with_weight_decay(self, name, shape, stddev, wd):
        """初始化权重变量
        Args:
          name: name of the variable
          shape: list of ints
          stddev: 高斯函数标准差
          wd: 添加L2范数损失权重衰减系数。如果没有，该变量不添加重量衰减。
        Returns:权重变量
        """
        dtype = tf.float32
        var = self._variable_on_cpu(name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
        if wd is not None:
            weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
            tf.add_to_collection('losses', weight_decay)
        return var

    def _activation_summary(self, x):
        """创建tensorboard摘要 好可视化查看
        """
        tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name)
        tf.summary.histogram(tensor_name + '/activations', x)
        tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x))

    def average_gradients(self, tower_grads):
        """计算所有tower上所有变量的平均梯度
        """
        average_grads = []
        for grad_and_vars in zip(*tower_grads):
            # 每个梯度和变量类似这样:
            #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
            grads = []
            for g, _ in grad_and_vars:
                # 添加一个0维度来代表tower  [grad0_gpuN]
                expanded_g = tf.expand_dims(g, 0)

                # [[grad0_gpu1],...,[grad0_gpuN]]
                grads.append(expanded_g)

            # 在tower上进行平均  (上面加维度那部分没理解 加了又合 不是白操作吗？,后续再研究一下)
            grad = tf.concat(axis=0, values=grads)  # [grad0_gpu1,..., grad0_gpuN]
            grad = tf.reduce_mean(grad, 0)          # 平均梯度

            # 把变量拼接回去
            v = grad_and_vars[0][1]
            grad_and_var = (grad, v)
            average_grads.append(grad_and_var)
        return average_grads

    def inference(self, images):
        """向前传播
        """
        # 第一层卷积
        with tf.variable_scope('conv1') as scope:
            kernel = self._variable_with_weight_decay('weights', shape=[5, 5, 3, 64], stddev=5e-2, wd=0.0)  # 权值矩阵
            # 二维卷积
            conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')   #  周围补0 保持形状不变
            biases = self._variable_on_cpu('biases', [64], tf.constant_initializer(0.0))
            pre_activation = tf.nn.bias_add(conv, biases)
            conv1 = tf.nn.relu(pre_activation, name=scope.name)  # relu激活

            self._activation_summary(conv1)

        # pool1 最大池化
        pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1')
        # norm1 增加一个LRN处理,可以增强模型的泛化能力
        norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1')

        # 第二层卷积
        with tf.variable_scope('conv2') as scope:
            kernel = self._variable_with_weight_decay('weights', shape=[5, 5, 64, 64], stddev=5e-2, wd=0.0)
            conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = self._variable_on_cpu('biases', [64], tf.constant_initializer(0.1))
            pre_activation = tf.nn.bias_add(conv, biases)
            conv2 = tf.nn.relu(pre_activation, name=scope.name)

            self._activation_summary(conv2)

        # 这次先进行LRN处理
        norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2')
        # 最大池化
        pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2')

        # 全连接隐层 映射到384维向量
        with tf.variable_scope('local3') as scope:
            # 将前面的最大池化输出扁平化成一个单一矩阵 好做全连接
            reshape = tf.reshape(pool2, [self.batch_size, -1])
            dim = reshape.get_shape()[1].value
            weights = self._variable_with_weight_decay('weights', shape=[dim, 384], stddev=0.04, wd=0.004)
            biases = self._variable_on_cpu('biases', [384], tf.constant_initializer(0.1))
            local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)

            self._activation_summary(local3)

        # 再接一个全连接层 映射到192维向量
        with tf.variable_scope('local4') as scope:
            weights = self._variable_with_weight_decay('weights', shape=[384, 192], stddev=0.04, wd=0.004)
            biases = self._variable_on_cpu('biases', [192], tf.constant_initializer(0.1))
            local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name=scope.name)

            self._activation_summary(local4)

        # 线性输出层 这里不做softmax  因为在损失函数内部执行了，那样效率更高
        with tf.variable_scope('softmax_linear') as scope:
            weights = self._variable_with_weight_decay('weights', [192, self.num_classes], stddev=1 / 192.0, wd=0.0)
            biases = self._variable_on_cpu('biases', [self.num_classes], tf.constant_initializer(0.0))
            softmax_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name)

            self._activation_summary(softmax_linear)

        return softmax_linear

    def loss(self, logits, labels):
        """损失函数
        """
        # Calculate the average cross entropy loss across the batch.
        labels = tf.cast(labels, tf.int64)
        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
            labels=labels, logits=logits, name='cross_entropy_per_example')
        cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
        tf.add_to_collection('losses', cross_entropy_mean)

        # The total loss is defined as the cross entropy loss plus all of the weight
        # decay terms (L2 loss).
        return tf.add_n(tf.get_collection('losses'), name='total_loss')

    def tower_loss(self, scope, logits, labels):
        _ = self.loss(logits, labels)
        # 把所有损失都集中到当前tower上
        losses = tf.get_collection('losses', scope)
        total_loss = tf.add_n(losses, name='total_loss')
        for l in losses + [total_loss]:
            # 去掉变量名前缀 tower_[0-9],变成和单GPU的时候一样
            loss_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', l.op.name)
            tf.summary.scalar(loss_name, l)

        return total_loss

    def evaluation(self, logits, labels, k=1):
        """评估函数
        :param logits: 预测
        :param labels: 标签
        """
        correct = tf.nn.in_top_k(logits, labels, k=k)
        # correct = tf.equal(self.predictions, tf.argmax(labels, 1))
        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
        tf.add_to_collection('accuracy', accuracy)
        return tf.add_n(tf.get_collection('accuracy'), name='accuracy')

    def tower_evaluation(self, scope, logits, labels, k=1):
        """多gpu的评估函数
        """
        _ = self.evaluation(logits, labels, k)
        accuracy = tf.get_collection('accuracy', scope)
        total_accuracy = tf.reduce_mean(accuracy, axis=0, name='total_accuracy')
        return total_accuracy

    def _add_loss_summaries(self, total_loss):
        """增加损失摘要
        """
        # Compute the moving average of all individual losses and the total loss.
        loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
        losses = tf.get_collection('losses')
        loss_averages_op = loss_averages.apply(losses + [total_loss])

        # Attach a scalar summary to all individual losses and the total loss; do the
        # same for the averaged version of the losses.
        for l in losses + [total_loss]:
            # Name each loss as '(raw)' and name the moving average version of the loss
            # as the original loss name.
            tf.summary.scalar(l.op.name + ' (raw)', l)
            tf.summary.scalar(l.op.name, loss_averages.average(l))

        return loss_averages_op

    def train_operation(self, total_loss, global_step):
        """训练操作
        """
        num_batches_per_epoch = self.num_train_examples / self.batch_size   # 每轮的批次数
        decay_steps = int(num_batches_per_epoch * self.num_epochs_per_decay)   # 多少步衰减

        # 基于步数，以指数方式衰减学习率。
        lr = tf.train.exponential_decay(self.initial_lr, global_step, decay_steps, self.lr_decay_factor, staircase=True)
        tf.summary.scalar('learning_rate', lr)
        # 损失移动平均
        loss_averages_op = self._add_loss_summaries(total_loss)

        with tf.control_dependencies([loss_averages_op]):
            opt = tf.train.GradientDescentOptimizer(lr)  # 优化器
            grads = opt.compute_gradients(total_loss)    # 梯度

        # 应用梯度
        apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)   # 训练操作

        # 为可训练的变量添加直方图
        for var in tf.trainable_variables():
            tf.summary.histogram(var.op.name, var)

        # 为梯度添加直方图
        for grad, var in grads:
            if grad is not None:
                tf.summary.histogram(var.op.name + '/gradients', grad)

        # 跟踪所有可训练变量的移动平均线
        variable_averages = tf.train.ExponentialMovingAverage(self.moving_average_decay, global_step)
        variables_averages_op = variable_averages.apply(tf.trainable_variables())

        with tf.control_dependencies([apply_gradient_op, variables_averages_op]):
            train_op = tf.no_op(name='train')

        return train_op

    def train_step(self, sess, summary_writer):
        """单步训练
        """
        _, step, cur_loss, cur_acc = sess.run([self.train_op, self.global_step, self._loss, self.accuracy])
        time_str = datetime.datetime.now().isoformat()
        print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, cur_loss, cur_acc))
        # 存储摘要
        if step % 100 == 0:
            summary_str = sess.run(self.summary)
            summary_writer.add_summary(summary_str, step)
            summary_writer.flush()

    def train(self, filename, out_dir):
        """训练
        """
        with tf.Graph().as_default():
            sess = tf.Session(config=self.session_conf)
            with sess.as_default():

                self.global_step = tf.contrib.framework.get_or_create_global_step()

                with tf.device('/cpu:0'):
                    images, labels = data_helper.distorted_inputs(filename, self.batch_size)
                logits = self.inference(images)
                self._loss = self.loss(logits, labels)
                self.train_op = self.train_operation(self._loss, self.global_step)
                self.accuracy = self.evaluation(logits, labels)
                self.summary = tf.summary.merge_all()

                # 保存点设置
                checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
                checkpoint_prefix = os.path.join(checkpoint_dir, "model")  # 模型存储前缀
                if not os.path.exists(checkpoint_dir):
                    os.makedirs(checkpoint_dir)
                saver = tf.train.Saver(tf.global_variables(), max_to_keep=self.num_checkpoints)
                summary_writer = tf.summary.FileWriter(out_dir + "/summary", sess.graph)

                # 初始化所有变量
                ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
                if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
                    saver.restore(sess, ckpt.model_checkpoint_path)
                else:
                    sess.run(tf.global_variables_initializer())

                tf.train.start_queue_runners(sess=sess)

                for step in range(self.max_steps):
                    self.train_step(sess, summary_writer)  # 训练
                    cur_step = tf.train.global_step(sess, self.global_step)
                    # checkpoint_every 次迭代之后 保存模型
                    if cur_step % self.checkpoint_every == 0 and cur_step != 0:
                        path = saver.save(sess, checkpoint_prefix, global_step=cur_step)
                        print("Saved model checkpoint to {}\n".format(path))

    def multi_gpu_train(self, filename, out_dir):
        with tf.Graph().as_default(), tf.device('/cpu:0'):
            sess = tf.Session(config=self.session_conf)
            with sess.as_default():
                # Create a variable to count the number of train() calls. This equals the
                # number of batches processed * FLAGS.num_gpus.
                self.global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)

                # 学习速率衰减设置
                num_batches_per_epoch = self.num_train_examples / self.batch_size
                decay_steps = int(num_batches_per_epoch * self.num_epochs_per_decay)

                # 根据步数衰减学习速率
                lr = tf.train.exponential_decay(self.initial_lr, self.global_step, decay_steps, self.lr_decay_factor,
                                                staircase=True)
                # 执行梯度下降的优化器
                opt = tf.train.GradientDescentOptimizer(lr)

                images, labels = data_helper.distorted_inputs(filename, self.batch_size)  # 取出数据
                # 批次队列 这个函数不是很懂
                batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue([images, labels], capacity=2 * self.num_gpus)

                tower_grads = []
                summaries = None
                with tf.variable_scope(tf.get_variable_scope()):
                    for i in range(self.num_gpus):
                        with tf.device('/gpu:{}'.format(i)):
                            with tf.name_scope('{}_{}'.format(TOWER_NAME, i)) as scope:
                                # 为gpu列出一个批次
                                image_batch, label_batch = batch_queue.dequeue()
                                # 计算一个tower的损失. 并且每个tower共享权重变量
                                logits = self.inference(image_batch)
                                self._loss = self.tower_loss(scope, logits, label_batch)
                                self.accuracy = self.tower_evaluation(scope, logits, label_batch)
                                # 下一个tower复用变量
                                tf.get_variable_scope().reuse_variables()

                                # 保存最终tower的摘要
                                summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)

                                # 计算梯度
                                grads = opt.compute_gradients(self._loss)

                                # 跟踪所有tower的梯度
                                tower_grads.append(grads)

                grads = self.average_gradients(tower_grads)  # 平均梯度

                # 添加学习速率的摘要
                summaries.append(tf.summary.scalar('learning_rate', lr))

                # 添加梯度直方图
                for grad, var in grads:
                    if grad is not None:
                        summaries.append(tf.summary.histogram(var.op.name + '/gradients', grad))

                # 应用梯度来调整共享变量
                apply_gradient_op = opt.apply_gradients(grads, global_step=self.global_step)

                # 所有可训练变量添加直方图
                for var in tf.trainable_variables():
                    summaries.append(tf.summary.histogram(var.op.name, var))

                # 跟踪所有可训练变量的移动平均线
                variable_averages = tf.train.ExponentialMovingAverage(self.moving_average_decay, self.global_step)
                variables_averages_op = variable_averages.apply(tf.trainable_variables())

                # 将所有更新集中到一个训练操作
                self.train_op = tf.group(apply_gradient_op, variables_averages_op)

                # 从最后的tower总结摘要
                self.summary = tf.summary.merge(summaries)

                # 保存点设置
                checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
                checkpoint_prefix = os.path.join(checkpoint_dir, "model")  # 模型存储前缀
                if not os.path.exists(checkpoint_dir):
                    os.makedirs(checkpoint_dir)
                saver = tf.train.Saver(tf.global_variables(), max_to_keep=self.num_checkpoints)
                summary_writer = tf.summary.FileWriter(out_dir + "/summary", sess.graph)

                # 初始化所有变量
                ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
                if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
                    saver.restore(sess, ckpt.model_checkpoint_path)
                else:
                    sess.run(tf.global_variables_initializer())

                # 启动队列
                tf.train.start_queue_runners(sess=sess)

                for step in range(self.max_steps):
                    self.train_step(sess, summary_writer)  # 训练
                    cur_step = tf.train.global_step(sess, self.global_step)
                    # checkpoint_every 次迭代之后 保存模型
                    if cur_step % self.checkpoint_every == 0 and cur_step != 0:
                        path = saver.save(sess, checkpoint_prefix, global_step=cur_step)
                        print("Saved model checkpoint to {}\n".format(path))


"""MYAPP Core application logic."""
from json import (
    JSONDecoder,
    JSONEncoder,
    loads as _json_loads,
)
from logging import getLogger
from pathlib import PosixPath
from http import HTTPStatus

from flask import Blueprint, current_app, request, Response
from flask.views import MethodView
from webargs.flaskparser import FlaskParser
from marshmallow import Schema, fields, pre_dump, RAISE, EXCLUDE

__all__ = [
    'APP_PATH',
    'APIMethodView',
    'APIBlueprint',
    'APIError',
    'APIRequestSchema',
    'APIResponseSchema',
    'APIMetadataSchema',
    'JSONEncoder',
    'JSONDecoder',
    'json_dump',
    'json_dumps',
    'json_loads',
    'parse',
    'log_request',
    'log_response',
]

LOG = getLogger(__name__)

# ----------------------------------CONSTANTS----------------------------------
APP_PATH = PosixPath(__file__).parent
# ----------------------------------CONSTANTS----------------------------------

# -------------------------------WEBARGS SETTINGS-------------------------------
class APIRequestParser(FlaskParser):
    def handle_error(self, error, req, schema, *, error_status_code, error_headers):
        raise APIError(
            'The request specification is invalid; check OpenAPI docs for more info.',
            metadata={'errors': error.messages},
            http_status=error_status_code or HTTPStatus.OK,
        )

    def parse_files(self, req, name, field):
        raise NotImplementedError

parser = APIRequestParser()
parse = parser.use_args
# -------------------------------WEBARGS SETTINGS-------------------------------

# --------------------------------SERIALIZATION--------------------------------
class APIRequestSchema(Schema):
    """MYAPP base request schema."""

    class Meta:
        """Raise on unknown parameters."""

        unknown = RAISE

class APICommonRequestSchema(Schema):
    """MYAPP common request parameters."""

    class Meta:
        """Do not react on unknown parameters."""

        unknown = EXCLUDE

    debug_tb_enabled = fields.Boolean(
        required=False,
        default=False,
    )

class APIResponseSchema(Schema):
    """MYAPP base response schema."""

    class Meta:
        """Exclude any unknown parameters."""

        unknown = EXCLUDE

    data = fields.Dict(
        required=True,
        default=dict,
    )

    metadata = fields.Nested(
        'APIMetadataSchema',
        required=True,
    )

    @classmethod
    def default_metadata(cls):
        """
        Create default metadata.

        :return: metadata fallback
        """
        return {
            'status': 0,
            'message': 'Nice',
            'headers': {},
            'errors': None,
            'details': None,
        }

    @pre_dump
    def pre_dump(self, response, many=None):
        """
        Make pre dump handling.

        :param response: raw response
        :param many: is many
        :return: enriched raw response
        """
        _ = many
        metadata = self.default_metadata()

        response_metadata = response.get('metadata', {})
        for field in 'status', 'message', 'headers', 'errors', 'details':
            if field in response_metadata:
                metadata[field] = response_metadata[field]

        # FIXME: dynamic messages
        if metadata['status'] and metadata['message'] == 'Nice':
            metadata['message'] = 'Not nice'

        response['metadata'] = metadata
        return response

class APIMetadataSchema(Schema):
    """MYAPP Metadata schema."""

    status = fields.Integer(
        required=True,
        default=0,
    )
    message = fields.String(
        required=True,
        default='Nice',
    )
    headers = fields.Dict(
        required=True,
        default=dict,
    )
    errors = fields.Dict(
        required=True,
        allow_none=True,
        default=None,
    )
    details = fields.Dict(
        required=True,
        allow_none=True,
        default=None,
    )
# --------------------------------SERIALIZATION--------------------------------

# ------------------------FLASK AND APPLICATION GENERICS------------------------
class APIJSONEncoder(JSONEncoder):
    """MYAPP JSON Encoder."""

    def __init__(
        self,
        *,
        skipkeys=False,
        check_circular=True,
        allow_nan=True,
        separators=None,
        default=None,
    ):
        """
        Initialize encoder.

        :param skipkeys: is skip
        :param check_circular: is check circular
        :param allow_nan: is allow nan
        :param separators: separator char
        :param default: default value
        """
        ensure_ascii = current_app.config['JSON_ENSURE_ASCII']
        sort_keys = current_app.config['JSON_SORT_KEYS']
        indent = current_app.config['JSON_INDENT']

        super().__init__(
            skipkeys=skipkeys,
            ensure_ascii=ensure_ascii,
            check_circular=check_circular,
            allow_nan=allow_nan,
            sort_keys=sort_keys,
            indent=indent,
            separators=separators,
            default=default,
        )

class APIJSONDecoder(JSONDecoder):
    """MYAPP JSON Decoder."""

def json_dumps(obj, **kwargs):
    """
    MYAPP json dumps.

    :param obj: object
    :param kwargs: any
    :return: json string
    """
    return APIJSONEncoder(**kwargs).encode(obj)

def json_dump(obj, file, **kwargs):
    """
    MYAPP json dump.

    :param obj: python object
    :param file: filename
    :param kwargs: any
    """
    for chunk in APIJSONEncoder(**kwargs).iterencode(obj):
        file.write(chunk)

def json_loads(string, **kwargs):
    """
    MYAPP json loads.

    :param string: json string
    :param kwargs: any
    :return: dict
    """
    return _json_loads(string, cls=APIJSONDecoder, **kwargs)

class APIMethodView(MethodView):
    """API Method View."""

    decorators = (
        parse(APICommonRequestSchema(), location='query'),
    )

class APIBlueprint(Blueprint):
    """API Blueprint."""

def log_request():
    """Log request in curl-based fashion."""
    msg = fr"curl -w '\n' -iX {request.method} '{request.url}' "
    msg += ''.join(f"-H '{h}:{v}' " for h, v in request.headers.items())
    if (
        request.method in {'POST', 'PUT', 'PATCH'}
        and request.headers.get('Content-Type') == 'application/json'
    ):
        msg += f"-d '{request.data.decode('utf8')}'"
    LOG.info(msg)

def log_response(response: Response):
    """
    Log response json.

    :param response: flask response
    :return: flask response
    """
    if response.is_json:
        LOG.info(f'Response: {response.json}')
    return response
# ------------------------FLASK AND APPLICATION GENERICS------------------------

# ---------------------------EXCEPTIONS AND MESSAGES---------------------------
class APIError(Exception):
    """Base API Exception."""

    def __init__(self, *args, **kwargs):
        """
        Initialize API exception.

        :param args: any
        :param kwargs: any
        """
        schema = kwargs.pop('schema', APIResponseSchema())
        data = kwargs.pop('data', {})
        metadata = kwargs.pop('metadata', {})
        metadata.setdefault('message', 'Error' if not args else args[0])
        metadata.setdefault('status', 3)
        self.json = schema.dump({'data': data, 'metadata': metadata})
        self.http_status = kwargs.pop('http_status', HTTPStatus.OK)

        super().__init__(*args)
# ---------------------------EXCEPTIONS AND MESSAGES---------------------------


import os, sys
import subprocess

# os.environ['DISPLAY'] = ':99.0'
# os.environ['PYVISTA_OFF_SCREEN'] = 'true'
# os.environ['PYVISTA_USE_IPYVTK'] = 'true'
# bashCommand ="Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & sleep 3"
# process = subprocess.Popen(bashCommand, stdout=subprocess.PIPE, shell=True)
# process.wait()
sys.path.insert(0, os.path.abspath("../../../.."))
from copy import deepcopy
import numpy as np
import torch
import pyvista as pv
import matplotlib.pyplot as plt
from shapmagn.global_variable import Shape, shape_type
from shapmagn.datasets.data_utils import read_json_into_list, get_obj, get_file_name
from shapmagn.shape.shape_pair_utils import create_shape_pair
from shapmagn.utils.obj_factory import obj_factory
from shapmagn.utils.visualizer import (
    visualize_point_fea,
    visualize_point_pair,
    visualize_multi_point,
)
from shapmagn.utils.local_feature_extractor import *

def get_pair(source_path, target_path, expand_bch_dim=True, return_tensor=True):

    get_obj_func = get_obj(
        reader_obj,
        normalizer_obj,
        sampler_obj,
        device,
        expand_bch_dim=expand_bch_dim,
        return_tensor=return_tensor,
    )
    source_obj, source_interval = get_obj_func(source_path)
    target_obj, target_interval = get_obj_func(target_path)
    return source_obj, target_obj

def plot_pair_weight_distribution(
    source_weight, target_weight, use_log=False, title="", show=True, save_path=None
):
    plt.style.use("bmh")
    fig, ax = plt.subplots()
    source_weight = np.log(source_weight) if use_log else source_weight
    target_weight = np.log(target_weight) if use_log else target_weight
    ax.hist(source_weight, bins=1000, density=0, histtype="stepfilled", alpha=0.7)
    ax.hist(target_weight, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    title += "weight" if not use_log else "log_weight"
    ax.set_title(title)
    if show:
        plt.show()
    if save_path:
        plt.savefig(save_path, dpi=300)
    plt.clf()

def plot_pair_weight_distribution_before_and_after_radius_matching(
    source_weight1,
    target_weight1,
    source_weight2,
    target_weight2,
    use_log=False,
    title="",
    show=True,
    save_path=None,
):
    plt.style.use("bmh")

    fig, axes = plt.subplots(nrows=2, ncols=2)
    ax0, ax1, ax2, ax3 = axes.flatten()
    source_weight_matched1 = matching_np_radius(source_weight1, target_weight1)
    smw_sum1, sw_sum1, tp_sum1 = (
        source_weight_matched1.sum(),
        source_weight1.sum(),
        target_weight1.sum(),
    )

    source_weight1 = np.log(source_weight1) if use_log else source_weight1
    target_weight1 = np.log(target_weight1) if use_log else target_weight1
    ax0.hist(source_weight1, bins=1000, density=0, histtype="stepfilled", alpha=0.7)
    ax0.hist(target_weight1, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax0.set_title("sw_sum: {:.3f}, tp_sum:{:.3f}".format(sw_sum1, tp_sum1), fontsize=10)
    source_weight_matched1_norm = (
        np.log(source_weight_matched1) if use_log else source_weight_matched1
    )
    ax1.hist(
        source_weight_matched1_norm,
        bins=1000,
        density=0,
        histtype="stepfilled",
        alpha=0.7,
    )
    ax1.hist(target_weight1, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax1.set_title(
        "smw_sum: {:.3f}, tp_sum:{:.3f}".format(smw_sum1, tp_sum1), fontsize=10
    )

    source_weight_matched2 = matching_np_radius(source_weight2, target_weight2)
    smw_sum2, sw_sum2, tp_sum2 = (
        source_weight_matched2.sum(),
        source_weight2.sum(),
        target_weight2.sum(),
    )
    source_weight2 = np.log(source_weight2) if use_log else source_weight2
    target_weight2 = np.log(target_weight2) if use_log else target_weight2
    ax2.hist(source_weight2, bins=1000, density=0, histtype="stepfilled", alpha=0.7)
    ax2.hist(target_weight2, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax2.set_title("sw_sum: {:.3f}, tp_sum:{:.3f}".format(sw_sum2, tp_sum2), fontsize=10)
    source_weight_matched2_norm = (
        np.log(source_weight_matched2) if use_log else source_weight_matched2
    )
    ax3.hist(
        source_weight_matched2_norm,
        bins=1000,
        density=0,
        histtype="stepfilled",
        alpha=0.7,
    )
    ax3.hist(target_weight2, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax3.set_title(
        "smw_sum: {:.3f}, tp_sum:{:.3f}".format(smw_sum2, tp_sum2), fontsize=10
    )

    fig.subplots_adjust(hspace=0.3)
    fig.suptitle(title)
    if show:
        plt.show()
    if save_path:
        plt.savefig(save_path, dpi=300)
    plt.clf()
    return source_weight_matched1, source_weight_matched2

def get_half_lung(lung, normalize_weight=False):
    weights = lung.weights.detach()
    points = lung.points.detach()
    pos_filter = points[..., 0] < 0
    points = points[pos_filter][None]
    weights = weights[pos_filter][None]
    weights = weights
    weights = weights / weights.sum() if normalize_weight else weights
    half_lung = Shape()
    half_lung.set_data(points=points, weights=weights)
    return half_lung

def get_key_vessel(lung, thre=2e-05):
    weights = lung.weights.detach()
    points = lung.points.detach()
    mask = (lung.weights > thre)[..., 0]
    weights = weights[mask][None]
    points = points[mask][None]
    key_lung = Shape()
    key_lung.set_data(points=points, weights=weights)
    return key_lung

def sampled_via_radius(source, target):
    min_npoints = min(source.npoints, target.npoints)
    tw = target.weights.squeeze()
    sw = source.weights.squeeze()
    t_sorted, t_indices = torch.sort(tw, descending=True)
    s_sorted, s_indices = torch.sort(sw, descending=True)
    t_sampled_indices = t_indices[:min_npoints]
    s_sampled_indices = s_indices[:min_npoints]
    tp_sampled = target.points[:, t_sampled_indices]
    sp_sampled = source.points[:, s_sampled_indices]
    tw_sampled = target.weights[:, t_sampled_indices]
    sw_sampled = source.weights[:, s_sampled_indices]
    target_sampled, source_sampled = Shape(), Shape()
    target_sampled.set_data(points=tp_sampled, weights=tw_sampled)
    source_sampled.set_data(points=sp_sampled, weights=sw_sampled)
    return source_sampled, target_sampled

def hist_match(source, template):
    """
    Adjust the pixel values of a grayscale image such that its histogram
    matches that of a target image.
    Code adapted from
    http://stackoverflow.com/questions/32655686/histogram-matching-of-two-images-in-python-2-x
    Arguments:
    -----------
        source: np.ndarray
            Image to transform; the histogram is computed over the flattened
            array
        template: np.ndarray
            Template image; can have different dimensions to source
    Returns:
    -----------
        matched: np.ndarray
            The transformed output image
    """

    oldshape = source.shape
    source = source.ravel()
    template = template.ravel()

    # get the set of unique pixel values and their corresponding indices and
    # counts
    s_values, bin_idx, s_counts = np.unique(
        source, return_inverse=True, return_counts=True
    )
    t_values, t_counts = np.unique(template, return_counts=True)

    # take the cumsum of the counts and normalize by the number of pixels to
    # get the empirical cumulative distribution functions for the source and
    # template images (maps pixel value --> quantile)
    s_quantiles = np.cumsum(s_counts).astype(np.float64)
    s_quantiles /= s_quantiles[-1]
    t_quantiles = np.cumsum(t_counts).astype(np.float64)
    t_quantiles /= t_quantiles[-1]

    # interpolate linearly to find the pixel values in the template image
    # that correspond most closely to the quantiles in the source image
    interp_t_values = np.interp(s_quantiles, t_quantiles, t_values)

    return interp_t_values[bin_idx].reshape(oldshape)

def matching_np_radius(source_weights, target_weights):
    """

    :param source_weights: Nx1
    :param target_weights: Mx1
    :param matched_weights: Nx1
    :return:
    """

    ns = source_weights.shape[0]
    sw = source_weights.squeeze()
    tw = target_weights.squeeze()
    range = [min(sw.min(), tw.min()), max(sw.max(), tw.max())]
    resol = 10000
    interp = (range[1] - range[0]) / resol
    bins = np.linspace(range[0] - 2 * interp, range[1] + 2 * interp, resol)
    sw_indice = np.digitize(sw, bins, right=False)
    tw_indice = np.digitize(tw, bins, right=False)
    sw_digitize = bins[sw_indice]
    tw_digitize = bins[tw_indice]
    sw_transformed = hist_match(sw_digitize, tw_digitize)
    return sw_transformed.reshape(ns, 1).astype(np.float32)

def matching_shape_radius(source, target, sampled_by_radius=False, show=True):
    if sampled_by_radius:
        source, target = sampled_via_radius(source, target)
    device = source.points.device
    sn = source.npoints
    tn = target.npoints
    sw = source.weights.squeeze().cpu().numpy()
    tw = target.weights.squeeze().cpu().numpy()
    range = [min(sw.min(), tw.min()), max(sw.max(), tw.max())]
    resol = 10000
    interp = (range[1] - range[0]) / resol
    bins = np.linspace(range[0] - 2 * interp, range[1] + 2 * interp, resol)
    sw_indice = np.digitize(sw, bins, right=False)
    tw_indice = np.digitize(tw, bins, right=False)
    sw_digitize = bins[sw_indice]
    tw_digitize = bins[tw_indice]
    sw_transformed = hist_match(sw_digitize, tw_digitize)
    if show:
        plot_pair_weight_distribution(sw_digitize, tw_digitize, use_log=True)
        plot_pair_weight_distribution(sw_transformed, tw_digitize, use_log=True)
        visualize_point_pair(
            source.points,
            target.points,
            source.weights,
            target.weights,
            title1="source(before)",
            title2="target(before)",
        )

        visualize_point_pair(
            source.points,
            target.points,
            sw_transformed,
            tw_digitize,
            title1="source(after)",
            title2="target(after)",
        )
    source.weights = (
        torch.tensor(sw_transformed.astype(np.float32)).to(device).view(1, sn, 1)
    )
    target.weights = (
        torch.tensor(tw_digitize.astype(np.float32)).to(device).view(1, tn, 1)
    )
    return source, target

def source_weight_transform(weights, compute_on_half_lung=False):
    weights = weights * 1
    weights_cp = deepcopy(weights)
    thre = 1.9e-05
    thre = thre  # if not compute_on_half_lung else thre*2
    weights[weights_cp < thre] = 1e-7
    return weights

def flowed_weight_transform(weights, compute_on_half_lung=False):
    weights = weights * 1
    weights_cp = deepcopy(weights)
    thre = 1.9e-05
    thre = thre  # if not compute_on_half_lung else thre * 2
    weights[weights_cp < thre] = 1e-7
    return weights

def target_weight_transform(weights, compute_on_half_lung=False):
    weights = weights * 1
    weights_cp = deepcopy(weights)
    thre = 1.9e-05
    thre = thre  # if not compute_on_half_lung else thre * 2
    weights[weights_cp < thre] = 1e-7
    # weights[weights_cp > 1.1e-05] = 1e-7
    return weights

def pair_shape_transformer(init_thres=2.9e-5, nstep=5):
    # todo the next step of the transformer is to return a smoothed mask to constrain the movement of the lung
    def transform(source, target, cur_step):
        min_weights = min(torch.min(source.weights), torch.min(target.weights))
        max_weights = min(torch.max(source.weights), torch.max(target.weights))
        max_weights = max_weights.item()
        cur_step = cur_step.item()
        assert init_thres > min_weights
        thres = init_thres - (init_thres - min_weights) / nstep * cur_step
        s_weights = source.weights.clone()
        t_weights = target.weights.clone()
        s_weights[source.weights < thres] = 1e-7
        t_weights[target.weights < thres] = 1e-7
        s_transformed, t_transformed = Shape(), Shape()
        s_transformed.set_data(
            points=source.points, weights=s_weights, pointfea=source.pointfea
        )
        t_transformed.set_data(
            points=target.points, weights=t_weights, pointfea=target.pointfea
        )
        print("the weight of the lung pair are updated")
        return s_transformed, t_transformed

    return transform

def capture_plotter(save_source=False):
    from shapmagn.utils.visualizer import visualize_point_pair_overlap

    inner_count = 0

    def save(record_path, name_suffix, shape_pair):
        nonlocal inner_count
        source, flowed, target = shape_pair.source, shape_pair.flowed, shape_pair.target
        for sp, fp, tp, sw, fw, tw, pair_name in zip(
            source.points,
            flowed.points,
            target.points,
            source.weights,
            flowed.weights,
            target.weights,
            pair_name_list,
        ):
            if inner_count == 0 or save_source:
                path = os.path.join(
                    record_path, "source_target" + "_" + name_suffix + ".png"
                )
                visualize_point_pair_overlap(
                    sp,
                    tp,
                    flowed_weight_transform(fw, True),
                    target_weight_transform(tw, True),
                    title1="source",
                    title2="target",
                    rgb_on=False,
                    saving_capture_path=path,
                    show=False,
                )
            path_1 = os.path.join(
                record_path,
                pair_name + "_flowed_target" + "_main_" + name_suffix + ".png",
            )
            path_2 = os.path.join(
                record_path,
                pair_name + "_flowed_target" + "_whole_" + name_suffix + ".png",
            )
            visualize_point_pair_overlap(
                fp,
                tp,
                flowed_weight_transform(fw, True),
                target_weight_transform(tw, True),
                title1="flowed",
                title2="target",
                rgb_on=False,
                saving_capture_path=path_1,
                show=False,
            )
            visualize_point_pair_overlap(
                fp,
                tp,
                fw,
                tw,
                title1="flowed",
                title2="target",
                rgb_on=False,
                saving_capture_path=path_2,
                show=False,
            )
        inner_count += 1

    return save

def lung_isolated_leaf_clean_up(
    lung, radius=0.032, principle_weight=None, normalize_weights=True
):

    points = lung.points.detach()
    weights = lung.weights.detach()
    mass, dev, cov = compute_local_moments(points, radius=radius)
    eigenvector_main = compute_local_fea_from_moments(
        "eigenvector_main", weights, mass, dev, cov
    )
    filter = mass[..., 0].squeeze() > 2
    to_remove = ~filter
    print(
        "In the first step, num of points are removed {}, {}".format(
            torch.sum(to_remove), torch.sum(to_remove) / len(filter)
        )
    )
    points_toremove = points[:, to_remove]
    mass_toremove = mass[:, to_remove]
    mass = mass[:, filter]
    points = points[:, filter]
    weights = weights[:, filter]
    eigenvector_main = eigenvector_main[:, filter]
    visualize_point_fea_with_arrow(points, mass, eigenvector_main * 0.01, rgb_on=False)
    visualize_point_overlap(
        points,
        points_toremove,
        mass,
        mass_toremove,
        title="cleaned points",
        point_size=(10, 20),
        rgb_on=False,
        opacity=("linear", 1.0),
    )

    Gamma = compute_anisotropic_gamma_from_points(
        points,
        cov_sigma_scale=radius,
        aniso_kernel_scale=radius,
        principle_weight=principle_weight,
    )
    mass, dev, cov = compute_aniso_local_moments(points, Gamma)
    eigenvector_main = compute_local_fea_from_moments(
        "eigenvector_main", weights, mass, dev, cov
    )
    filter = mass[..., 0].squeeze() > 2.5
    to_remove = ~filter
    print(
        "In the second step, num of points are removed {}, {}".format(
            torch.sum(to_remove), torch.sum(to_remove) / len(filter)
        )
    )
    points_toremove = points[:, to_remove]
    mass_toremove = mass[:, to_remove]
    mass = mass[:, filter]
    points = points[:, filter]
    weights = weights[:, filter]
    eigenvector_main = eigenvector_main[:, filter]
    visualize_point_fea_with_arrow(points, mass, eigenvector_main * 0.01, rgb_on=False)
    visualize_point_overlap(
        points,
        points_toremove,
        mass,
        mass_toremove,
        title="cleaned points",
        point_size=(10, 20),
        rgb_on=False,
        opacity=("linear", 1.0),
    )

    Gamma = compute_anisotropic_gamma_from_points(
        points,
        cov_sigma_scale=radius,
        aniso_kernel_scale=radius,
        principle_weight=principle_weight,
    )
    mass, dev, cov = compute_aniso_local_moments(points, Gamma)
    eigenvector_main = compute_local_fea_from_moments(
        "eigenvector_main", weights, mass, dev, cov
    )
    filter = mass[..., 0].squeeze() > 3
    to_remove = ~filter
    print(
        "In the third step, num of points are removed {}, {}".format(
            torch.sum(to_remove), torch.sum(to_remove) / len(filter)
        )
    )
    points_toremove = points[:, to_remove]
    mass_toremove = mass[:, to_remove]
    mass = mass[:, filter]
    points = points[:, filter]
    weights = weights[:, filter]
    eigenvector_main = eigenvector_main[:, filter]
    visualize_point_fea_with_arrow(points, mass, eigenvector_main * 0.01, rgb_on=False)
    visualize_point_overlap(
        points,
        points_toremove,
        mass,
        mass_toremove,
        title="cleaned points",
        point_size=(10, 20),
        rgb_on=False,
        opacity=("linear", 1.0),
    )

    cleaned_lung = Shape()
    cleaned_lung.points, cleaned_lung.weights = (
        points,
        weights / torch.sum(weights) if normalize_weights else weights,
    )
    return cleaned_lung

def analysis_large_vessel(
    source,
    target,
    source_weight_transform=source_weight_transform,
    target_weight_transform=target_weight_transform,
    title1="source",
    title2="target",
):
    source_points, source_weights, = (
        source.points.detach().cpu(),
        source.weights.detach().cpu(),
    )
    target_points, target_weights, = (
        target.points.detach().cpu(),
        target.weights.detach().cpu(),
    )
    plot_pair_weight_distribution(
        source_weight_transform(source_weights).squeeze().numpy(),
        target_weight_transform(target_weights).squeeze().numpy(),
        use_log=True,
    )
    visualize_point_pair(
        source_points,
        target_points,
        source_weight_transform(source_weights),
        target_weight_transform(target_weights),
        title1=title1,
        title2=title2,
    )

def compute_atlas(weight_list):
    atlas_weight = np.concatenate(weight_list)
    return atlas_weight

def transfer_radius_and_save_sample(
    cur_obj, atlas_distri, radius_transfered_saing_path
):
    cur_obj["weights"] = matching_np_radius(cur_obj["weights"], atlas_distri)
    data = pv.PolyData(cur_obj["points"])
    for key, item in cur_obj.items():
        if key not in ["points"]:
            data.point_arrays[key] = item
    data.save(radius_transfered_saing_path)
    return cur_obj

if __name__ == "__main__":
    assert (
        shape_type == "pointcloud"
    ), "set shape_type = 'pointcloud'  in global_variable.py"
    device = torch.device("cpu")  # cuda:0  cpu
    reader_obj = "lung_dataloader_utils.lung_reader()"
    normalizer_obj = (
        "lung_dataloader_utils.lung_normalizer(weight_scale=60000,scale=[100,100,100])"
    )
    phase = "train"

    use_local_mount = False
    remote_mount_transfer = lambda x: x.replace(
        "/playpen-raid1", "/home/zyshen/remote/llr11_mount"
    )
    path_transfer = (
        (lambda x: remote_mount_transfer(x)) if use_local_mount else (lambda x: x)
    )

    dataset_json_path = (
        "/playpen-raid1/zyshen/data/lung_expri/{}/pair_data.json".format(phase)
    )
    dataset_json_path = path_transfer(dataset_json_path)

    sampler_obj = "lung_dataloader_utils.lung_sampler( method='voxelgrid',scale=0.0003)"
    get_obj_func = get_obj(
        reader_obj,
        normalizer_obj,
        sampler_obj,
        device,
        expand_bch_dim=False,
        return_tensor=False,
    )
    altas_path = "/playpen-raid1/Data/UNC_vesselParticles/10067M_INSP_STD_MSM_COPD_wholeLungVesselParticles.vtk"
    altas_path = path_transfer(altas_path)
    atlas, _ = get_obj_func(altas_path)
    sampler_obj = "lung_dataloader_utils.lung_sampler( method='combined',scale=0.0003,num_sample=30000,sampled_by_weight=True)"
    get_obj_func = get_obj(
        reader_obj,
        normalizer_obj,
        sampler_obj,
        device,
        expand_bch_dim=False,
        return_tensor=False,
    )
    sampled_atlas, _ = get_obj_func(altas_path)

    radius_transfered_saing_path = "/playpen-raid1/zyshen/data/lung_atlas/{}".format(
        phase
    )
    radius_transfered_saing_path = path_transfer(radius_transfered_saing_path)
    os.makedirs(radius_transfered_saing_path, exist_ok=True)

    pair_name_list, pair_info_list = read_json_into_list(dataset_json_path)
    pair_path_list = [
        [pair_info["source"]["data_path"], pair_info["target"]["data_path"]]
        for pair_info in pair_info_list
    ]
    pair_id = 3
    output_path = "/playpen-raid1/zyshen/data/lung_data_analysis/val"
    for pair_id in range(len(pair_name_list)):
        pair_path = pair_path_list[pair_id]
        pair_path = [path_transfer(path) for path in pair_path]
        sampler_obj = (
            "lung_dataloader_utils.lung_sampler( method='voxelgrid',scale=0.0003)"
        )

        ########################
        plot_saving_path = os.path.join(radius_transfered_saing_path, "origin_plots")
        os.makedirs(plot_saving_path, exist_ok=True)

        source_path, target_path = pair_path_list[pair_id]
        source, target = get_pair(
            source_path, target_path, expand_bch_dim=False, return_tensor=False
        )

        saving_path = os.path.join(plot_saving_path, pair_name_list[pair_id] + ".png")
        camera_pos = [
            (-4.924379645467042, 2.17374925796456, 1.5003730890759344),
            (0.0, 0.0, 0.0),
            (0.40133888001174545, 0.31574165540339943, 0.8597873634998591),
        ]
        visualize_point_pair(
            source["points"],
            target["points"],
            source["weights"],
            target["weights"],
            title1="source",
            title2="target",
            saving_capture_path=saving_path,
            camera_pos=camera_pos,
            show=False,
        )
        plot_saving_path = os.path.join(radius_transfered_saing_path, "plots")
        os.makedirs(plot_saving_path, exist_ok=True)

        # vtk_saving_path = os.path.join(radius_transfered_saing_path,"data")
        # os.makedirs(vtk_saving_path,exist_ok=True)
        # saving_path = os.path.join(vtk_saving_path,get_file_name(source_path)+".vtk")
        # mapped_source = transfer_radius_and_save_sample(source, atlas["weights"], saving_path)
        # saving_path = os.path.join(vtk_saving_path,get_file_name(target_path)+".vtk")
        # mapped_target = transfer_radius_and_save_sample(target, atlas["weights"], saving_path)
        # plot_saving_path = os.path.join(radius_transfered_saing_path, "plots")
        # source_vg_weight, target_vg_weight = source["weights"], target["weights"]
        # sampler_obj ="lung_dataloader_utils.lung_sampler( method='combined',scale=0.0003,num_sample=30000,sampled_by_weight=True)"
        # source, target = get_pair(source_path, target_path, expand_bch_dim=False, return_tensor=False)
        # source_combined_weight, target_combined_weight = source["weights"], target["weights"]
        # os.makedirs(plot_saving_path,exist_ok=True)
        # saving_file_path = os.path.join(plot_saving_path,pair_info_list[pair_id]["source"]["name"]+"_weights_distribution.png")
        # title = pair_info_list[pair_id]["source"]["name"] + "_" +"n_sp:{} ".format(len(source_vg_weight))+"n_tp:{}".format(len(atlas["weights"]))
        # _,source_combined_mapped_weight =plot_pair_weight_distribution_before_and_after_radius_matching(source_vg_weight, atlas["weights"],source_combined_weight,sampled_atlas["weights"], use_log=True,title=title,show=False,save_path=saving_file_path)
        # saving_file_path = os.path.join(plot_saving_path,    pair_info_list[pair_id]["target"]["name"] + "_weights_distribution.png")
        # title = pair_info_list[pair_id]["target"]["name"] + "_" + "n_sp:{} ".format(len(target_vg_weight)) + "n_tp:{}".format(len(atlas["weights"]))
        # _,target_combined_mapped_weight =plot_pair_weight_distribution_before_and_after_radius_matching(target_vg_weight, atlas["weights"], target_combined_weight, sampled_atlas["weights"],use_log=True, title=title, show=False,save_path=saving_file_path)

        # saving_path = os.path.join(plot_saving_path, pair_name_list[pair_id]+"_mapped.png")
        # camera_pos = [(-4.924379645467042, 2.17374925796456, 1.5003730890759344), (0.0, 0.0, 0.0),
        #               (0.40133888001174545, 0.31574165540339943, 0.8597873634998591)]
        # visualize_point_pair(source["points"], target["points"],
        #                      source_combined_mapped_weight,
        #                      target_combined_mapped_weight,
        #                      title1="source", title2="target", rgb_on=False,saving_capture_path=saving_path,camera_pos=camera_pos,show=False )

    # source, target = get_pair(*pair_path)
    # source_vg_weight, target_vg_weight = source["weights"], target["weights"]
    # title = pair_name_list[pair_id] + "_" +"n_sp:{} ".format(len(source_vg_weight))+"n_tp:{}".format(len(target_vg_weight))
    # sampler_obj ="lung_dataloader_utils.lung_sampler( method='combined',scale=0.0003,num_sample=30000,sampled_by_weight=True)"
    # source, target = get_pair(source_path, target_path, expand_bch_dim=False, return_tensor=False)
    # source_combined_weight, target_combined_weight = source["weights"], target["weights"]
    # plot_saving_path = os.path.join(radius_transfered_saing_path,"plots")
    # saving_folder_path = os.path.join(output_path,pair_name_list[pair_id])
    # os.makedirs(saving_folder_path,exist_ok=True)
    # saving_file_path = os.path.join(saving_folder_path,pair_name_list[pair_id]+"_weights_distribution.png")
    # plot_pair_weight_distribution_before_and_after_radius_matching(source_vg_weight, target_vg_weight,source_combined_weight,target_combined_weight, use_log=True,title=title,show=False,save_path=saving_file_path)
    #
    # visualize_point_pair(source["points"], target["points"],
    #                 source["weights"],
    #                  target["weights"],
    #                  title1="source", title2="target", rgb_on=False)

    #
    #
    # shape_pair = create_shape_pair(source, target)
    # source_half = get_half_lung(source)
    # target_half = get_half_lung(target)
    # cleaned_source_half = lung_isolated_leaf_clean_up(source_half,radius=0.02, principle_weight=[2,1,1], normalize_weights=False)
    # # visualize_point_pair(source_half.points, cleaned_source_half.points,
    # #                      source_weight_transform(source_half.weights),
    # #                      source_weight_transform(cleaned_source_half.weights),
    # #                      title1="source", title2="cleaned_source", rgb_on=False)
    # #
    # # plot_pair_weight_distribution(source_weight_transform(source_half.weights).cpu().squeeze().numpy(),
    # #                               target_weight_transform(target_half.weights).cpu().squeeze().numpy(),
    # #                               use_log=True)
    #
    # visualize_point_pair(source_half.points, target_half.points,
    #                      source_weight_transform(source_half.weights),
    #                      target_weight_transform(target_half.weights),
    #                      title1="source", title2="target", rgb_on=False)


from datetime import datetime
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.template import loader
from django.urls import reverse

from .models import BusinessIdea

# Create your views here.

def list(request):
    ideas_list = BusinessIdea.objects.order_by("-publish_date")[:10]
    template = loader.get_template('ideas/list.html')
    context = {
        'ideas_list': ideas_list,
    }
    return HttpResponse(template.render(context, request))

def idea(request, idea_id):
    try:
        idea = BusinessIdea.objects.get (pk=idea_id)
    except BusinessIdea.DoesNotExist:
        raise Http404("Idea does not exist")

    #comments = IdeaComment.objects.filter()
    print(idea.__dir__())
    return render(request, 'ideas/detail.html', {"idea": idea, "comments": ""})

def idea_new(request):
    return render(request, "ideas/idea_new.html")

def idea_new_post(request):
    print(request.POST.keys())
    try:
        username = request.POST['username']
        title = request.POST["title"]
        body = request.POST["body"]
    except (KeyError):
        # Redisplay the form.
        return render(request, 'ideas/idea_new.html', {
            'error_message': "Invalid form.",
        })

    newIdea = BusinessIdea(
        username = username,
        title = title,
        body = body,
        publish_date = datetime.now()
    )
    newIdea.save()
    context = {
        "idea": newIdea
    }
    return HttpResponseRedirect(reverse("ideas:idea", args=(newIdea.id,)))


from torch.utils.data import Dataset
from transformers import Trainer
from transformers import TrainingArguments
from trainer.callbacks.printer import PrinterCallback
from data_manager.batch_sampler import Batch_Sampler
from model.model_parameters import Model_Parameters
from trainer.tne_config import TNE_Config
import torch
import os
import json

os.environ["WANDB_DISABLED"] = "true"

class TNETrainer():
    def __init__(self, model: torch.nn.Module, train_set: Dataset, evaluation_set: Dataset, test_set: Dataset,
                 config: TNE_Config, hyper_parameters: Model_Parameters) -> None:
        # Init Trainer properties
        self.model = model
        self.config = config
        self.prepositions_list = config.prepositions_list
        self.num_labels = config.num_labels

        #################################################
        #                Init TNE Model                 #
        #################################################

        self.train_set = train_set
        self.evaluation_set = evaluation_set
        self.test_set = test_set
        self.test_output_path = self.config.test_output
        self.hyper_parameters = hyper_parameters
        self.model = model

        #################################################
        #           Init Training Arguments             #
        #################################################

        training_params = hyper_parameters.training_params
        evaluation_params = hyper_parameters.evaluation_params
        self.training_args = TrainingArguments(output_dir=config.output_dir,
                                               num_train_epochs=training_params["epochs"],
                                               per_device_train_batch_size=training_params['batch_size'],
                                               per_device_eval_batch_size=evaluation_params['batch_size'],
                                               learning_rate=training_params['learning_rate'],
                                               weight_decay=training_params['weight_decay'],
                                               warmup_steps=training_params['warmup_steps'],
                                               logging_dir=config.logs_dir,
                                               logging_steps=5000,  # log & save weights each logging_steps
                                               evaluation_strategy="steps", # evaluate each `logging_steps`
                                               eval_steps=evaluation_params['eval_steps'],
                                               save_strategy="no")

        #############################################
        #               Init Trainer                #
        #############################################

        # Metrics
        self.batch_collator = Batch_Sampler(tokenizer=self.config.tokenizer,
                                       device_type=self.config.device)
        self.trainer = Trainer(
            model=self.model,                                   # TNE model
            args=self.training_args,                            # Training arguments, defined above
            train_dataset=self.train_set,                       # Training set
            eval_dataset=self.evaluation_set,                   # Evaluation set
            #compute_metrics=self.metrics.compute_metrics,      # Callback that computes metrics of interest
            callbacks=[
                # a printer callback used to draw a graph showing the
                # evaluation accuracy of the model over the epochs in the training.
                PrinterCallback
            ],
            data_collator=self.batch_collator,
        )

    def train(self):
        # train the model
        self.trainer.train()

    def evaluate(self):
        # evaluate the model performance
        self.trainer.evaluate()

    def test(self):
        # test the model and create a file with the predicted prepositions.
        with open(self.test_output_path, 'w') as outfile:
            for sample in self.test_set:
                batch = self.batch_collator.__call__(batch=[sample])
                predictions = self.model(batch['input'], None)
                predictions[predictions == 25] = 0
                predictions_json = json.dumps({'predicted_prepositions': predictions.flatten().tolist()})
                outfile.write(predictions_json + "\n")


from typing import List

from csvcubed.models.cube import (
    Cube,
    QbDimension,
    ExistingQbDimension,
    QbColumn,
    CsvColumnUriTemplateMissingError,
    QbAttributeLiteral,
    CsvColumnLiteralWithUriTemplate,
    QbAttribute,
    NoDimensionsDefinedError,
)
from csvcubed.models.validationerror import ValidationError
from csvcubed.utils.qb.cube import get_columns_of_dsd_type
from csvcubed.utils.qb.validation.observations import (
    validate_observations,
)

def validate_qb_component_constraints(cube: Cube) -> List[ValidationError]:
    """
    Validate a :class:`QbCube` to highlight errors in configuration.

    :return: A list of :class:`ValidationError <csvcubed.models.validationerror.ValidationError>` s.
    """

    errors = _validate_dimensions(cube)
    errors += _validate_attributes(cube)
    errors += validate_observations(cube)

    return errors

def _validate_dimensions(cube: Cube) -> List[ValidationError]:
    errors: List[ValidationError] = []
    dimension_columns = get_columns_of_dsd_type(cube, QbDimension)

    for c in cube.columns:
        if isinstance(c, QbColumn) and isinstance(
            c.structural_definition, ExistingQbDimension
        ):
            if c.csv_column_uri_template is None:
                errors.append(
                    CsvColumnUriTemplateMissingError(
                        c.csv_column_title, ExistingQbDimension
                    )
                )

    if len(dimension_columns) == 0:
        errors.append(NoDimensionsDefinedError())
    return errors

def _validate_attributes(cube: Cube) -> List[ValidationError]:
    errors: List[ValidationError] = []

    for c in cube.columns:
        if isinstance(c, QbColumn) and isinstance(c.structural_definition, QbAttribute):
            if isinstance(c.structural_definition, QbAttributeLiteral):
                if c.csv_column_uri_template is not None:
                    errors.append(
                        CsvColumnLiteralWithUriTemplate(
                            c.csv_column_title,
                            f"{c.structural_definition.__class__.__name__} "
                            + "cannot have a uri_tempate as it holds literal values",
                        )
                    )
            else:
                # Not a QbAttributeLiteral
                if (
                    c.csv_column_uri_template is None
                    and len(c.structural_definition.new_attribute_values) == 0  # type: ignore
                ):
                    errors.append(
                        CsvColumnUriTemplateMissingError(
                            c.csv_column_title,
                            f"{c.structural_definition.__class__.__name__} using existing attribute values",
                        )
                    )

    return errors


import errno
import logging as _logging
import socket
import socketserver
import threading
import time

from napalm import utils

# Log
logging = _logging.getLogger("SERVER")
# Temp
# utils.default_logging_setup()

try:
    from twisted.internet import reactor
    from twisted.internet.protocol import connectionDone, Protocol, ServerFactory
    from twisted.protocols.basic import LineReceiver
except ImportError:
    logging.warning("There is no Twisted module!")

"""
Conventions:
"raw" - means data with delimiters, not splitted yet.
"data" - str data.
"data_bytes" - bytes data.
Servers and clients operate only with bytes. Protocol converts bytes to str and wise versa.
"""

# Common

class Config:
    DELIMITER = b"\x00"
    # 1200 - the most optimal max message size to fit IP(?) frame when using TCP
    RECV_SIZE = 1200  # 1024  # 4096

    @property
    def host(self):
        return self._host

    @property
    def port(self):
        return self._port

    def __init__(self, host="", port=0, protocol_class=None):
        self._host = host
        self._port = port
        if protocol_class:
            self.protocol_class = protocol_class

class ServerConfig(Config):
    logging = None
    pass

class ProtocolFactory:
    """
    Single point of creating protocols to be used by any server type.
    """

    def __init__(self, config, app=None):
        self.config = config
        self.app = app

        self.protocol_class = config.protocol_class if config and hasattr(config, "protocol_class") else None
        self.logging = logging if self.protocol_class and self.protocol_class.is_server_protocol else \
            _logging.getLogger("CLIENT")

    def dispose(self):
        self.logging.debug("ProtocolFactory dispose")
        self.config = None
        self.app = None
        self.protocol_class = None
        # self.logging = None

    def create(self, send_bytes_method, close_connection_method, address):
        if not self.protocol_class:
            return None
        protocol = self.protocol_class(send_bytes_method, close_connection_method, address, self.config, self.app)
        self.logging.debug("ProtocolFactory create new protocol: %s for address: %s", protocol, address)
        return protocol

class AbstractServer:
    def __init__(self, config, app=None):
        self.config = config
        self.protocol_factory = ProtocolFactory(config, app)
        logging.debug("Server created. %s", self)

    def dispose(self):
        logging.debug("Server disposing...")
        self.stop()

        if self.protocol_factory:
            self.protocol_factory.dispose()
            self.protocol_factory = None
        self.config = None
        logging.debug("Server disposed")

    def start(self):
        raise NotImplemented

    def stop(self):
        raise NotImplemented

# Twisted

# TODO try to rename all protocol to protocol (all depend on TwistedHandler)
class TwistedHandler(LineReceiver):
    delimiter = b"\x00"
    protocol = None

    def connectionMade(self):
        # Config
        self.delimiter = self.factory.config.DELIMITER
        # Create app protocol
        address = self.transport.getPeer()
        self.protocol = self.factory.protocol_factory.create(self.sendLine, self.transport.loseConnection,
                                                             (address.host, address.port))
        logging.debug("connectionMade for %s protocol: %s", address, self.protocol)

    def rawDataReceived(self, data):
        # Not used while in line_mode
        pass

    def lineReceived(self, line):
        # logging.debug("dataReceived for %s line: %s", self.protocol, line)
        if line:
            self.protocol.process_bytes_list((line,))

    # def sendLine(self, line):
    #     logging.debug("sendData for %s line: %s", self.protocol, line)
    #     super().sendLine(line)

    def connectionLost(self, reason=connectionDone):
        logging.debug("connectionLost for %s reason: %s", self.protocol, reason)
        self.protocol.dispose()
        self.protocol = None

class TwistedTCPServer(AbstractServer):
    factory = None
    port = None

    def __init__(self, config, app=None):
        super().__init__(config, app)

        self.factory = ServerFactory()
        self.factory.protocol = TwistedHandler
        # Custom references
        self.factory.config = config
        self.factory.protocol_factory = self.protocol_factory

        self.started = False
        self.__started_lock = threading.RLock()

    def dispose(self):
        super().dispose()
        if self.factory:
            self.factory.config = None
            self.factory.protocol = None
            self.factory.protocol_factory = None
            self.factory = None

    def start(self):
        self.__started_lock.acquire()
        if self.started:
            logging.warning("Server is already running. address: %s", (self.config.host, self.config.port))
            self.__started_lock.release()
            return

        logging.debug("Server starting... address: %s", (self.config.host, self.config.port))
        self.started = True
        self.__started_lock.release()
        self.port = reactor.listenTCP(self.config.port, self.factory)
        if not reactor.running:
            reactor.run()
        logging.debug("Server started")

    def stop(self):
        self.__started_lock.acquire()
        if not self.started:
            logging.warning("Server is not running. address: %s", (self.config.host, self.config.port))
            self.__started_lock.release()
            return

        logging.debug("Server stopping...")
        self.started = False
        self.__started_lock.release()
        if self.port:
            # deferred = self.port.stopListening()
            # if deferred:
            #     event = threading.Event()
            #     event.clear()
            #
            #     def event_set():
            #         print("Waiting finished")
            #         event.set()
            #     deferred.addCallback(event_set)
            #     print("Waiting while listening stopping...", deferred)
            #     event.wait()
            #     print("Listening stopped")
            self.port.loseConnection()
            try:
                self.port.connectionLost(None)
            except Exception as error:
                # Bug in Twisted: sometimes AttributeError ('Port' object has no attribute 'socket') occurs
                # print("ERROR", error)
                pass
            self.port = None
        # -reactor.stop()
        # reactor.crash()
        logging.debug("Server stopped")

        # print("Press Enter to exit...")
        # input()
        # # Needed to save lobby state using atexit.register() in app
        # sys.exit()

# Threaded

class ThreadedTCPHandler(socketserver.BaseRequestHandler):
    # static
    abort = False

    buffer_bytes = b""
    # is_first = True

    config = None
    protocol = None

    def setup(self):
        threading.current_thread().name += "-srv-handler"
        self.config = self.server.config
        self.protocol = self.server.protocol_factory.create(self.send_bytes, self.request.close,
                                                            self.client_address)
        logging.debug("connectionMade for %s protocol: %s", self.client_address, self.protocol)

    def finish(self):
        logging.debug("connectionLost for %s", self.protocol)
        self.protocol.dispose()
        self.protocol = None
        self.config = None

    def send_bytes(self, data_bytes):
        # logging.debug("sendData for %s line: %s", self.protocol, data_bytes)
        self.request.sendall(data_bytes + self.config.DELIMITER)

    def handle(self):
        while not self.server.abort:
            # Read
            is_data = True
            data_bytes = None
            while not self.server.abort and is_data and self.config.DELIMITER not in self.buffer_bytes:
                try:
                    data_bytes = self.request.recv(self.config.RECV_SIZE)
                    is_data = bool(data_bytes)
                    self.buffer_bytes += data_bytes
                except socket.error as error:
                    # Note: current buffer won't be processed, but it usually empty in such cases
                    logging.debug(" (connectionLost (abort) for %s reason: %s)", self.protocol, error)
                    return

            # Parse bytes
            # b"command1##command2##\x00command3##\x00" -> [b"command1##command2##", b"command3##", b""]
            # b"1||param||##5||param||##\x0010||param||##\x00" ->
            #  [b"1||param||##5||param||##", b"10||param||##", b""]
            if self.buffer_bytes:
                # print("TEMP SERVER config:", self.server and self.config)
                data_bytes_list = self.buffer_bytes.split(self.config.DELIMITER)
                self.buffer_bytes = data_bytes_list.pop()

                # Process
                try:
                    # (Try-except: because send method could be invoked during processing)
                    if self.protocol and data_bytes_list:
                        self.protocol.process_bytes_list(data_bytes_list)
                # (Don't use socket.error because it causes StopIteration, which would not be caught)
                # except socket.error as error:
                except Exception as error:
                    logging.debug(" (connectionLost for %s reason: %s)", self.protocol, error)
                    return

            if not data_bytes:
                if not self.server.abort:
                    reason = "(Empty data received: %s)" % data_bytes
                    logging.debug(" (connectionLost for %s reason: %s)", self.protocol, reason)
                return

class ThreadedTCPServer(AbstractServer):
    server = None

    def __init__(self, config, app=None):
        super().__init__(config, app)

        self.started = False
        self.__started_lock = threading.RLock()
        self.__shutdown_event = threading.Event()
        self.__shutdown_event.set()

    # def dispose(self):
    #     super().dispose()

    def start(self):
        if not self.config:
            logging.error("Server is not initialized")
            return
        self.__started_lock.acquire()
        if self.started:
            logging.warning("Server is already running. address: %s", (self.config.host, self.config.port))
            self.__started_lock.release()
            return

        # Create and start server
        address = (self.config.host, self.config.port)
        logging.debug("Server starting... address: %s", address)
        self.started = True
        self.__started_lock.release()
        self.server = socketserver.ThreadingTCPServer(address, ThreadedTCPHandler)
        self.server.protocol_factory = self.protocol_factory
        self.server.config = self.config
        self.server.abort = False
        logging.debug("Server started")

        self.__shutdown_event.clear()
        try:
            self.server.serve_forever()
        except KeyboardInterrupt as error:
            logging.info("^C KeyboardInterrupt", error)

        # Here we shutting down the server
        logging.debug("Server shutting down...")
        # (Abort other threads)
        self.server.abort = True
        self.server.server_close()
        self.server.protocol_factory = None
        self.server.config = None
        self.server = None
        logging.debug("Server shut down")
        self.__shutdown_event.set()

        # print("Press Enter to exit...")
        # input()
        # # Needed to save lobby state using atexit.register() in app
        # sys.exit()

    def stop(self):
        self.__started_lock.acquire()
        if not self.started:
            logging.warning("Server is not running. address: %s", (self.config.host, self.config.port))
            self.__started_lock.release()
            return

        # Preventing
        logging.debug("Server stopping... address: %s", (self.config.host, self.config.port))
        self.started = False
        self.__started_lock.release()
        t = time.time()
        self.server.shutdown()
        self.__shutdown_event.wait()
        logging.debug("Server stopped in %f sec (95%% of time is exiting from serve_forever())", time.time() - t)

# Non-blocking

class NonBlockingTCPServer(AbstractServer):
    _sock = None

    def __init__(self, config, app=None):
        super().__init__(config, app)

        # (Needed for walking through all connections on each tick and receiving available data)
        self._protocol_list = []
        self._request_by_protocol = {}
        self._buffer_by_protocol = {}

        self._abort = False
        self.started = False
        self.__started_lock = threading.RLock()
        self.__shutdown_event = threading.Event()
        self.__shutdown_event.set()

    def start(self):
        if not self.config:
            logging.warning("Server is not initialized")
            return
        address = (self.config.host, self.config.port)
        logging.debug("Server starting... address: %s", address)
        self.__started_lock.acquire()
        if self.started:
            logging.warning("Server is already running. address: %s", (self.config.host, self.config.port))
            self.__started_lock.release()
            return

        self.started = True
        self.__started_lock.release()

        # (If restarting)
        self._abort = False

        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        self._sock.bind(address)
        self._sock.listen()
        self._sock.setblocking(0)
        logging.debug("Server started")

        self.__shutdown_event.clear()
        try:
            self._workflow(self._sock)
        except KeyboardInterrupt as error:
            logging.debug("^C KeyboardInterrupt %s", error)

        logging.debug("Server shutting down...")
        # self._abort = True
        try:
            self._sock.shutdown(socket.SHUT_RDWR)
        except socket.error as error:
            logging.error("Error while shutting down: %s", error)
        self._sock.close()
        self._sock = None

        # (list() needed to make a copy)
        for protocol in list(self._protocol_list):
            protocol.dispose()
        self._protocol_list.clear()
        self._request_by_protocol.clear()
        self._buffer_by_protocol.clear()
        logging.debug("Server shut down")
        # logging.debug("Server stopped")
        self.__shutdown_event.set()

        # (For standalone. Bad for tests)
        # print("Press Enter to exit...")
        # input()
        # # Needed to save lobby state using atexit.register() in app
        # sys.exit()

    def stop(self):
        logging.debug("Server stopping...")
        self.__started_lock.acquire()
        if not self.started:
            logging.warning("Server is not running. address: %s", (self.config.host, self.config.port))
            self.__started_lock.release()
            return

        # If was started, but yet is not stopping
        self.started = False
        self.__started_lock.release()
        self._abort = True
        self.__shutdown_event.wait()
        logging.debug("Server stopped")

    def _process_disconnect(self, protocol, error):
        logging.debug("connectionLost for %s reason: %s", protocol, error)
        protocol.dispose()
        self._protocol_list.remove(protocol)
        if protocol in list(self._request_by_protocol):
            del self._request_by_protocol[protocol]
        if protocol in list(self._buffer_by_protocol):
            del self._buffer_by_protocol[protocol]

    def _workflow(self, sock):
        while not self._abort:
            # print("SERVER. While...")
            # Connect
            request, address = None, None
            try:
                request, address = sock.accept()
            # socket.error (real error is [WinError 10035])
            except Exception as error:
                # print("accept error:", error)
                # There is no new connections - skip
                pass
            if request:
                # New connection
                def send_bytes(data_bytes):
                    # logging.debug("sendData for %s line: %s", self.protocol, data_bytes)
                    request.sendall(data_bytes + self.config.DELIMITER)

                # Create protocol
                protocol = self.protocol_factory.create(send_bytes, request.close, address)
                logging.debug("connectionMade for %s protocol: %s", address, protocol)
                self._protocol_list.append(protocol)
                self._request_by_protocol[protocol] = request

            # Walk through all connections looking for new data to receive
            i = 0
            for protocol in self._protocol_list:
                i += 1
                request = self._request_by_protocol[protocol]

                # Read
                buffer_bytes = self._buffer_by_protocol.get(self, b"")
                is_data = True
                data_bytes = None
                while is_data:
                    try:
                        data_bytes = request.recv(self.config.RECV_SIZE)
                        is_data = bool(data_bytes)
                        buffer_bytes += data_bytes
                        # print("SERVER. recv data_bytes:", data_bytes, "buffer_bytes:", buffer_bytes)
                    # socket.error
                    except Exception as error:
                        # (break) is_data = False
                        # print("SERVER. Error (recv)", error)
                        if not hasattr(error, "errno") or error.errno != errno.EWOULDBLOCK:
                            self._process_disconnect(protocol, error)
                        # Process next connection for both disconnect and no data received now
                        break
                    if not data_bytes:
                        self._process_disconnect(protocol, "(Empty data received: %s)" % data_bytes)

                if not buffer_bytes:
                    continue

                # Parse bytes
                data_bytes_list = buffer_bytes.split(self.config.DELIMITER)
                self._buffer_by_protocol[self] = data_bytes_list.pop()

                # Process
                try:
                    # (Try-except: because send method could be invoked during processing)
                    if protocol and data_bytes_list:
                        logging.debug("dataReceived for %s line: %s", protocol, buffer_bytes)
                        protocol.process_bytes_list(data_bytes_list)
                # socket.error
                except Exception as error:
                    self._process_disconnect(protocol, error)
                    break