# -*- coding: 1251 -*-

import argparse

from collections import defaultdict
import scipy
import sys

from itertools import  product
from scipy.special import gamma
from scipy.spatial.distance import euclidean
from math import sqrt

from sklearn import datasets
from sklearn.decomposition import PCA
from sklearn.metrics.pairwise import euclidean_distances
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.neighbors import KDTree

import gc

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from scipy.spatial.distance import pdist, squareform
from scipy.special import gamma

from itertools import cycle, islice
from sklearn.metrics import silhouette_score, davies_bouldin_score, calinski_harabasz_score, calinski_harabasz_score
from jqm_cvi.jqmcvi import base
from S_Dbw import S_Dbw
from sklearn.neighbors import NearestNeighbors
from scipy.spatial.distance import cdist
from scipy.stats import ranksums, kstest, ks_2samp

def volume(r, m):
    return np.pi ** (m / 2) * r ** m / gamma(m / 2 + 1)

def significant(cluster, h, p):
    max_diff = max(abs(p[i] - p[j]) for i, j in product(cluster, cluster))

    # print(max_diff)
    return max_diff >= h

class WishartClusterization:
    def __init__(self, wishart_neighbors, significance_level):
        self.wishart_neighbors = wishart_neighbors  # Number of neighbors
        self.significance_level = significance_level  # Significance level

    def fit(self, X):
        kdt = KDTree(X, metric='euclidean')
        print('KDTree has been built')
        #add one because you are your neighb.
        distances, neighbors = kdt.query(X, k = self.wishart_neighbors + 1, return_distance = True)
        neighbors = neighbors[:, 1:]

        distances = distances[:, -1]
        print('distances')
        print(' '.join(list(map(str, distances))))

        indexes = np.argsort(distances)
        print('indexes sorted')
        print(' '.join(list(map(str, indexes))))

        size, dim = X.shape

        self.object_labels = np.zeros(size, dtype = int) - 1

        #index in tuple
        #min_dist, max_dist, flag_to_significant
        self.clusters = np.array([(1., 1., 0)])
        self.clusters_to_objects = defaultdict(list)
        #print('Start clustering')
        counter_to_print = 0
        for index in indexes:

            if counter_to_print % 100000 == 0:
                print(counter_to_print)
                print(' '.join(list(map(str, self.object_labels))), flush=True)

            counter_to_print += 1

            neighbors_clusters =\
                np.concatenate([self.object_labels[neighbors[index]], self.object_labels[neighbors[index]]])
            unique_clusters = np.unique(neighbors_clusters).astype(int)
            unique_clusters = unique_clusters[unique_clusters != -1]

            if len(unique_clusters) == 0:
                self._create_new_cluster(index, distances[index])
            else:
                max_cluster = unique_clusters[-1]
                min_cluster = unique_clusters[0]
                if max_cluster == min_cluster:
                    if self.clusters[max_cluster][-1] < 0.5:
                        self._add_elem_to_exist_cluster(index, distances[index], max_cluster)
                    else:
                        self._add_elem_to_noise(index)
                else:
                    my_clusters = self.clusters[unique_clusters]
                    flags = my_clusters[:, -1]
                    if np.min(flags) > 0.5:
                        self._add_elem_to_noise(index)
                    else:
                        significan = np.power(my_clusters[:, 0], -dim) - np.power(my_clusters[:, 1], -dim)
                        significan *= self.wishart_neighbors
                        significan /= size
                        significan /= np.power(np.pi, dim / 2)
                        significan *= gamma(dim / 2 + 1)
                        significan_index = significan >= self.significance_level

                        significan_clusters = unique_clusters[significan_index]
                        not_significan_clusters = unique_clusters[~significan_index]
                        significan_clusters_count = len(significan_clusters)
                        if significan_clusters_count > 1 or min_cluster == 0:
                            self._add_elem_to_noise(index)
                            self.clusters[significan_clusters, -1] = 1
                            for not_sig_cluster in not_significan_clusters:
                                if not_sig_cluster == 0:
                                    continue

                                for bad_index in self.clusters_to_objects[not_sig_cluster]:
                                    self._add_elem_to_noise(bad_index)
                                self.clusters_to_objects[not_sig_cluster].clear()
                        else:
                            for cur_cluster in unique_clusters:
                                if cur_cluster == min_cluster:
                                    continue

                                for bad_index in self.clusters_to_objects[cur_cluster]:
                                    self._add_elem_to_exist_cluster(bad_index, distances[bad_index], min_cluster)
                                self.clusters_to_objects[cur_cluster].clear()

                            self._add_elem_to_exist_cluster(index, distances[index], min_cluster)
        self.labels_ = self.clean_data()
        return self.labels_

    def clean_data(self):
        unique = np.unique(self.object_labels)
        index = np.argsort(unique)
        if unique[0] != 0:
            index += 1
        true_cluster = {unq :  index for unq, index in zip(unique, index)}
        result = np.zeros(len(self.object_labels), dtype = int)
        for index, unq in enumerate(self.object_labels):
            result[index] = true_cluster[unq]
        return result

    def _add_elem_to_noise(self, index):
        self.object_labels[index] = 0
        self.clusters_to_objects[0].append(index)

    def _create_new_cluster(self, index, dist):
        self.object_labels[index] = len(self.clusters)
        self.clusters_to_objects[len(self.clusters)].append(index)
        self.clusters = np.append(self.clusters, [(dist, dist, 0)], axis = 0)

    def _add_elem_to_exist_cluster(self, index, dist, cluster_label):
        self.object_labels[index] = cluster_label
        self.clusters_to_objects[cluster_label].append(index)
        self.clusters[cluster_label][0] = min(self.clusters[cluster_label][0], dist)
        self.clusters[cluster_label][1] = max(self.clusters[cluster_label][1], dist)

def volume(r, dim):
    """
    Helper function to calculate volumes of a dim-dimensional spheres with radiuses r
    param r: radiuses of a spheres of shape (n_samples,): ndarray
    param dim: dimensionality of a sphere: int

    Returns:
        volumes: ndarray
    """
    dim_const = (np.pi ** (dim / 2)) / gamma(dim / 2 + 1)
    return dim_const * (r ** dim)

def significant(cluster, h, p):
    """
    Helper function to tell if a cluster is significant
    param cluster: vertices in cluster (indexed): list
    h: height hyperparameter: double
    p: estimated saliency for all points
    """
    p_cluster = p[np.array(cluster)]
    pw_difference = np.abs(p_cluster[:, np.newaxis] - p_cluster)
    max_diff = pw_difference.max()

    # print(max_diff)
    return max_diff >= h

class ClusterValidation:
    def __init__(self, data, labels, dist='euclidean'):
        """
        @param data: ndarray of shape (n_samples, n_features)
        @param labels: ndarray of shape (n_samples,), cluster labels for each sample
        @param dist: distance metric to use: str or callable; defaults to 'euclidean'
        """
        self.data = data
        self.labels = labels
        self.dist = dist
        self.num_cluster = labels.max() + 1
        self.cluster_list = list(range(self.num_cluster))

    def RMSSTD(self):
        dof = self.data.shape[0] - self.num_cluster
        n_features = self.data.shape[1]
        result = 0
        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            center = cluster_data.mean(axis=0)
            result += ((cluster_data - center) ** 2).sum()

        return (result / (dof * n_features)) ** (1 / 2)

    def RS(self):
        all_mean = ((self.data - self.data.mean(axis=0)) ** 2).sum()
        part_mean = 0

        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            center = cluster_data.mean(axis=0)
            part_mean += ((cluster_data - center) ** 2).sum()

        return (all_mean - part_mean) / all_mean

    def hubert(self):
        dist_matrix = squareform(pdist(self.data, self.dist))
        n = self.data.shape[0]

        centers = np.empty((self.num_cluster, self.data.shape[1]))
        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            center = cluster_data.mean(axis=0)
            centers[cluster] = center
        center_dist = squareform(pdist(centers, self.dist))

        result = 0
        for i in range(n):
            for j in range(n):
                result += (dist_matrix[i, j] * center_dist[self.labels[i], self.labels[j]])
        return (result * 2) / (n * (n-1))

    def CH(self):
        return calinski_harabasz_score(self.data, self.labels)

    def I_index(self):
        all_mean = np.sqrt(((self.data - self.data.mean(axis=0)) ** 2).sum())
        part_mean = 0
        centers = np.empty((self.num_cluster, self.data.shape[1]))

        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            center = cluster_data.mean(axis=0)
            centers[cluster] = center
            part_mean += np.sqrt(((cluster_data - center) ** 2).sum())
        center_dist = squareform(pdist(centers, self.dist))

        max_center_dist = center_dist.max()

        return (all_mean * max_center_dist) / (self.num_cluster * part_mean)

    def dunn(self):
        return base.dunn_fast(self.data, self.labels)

    def silhouette(self):
        return silhouette_score(self.data, self.labels)

    def davies_bouldin(self):
        return davies_bouldin_score(self.data, self.labels)

    def xie_beni(self):
        n = self.data.shape[0]
        part_mean = 0

        centers = np.empty((self.num_cluster, self.data.shape[1]))
        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            center = cluster_data.mean(axis=0)
            centers[cluster] = center
            part_mean += ((cluster_data - center) ** 2).sum()
        center_dist = squareform(pdist(centers, self.dist))

        center_dist = center_dist + np.diag(np.full_like(center_dist[0], center_dist.max()))

        return part_mean / (n * (center_dist.min()**2))

    def SD(self):
        scat, dis = 0, 0

        all_var_vector = np.var(self.data, 0)
        all_var = np.sqrt(np.sum(all_var_vector ** 2))
        part_var = 0

        centers = np.empty((self.num_cluster, self.data.shape[1]))
        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            center = cluster_data.mean(axis=0)
            centers[cluster] = center
            part_var_vector = np.var(cluster_data, 0)
            part_var += np.sqrt((part_var_vector ** 2).sum())
        center_dist = squareform(pdist(centers, self.dist))

        scat = part_var / (all_var * self.num_cluster)

        max_dist = center_dist.max()
        min_dist = (center_dist + np.diag(np.full_like(center_dist[0], max_dist))).min()
        sum_dist = np.sum(1.0 / np.sum(center_dist, axis=1))

        dis = (max_dist * sum_dist) / min_dist

        return scat + dis

    def S_Dbw(self):
        centers = np.empty((self.num_cluster, self.data.shape[1]))
        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            center = cluster_data.mean(axis=0)
            centers[cluster] = center

        sdbw = S_Dbw(self.data, self.labels, centers)
        return sdbw.S_Dbw_result()

    def CVNN(self, n_neighbors=10):
        com = 0
        sep = []
        neigh = NearestNeighbors(n_neighbors=n_neighbors)
        neigh.fit(self.data)
        knbrs = neigh.kneighbors(return_distance=False)

        for cluster in self.cluster_list:
            cluster_data = self.data[self.labels == cluster, :]
            cluster_ind = np.arange(self.data.shape[0])[self.labels == cluster]
            n_i = cluster_data.shape[0]
            if (n_i > 1):
                com += np.sum(pdist(cluster_data, metric=self.dist)) * 2 / (n_i * (n_i-1))
            sep.append(np.isin(knbrs[cluster_ind], cluster_ind, invert=True).sum() / (n_i * n_neighbors))

        sep = np.max(sep)

        return com + sep

    def run_all(self, n_neighbors=10, calc_hubert=True):
        results = {}
        results['rmsstd'] = self.RMSSTD()
        results['r-squared'] = self.RS()
        if calc_hubert:
            results['hubert'] = self.hubert()
        results['ch_index'] = self.CH()
        results['i_index'] = self.I_index()
        try:
            results['dunn'] = self.dunn()
        except ValueError as e:
            print('dunn failed with')
            print(e)
            results['dunn'] = np.nan
        results['silhouette'] = self.silhouette()
        results['davies_bouldin'] = self.davies_bouldin()
        results['xie_beni'] = self.xie_beni()
        results['sd'] = self.SD()
        try:
            results['s_dbw'] = self.S_Dbw()
        except:
            results['s_dbw'] = np.nan
        results['cvnn'] = self.CVNN(n_neighbors)

        return results

def cluster_mean_intracluster(X, y):
    mean_intracluster_dist = []
    for cluster_id in range(len(np.unique(y))):
        X_cluster = X[y == cluster_id]
        if len(X_cluster) < 2:
            mean_intracluster_dist.append(0)
            continue
        cluster_dist = squareform(pdist(X_cluster))
        n = X_cluster.shape[0]
        mean_intracluster_dist.append(2 * cluster_dist.sum() / (n * (n - 1)))
    return mean_intracluster_dist

def cluster_max_intracluster(X, y):
    max_intracluster_dist = []
    for cluster_id in range(len(np.unique(y))):
        X_cluster = X[y == cluster_id]
        if len(X_cluster) < 2:
            continue
        cluster_dist = squareform(pdist(X_cluster))
        max_intracluster_dist.append(cluster_dist.max())
    return max_intracluster_dist

def cluster_centroid_dist(X, y):
    centroid_dist = []
    for cluster_id in range(len(np.unique(y))):
        X_cluster = X[y == cluster_id]
        if len(X_cluster) < 2:
            continue
        centroid = np.mean(X_cluster, axis=0)
        c_dist = cdist(X_cluster, centroid.reshape(1, -1))
        centroid_dist.append(2 * c_dist.mean())
    return centroid_dist

def cluster_rmsstd(X, y):
    rmsstd_centroid_dist = []
    dof = X.shape[0] - len(np.unique(y))
    n_feat = X.shape[1]
    for cluster_id in range(len(np.unique(y))):
        X_cluster = X[y == cluster_id]
        if len(X_cluster) < 2:
            continue
        centroid = np.mean(X_cluster, axis=0)
        centroid_dist = cdist(X_cluster, centroid.reshape(1, -1))
        rmsstd_centroid_dist.append((centroid_dist.sum() / (dof * n_feat)) ** (0.5))

    return rmsstd_centroid_dist

def cluster_sd_scat(X, y):
    sd_scat = []
    all_var_vector = np.var(X, 0)
    all_var = np.sqrt(np.sum(all_var_vector ** 2))
    for cluster_id in range(len(np.unique(y))):
        X_cluster = X[y == cluster_id]
        if len(X_cluster) < 2:
            continue
        part_var_vector = np.var(X_cluster, 0)
        sd_scat.append(np.sqrt((part_var_vector ** 2).sum()))
    return sd_scat

def cluster_silhouette(X, y):
    result = []
    n_clusters = len(np.unique(y))
    for cluster_id in range(n_clusters):
        X_cluster = X[y == cluster_id]
        if len(X_cluster) < 2:
            continue

        cluster_dist = squareform(pdist(X_cluster))
        n = X_cluster.shape[0]
        a = cluster_dist.sum(axis=1) / (n - 1)
        # other clusters
        b = None

        for other_cluster_id in range(n_clusters):
            if other_cluster_id == cluster_id:
                continue
            X_other_cluster = X[y == other_cluster_id]
            between_cluster_dist = cdist(X_cluster, X_other_cluster)
            mean_between = between_cluster_dist.mean(axis=1)

            if b is None:
                b = mean_between
            else:
                b = np.minimum(b, mean_between)

        result.append(np.mean((b - a) / np.maximum(a, b)) / n_clusters)

    return result

def cluster_db(X, y):
    n_clusters = len(np.unique(y))
    intra_dists = np.empty(n_clusters)
    centroids = np.empty((n_clusters, len(X[0])), dtype=float)

    for cluster_id in range(n_clusters):
        X_cluster = X[y == cluster_id]
        centroid = np.mean(X_cluster, axis=0)
        centroids[cluster_id] = centroid
        intra_dists[cluster_id] = np.mean(cdist(X_cluster, centroid.reshape(1, -1)))

    centroid_dist = squareform(pdist(centroids))

    centroid_dist[centroid_dist == 0] = np.inf
    intra_dists_between_clusters = intra_dists[:, None] + intra_dists
    scores = np.max(intra_dists_between_clusters / centroid_dist, axis=1)

    return scores

def cluster_cvnn_sep(X, y, n_neighbors=10):
    n_clusters = len(np.unique(y))
    neigh = NearestNeighbors(n_neighbors=n_neighbors)
    neigh.fit(X)
    knbrs = neigh.kneighbors(return_distance=False)

    sep = []

    for cluster_id in range(n_clusters):
        X_cluster = X[y == cluster_id]
        cluster_ind = np.arange(X.shape[0])[y == cluster_id]
        n_i = X_cluster.shape[0]

        sep.append(np.isin(knbrs[cluster_ind], cluster_ind, invert=True).sum() / (n_i * n_neighbors))

    return sep

def run_all_stat_tests(X_real, y_real, X_gen, y_gen):
    wilcoxon_res = {}
    ks_res = {}
    fig, axs = plt.subplots(nrows=3, ncols=3, figsize=(15, 15))

    # sizes
    _, real_sizes = np.unique(y_real, return_counts=True)
    _, gen_sizes = np.unique(y_gen, return_counts=True)
    wilcoxon_res['sizes'] = ranksums(real_sizes, gen_sizes)
    ks_res['sizes'] = ks_2samp(real_sizes, gen_sizes)

    sns.kdeplot(x=real_sizes, label='real', ax=axs[0][0])
    sns.kdeplot(x=gen_sizes, label='gen', ax=axs[0][0])
    axs[0][0].set_title('Sizes of clusters')
    axs[0][0].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['sizes'].pvalue,
                       ks_res['sizes'].pvalue),
                   transform=axs[0][0].transAxes)
    axs[0][0].legend()

    # mean intracluster
    stat_real = cluster_mean_intracluster(X_real, y_real)
    stat_gen = cluster_mean_intracluster(X_gen, y_gen)
    wilcoxon_res['mean_dist'] = ranksums(stat_real, stat_gen)
    ks_res['mean_dist'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[0][1])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[0][1])
    axs[0][1].set_title('Mean intracluster distance')
    axs[0][1].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['mean_dist'].pvalue,
                       ks_res['mean_dist'].pvalue),
                   transform=axs[0][1].transAxes)
    axs[0][1].legend()

    # max intracluster
    stat_real = cluster_max_intracluster(X_real, y_real)
    stat_gen = cluster_max_intracluster(X_gen, y_gen)
    wilcoxon_res['max_dist'] = ranksums(stat_real, stat_gen)
    ks_res['max_dist'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[0][2])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[0][2])
    axs[0][2].set_title('Max intracluster distance')
    axs[0][2].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['max_dist'].pvalue,
                       ks_res['max_dist'].pvalue),
                   transform=axs[0][2].transAxes)
    axs[0][2].legend()

    # centroid dist
    stat_real = cluster_centroid_dist(X_real, y_real)
    stat_gen = cluster_centroid_dist(X_gen, y_gen)
    wilcoxon_res['centroid_dist'] = ranksums(stat_real, stat_gen)
    ks_res['centroid_dist'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[1][0])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[1][0])
    axs[1][0].set_title('Centroid distance')
    axs[1][0].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['centroid_dist'].pvalue,
                       ks_res['centroid_dist'].pvalue),
                   transform=axs[1][0].transAxes)
    axs[1][0].legend()

    # rmsstd compactness
    stat_real = cluster_rmsstd(X_real, y_real)
    stat_gen = cluster_rmsstd(X_gen, y_gen)
    wilcoxon_res['rmsstd'] = ranksums(stat_real, stat_gen)
    ks_res['rmsstd'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[1][1])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[1][1])
    axs[1][1].set_title('RMSSTD Compactness')
    axs[1][1].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['rmsstd'].pvalue,
                       ks_res['rmsstd'].pvalue),
                   transform=axs[1][1].transAxes)
    axs[1][1].legend()

    # sd scattering
    stat_real = cluster_sd_scat(X_real, y_real)
    stat_gen = cluster_sd_scat(X_gen, y_gen)
    wilcoxon_res['sd_scat'] = ranksums(stat_real, stat_gen)
    ks_res['sd_scat'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[1][2])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[1][2])
    axs[1][2].set_title('SD scattering')
    axs[1][2].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['sd_scat'].pvalue,
                       ks_res['sd_scat'].pvalue),
                   transform=axs[1][2].transAxes)
    axs[1][2].legend()

    # silhouette index
    stat_real = cluster_silhouette(X_real, y_real)
    stat_gen = cluster_silhouette(X_gen, y_gen)
    wilcoxon_res['silhouette'] = ranksums(stat_real, stat_gen)
    ks_res['silhouette'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[2][0])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[2][0])
    axs[2][0].set_title('Silhouette index')
    axs[2][0].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['silhouette'].pvalue,
                       ks_res['silhouette'].pvalue),
                   transform=axs[2][0].transAxes)
    axs[2][0].legend()

    # db cluster scores
    stat_real = cluster_db(X_real, y_real)
    stat_gen = cluster_db(X_gen, y_gen)
    wilcoxon_res['db_score'] = ranksums(stat_real, stat_gen)
    ks_res['db_score'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[2][1])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[2][1])
    axs[2][1].set_title('DB cluster scores')
    axs[2][1].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['db_score'].pvalue,
                       ks_res['db_score'].pvalue),
                   transform=axs[2][1].transAxes)
    axs[2][1].legend()

    # cvnn separation
    stat_real = cluster_cvnn_sep(X_real, y_real)
    stat_gen = cluster_cvnn_sep(X_gen, y_gen)
    wilcoxon_res['cvnn_sep'] = ranksums(stat_real, stat_gen)
    ks_res['cvnn_sep'] = ks_2samp(stat_real, stat_gen)

    sns.kdeplot(x=stat_real, label='real', ax=axs[2][2])
    sns.kdeplot(x=stat_gen, label='gen', ax=axs[2][2])
    axs[2][2].set_title('CVNN separation')
    axs[2][2].text(0.01, 0.9,
                   'Wilcoxon pvalue = {:.4f}\nKS pvalue = {:.4f}'.format(
                       wilcoxon_res['cvnn_sep'].pvalue,
                       ks_res['cvnn_sep'].pvalue),
                   transform=axs[2][2].transAxes)
    axs[2][2].legend()

    # plt.tight_layout()
    return fig, wilcoxon_res, ks_res

def get_ngrams_from_vectors(X, vector_transform, n=3):
    """
    :param X: данные (n_samples, n_features)
    :param vector_transform: функция, которая преобразует вектора
    :param n: количество векторов в n-грамме
    :return: массив уникальных n-грамм
    """
    X = np.asarray(X)
    ngrams = []
    for i in range(len(X) - n + 1):
        # e.g.:
        # ngrams.append(encoder(np.asarray(X[i:i + n])).numpy().flatten())
        # ngrams.append(pca.transform(np.asarray(X[i:i + n])).flatten())
        ngrams.append(vector_transform(X[i:i + n]).flatten())
    return np.vstack(ngrams)

def clusterize(ngram_series: np.ndarray, n: int, h: float) -> np.array:
    """
    Кластеризация Уишартом
    :param ngram_series: векторные представления ngram_series
    :param n: параметр для Уишарта
    :param h: параметр для Уишарта
    :return: лейблы кластеров
    """
    wishart = WishartClusterization(n, h)
    labels = wishart.fit(ngram_series)
    return labels

def train_PCA(train_dataset, target_dim):
    print('training pca...', file=sys.stderr, flush=True)
    pca = PCA(target_dim)
    pca.fit(train_dataset)
    print('done training pca!', file=sys.stderr, flush=True)
    return pca

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('-T', '--train_dataset')
    # parser.add_argument('-v', '--val_dataset')
    # parser.add_argument('-t', '--test_dataset')

    parser.add_argument('-r', '--embeddings_real')
    parser.add_argument('-g', '--embeddings_gen')

    parser.add_argument('-n', '--ngram', type=int, default=3)
    parser.add_argument('-w', '--wishartn', type=int, default=2)
    parser.add_argument('-s', '--significance', type=float, default=1.)
    parser.add_argument('-d', '--dim', type=int, default=16)
    args = parser.parse_args()

    print(args, '\n', flush=True)

    print('loading data...', file=sys.stderr, flush=True)
    train_dataset = np.load(args.train_dataset)
    embeddings_real = np.load(args.embeddings_real)
    embeddings_gen = np.load(args.embeddings_gen)

    print('training pca...', file=sys.stderr, flush=True)
    pca = train_PCA(train_dataset, args.dim)

    print('getting ngrams...', file=sys.stderr, flush=True)
    ngrams_real = get_ngrams_from_vectors(embeddings_real, vector_transform=pca.transform, n=args.ngram)
    ngrams_gen = get_ngrams_from_vectors(embeddings_gen, vector_transform=pca.transform, n=args.ngram)

    for wishartn in [2, 3, 4]:
        for sig in [0.01, 1]:
            try:
                print(wishartn, sig)
                print('clustering real...', file=sys.stderr, flush=True)
                labels_real = clusterize(ngrams_real, wishartn, sig)
                print(ClusterValidation(ngrams_real, labels_real).run_all(calc_hubert=False), flush=True)

                print('clustering gen...', file=sys.stderr, flush=True)
                labels_gen = clusterize(ngrams_gen, wishartn, sig)
                print(ClusterValidation(ngrams_gen, labels_gen).run_all(calc_hubert=False), flush=True)

                print('processing metrics...', file=sys.stderr, flush=True)
                fig, wilcoxon_res, ks_res = run_all_stat_tests(ngrams_real, labels_real, ngrams_gen, labels_gen)
                fig.savefig('graphs/cmp/' + '_'.join(map(str, [args.wishartn, args.significance, args.ngram, args.dim, 'pca20'])) + '.png')
            except Exception as e:
                print(e)

    """
    print('\nreal labels:')
    for label in labels_real:
        print(label)
    print('\ngen labels:')
    for label in labels_gen:
        print(label)
    """


import numpy as np
import pandas as pd
import patsy
from scipy import stats
from tqdm import tqdm

def lr_tests(ds, alt_model, null_model='~ 1', gene_column='Gene', batch_size=2000, transformation=np.log1p, rcond=-1):
    '''
    Compare alt_model and null_model by a likelihood ratio test for every gene in ds.

    Args:
        ds (LoomConnection):    Dataset
        alt_model (str):    Formula describing alternative model
        null_model (str):   Formula describing null model
        gene_column (str):  Name of the gene labels to use in the ds (default "Gene")
        batch_size (int):   The number of genes to read from disk in each iteration (default 2000)
        transformation (function):  Transformation to apply to expression values before fitting (default np.log1p)
        rcond (float):  Conditioning for the least square fitting (default -1, which has no effect)

    Returns:
        results (DataFrame):    Dataframe with model parameter estimates for each gene, with P values from LRT.
    '''
    sample_info = pd.DataFrame()
    for k in ds.ca.keys():
        sample_info[k] = ds.ca[k]

    alt_design = patsy.dmatrix(alt_model, sample_info, return_type='dataframe')
    null_design = patsy.dmatrix(null_model, sample_info, return_type='dataframe')

    n = ds.shape[1]

    genes = []
    betas = []
    pvals = []

    total_batches = np.ceil(ds.shape[0] / batch_size).astype(int)
    for (ix, selection, vals) in tqdm(ds.scan(axis=0, batch_size=batch_size), total=total_batches):
        expression_matrix = transformation(vals[:, :])
        beta_alt, res_alt, rank_alt, s_alt = np.linalg.lstsq(alt_design, expression_matrix.T, rcond=rcond)
        beta_null, res_null, rank_null, s_null = np.linalg.lstsq(null_design, expression_matrix.T, rcond=rcond)

        genes.append(vals.ra[gene_column])

        ll_alt  = -n / 2. * np.log(2 * np.pi) - n / 2. * np.ma.log(res_alt  / n) - n / 2.
        ll_null = -n / 2. * np.log(2 * np.pi) - n / 2. * np.ma.log(res_null / n) - n / 2.

        llr = ll_alt - ll_null

        pval = stats.chi2.sf(2 * llr, df=(beta_alt.shape[0] - beta_null.shape[0]))
        pval = np.ma.MaskedArray(pval, mask=llr.mask).filled(1.)

        betas.append(beta_alt)
        pvals.append(pval)

    results = pd.DataFrame({gene_column: np.hstack(genes)})

    for name, beta in zip(alt_design.columns, np.hstack(betas)):
        results[name] = beta

    results['pval'] = np.hstack(pvals)

    min_pval = results.pval[results.pval != 0].min()
    results['pval'] = results.pval.clip_lower(min_pval)

    return results


import pytest
import pytorch_lightning as pl
import torch
from torch.nn import CrossEntropyLoss

from kale.embed.mogonet import MogonetGCN
from kale.loaddata.multiomics_datasets import SparseMultiomicsDataset
from kale.pipeline.multiomics_trainer import MultiomicsTrainer
from kale.predict.decode import LinearClassifier, VCDN
from kale.prepdata.tabular_transform import ToOneHotEncoding, ToTensor
from kale.utils.seed import set_seed

binary_class_data_url = "https://github.com/pykale/data/raw/main/multiomics/ROSMAP.zip"
multi_class_data_url = "https://github.com/pykale/data/raw/main/multiomics/TCGA_BRCA.zip"

@pytest.fixture
def test_model(num_classes, url):
    num_modalities = 3
    gcn_hidden_dim = [200, 200, 100]
    vcdn_hidden_dim = pow(num_classes, num_modalities)
    gcn_dropout_rate = 0.5
    gcn_lr = 5e-4
    vcdn_lr = 1e-3
    loss_function = CrossEntropyLoss(reduction="none")

    unimodal_encoder = []
    unimodal_decoder = []
    multimodal_decoder = None

    if num_classes > 2:
        root = "tests/test_data/multiomics/trainer/multi_class"
    else:
        root = "tests/test_data/multiomics/trainer/binary_class"

    file_names = []
    for modality in range(1, num_modalities + 1):
        file_names.append(f"{modality}_tr.csv")
        file_names.append(f"{modality}_lbl_tr.csv")
        file_names.append(f"{modality}_te.csv")
        file_names.append(f"{modality}_lbl_te.csv")

    dataset = SparseMultiomicsDataset(
        root=root,
        raw_file_names=file_names,
        num_modalities=num_modalities,
        num_classes=num_classes,
        edge_per_node=10,
        url=url,
        random_split=False,
        train_size=0.7,
        equal_weight=False,
        pre_transform=ToTensor(dtype=torch.float),
        target_pre_transform=ToOneHotEncoding(dtype=torch.float),
    )

    for modality in range(num_modalities):
        unimodal_encoder.append(
            MogonetGCN(
                in_channels=dataset.get(modality).num_features,
                hidden_channels=gcn_hidden_dim,
                dropout=gcn_dropout_rate,
            )
        )

        unimodal_decoder.append(LinearClassifier(in_dim=gcn_hidden_dim[-1], out_dim=num_classes))

    if num_modalities >= 2:
        multimodal_decoder = VCDN(num_modalities=num_modalities, num_classes=num_classes, hidden_dim=vcdn_hidden_dim)

    trainer = MultiomicsTrainer(
        dataset=dataset,
        num_modalities=num_modalities,
        num_classes=num_classes,
        unimodal_encoder=unimodal_encoder,
        unimodal_decoder=unimodal_decoder,
        loss_fn=loss_function,
        multimodal_decoder=multimodal_decoder,
        train_multimodal_decoder=True,
        gcn_lr=gcn_lr,
        vcdn_lr=vcdn_lr,
    )

    return trainer

@pytest.mark.parametrize("num_classes, url", [(2, binary_class_data_url), (5, multi_class_data_url)])
def test_init(test_model, num_classes, url):
    assert isinstance(test_model.dataset, SparseMultiomicsDataset)
    assert test_model.num_modalities == 3
    assert test_model.num_classes == num_classes
    assert all(isinstance(encoder, MogonetGCN) for encoder in test_model.unimodal_encoder)
    assert all(isinstance(decoder, LinearClassifier) for decoder in test_model.unimodal_decoder)
    assert isinstance(test_model.loss_fn, CrossEntropyLoss)
    assert isinstance(test_model.multimodal_decoder, VCDN)
    assert test_model.train_multimodal_decoder
    assert test_model.gcn_lr == 5e-4
    assert test_model.vcdn_lr == 1e-3

@pytest.mark.parametrize("num_classes, url", [(2, binary_class_data_url), (5, multi_class_data_url)])
def test_configure_optimizers(test_model, num_classes, url):
    optimizers = test_model.configure_optimizers()
    assert len(optimizers) == 4
    assert all(isinstance(optimizer, torch.optim.Adam) for optimizer in optimizers)
    for modality in range(test_model.num_modalities):
        assert optimizers[modality].param_groups[0]["lr"] == 5e-4
    assert optimizers[test_model.num_modalities].param_groups[0]["lr"] == 1e-3

@pytest.mark.parametrize(
    "num_classes, url, multimodal",
    [
        (2, binary_class_data_url, False),
        (2, binary_class_data_url, True),
        (5, multi_class_data_url, False),
        (5, multi_class_data_url, True),
    ],
)
def test_forward2(test_model, num_classes, url, multimodal):
    x = []
    adj_t = []
    for modality in range(test_model.num_modalities):
        data = test_model.dataset.get(modality)
        x.append(data.x[data.train_idx])
        adj_t.append(data.adj_t_train)

    outputs = test_model.forward(x, adj_t, multimodal)

    assert isinstance(outputs, list) != multimodal
    if not multimodal:
        assert len(outputs) == test_model.num_modalities
        for modality in range(test_model.num_modalities):
            assert outputs[modality].shape == (test_model.dataset.get(modality).num_train, test_model.num_classes)
    else:
        assert isinstance(outputs, torch.Tensor)
        assert outputs.shape == (test_model.dataset.get(0).num_train, test_model.num_classes)
        test_model.multimodal_decoder = None
        with pytest.raises(TypeError):
            _ = test_model.forward(x, adj_t, multimodal)

@pytest.mark.parametrize("num_classes, url", [(2, binary_class_data_url), (5, multi_class_data_url)])
def test_pipeline(test_model, num_classes, url):
    set_seed(2023)
    trainer_pretrain = pl.Trainer(
        default_root_dir="./tests/outputs",
        max_epochs=2,
        accelerator="cpu",
        enable_model_summary=False,
    )
    trainer_pretrain.fit(test_model)
    result = trainer_pretrain.test(test_model)
    if num_classes > 2:
        assert 0 <= result[0]["Accuracy"] <= 1
        assert 0 <= result[0]["F1 weighted"] <= 1
        assert 0 <= result[0]["F1 macro"] <= 1
    else:
        assert 0 <= result[0]["Accuracy"] <= 1
        assert 0 <= result[0]["F1"] <= 1
        assert 0 <= result[0]["AUC"] <= 1

    assert str(test_model) is not None
    assert len(str(test_model)) > 0


import torch
import torch.nn as nn
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
import torch.nn.functional as F
import spacy
from torch.utils.data import Dataset, DataLoader
import re
import subprocess as call
import json
import pandas as pd
from collections import Counter
from tqdm.auto import tqdm, trange
tqdm.pandas(desc='Progress')

# ******************************************* #
# * Make batches through PyTorch Dataloader * #
# ******************************************* #

def collate_fn(data):
    """This function will be used to pad the tweets to max length
       in the batch and transpose the batch from
       batch_size x max_seq_len to max_seq_len x batch_size.
       It will return padded vectors, labels and lengths of each tweets (before padding)
       It will be used in the Dataloader
    """
    data.sort(key=lambda x: len(x[0]), reverse=True)
    lens = [len(sent) for sent, label in data]
    labels = []
    padded_sents = torch.zeros(len(data), max(lens)).long()
    for i, (sent, label) in enumerate(data):
        padded_sents[i,:lens[i]] = torch.LongTensor(sent)
        labels.append(label)

    padded_sents = padded_sents.transpose(0,1)
    return padded_sents, torch.tensor(labels).long(), lens

# ************************ #
# * Custom class for data  #
# ************************ #
PAD = 0
UNK = 1

print('Pytorch version:', torch.__version__)
print('Spacy version:', spacy.__version__)

class SentimentDataset(Dataset):
    """Define the pytorch Dataset to process the tweets
       This class can be used for both training and validation dataset
       Run it for training data and pass the word2idx and idx2word when running
       for validation data
    """

    def __init__(self, df=None, word2idx=None, idx2word=None, max_vocab_size=50000):
        print('Processing Data')
        self.df = df
        if(self.df is not None):
            print('Removing white space...')
            self.df.SentimentText = self.df.SentimentText.progress_apply(lambda x: x.strip())
        try:
            self.nlp = spacy.load('en', disable=['parser', 'tagger', 'ner'])
        except IOError:
            call.check_output("python -m spacy download en", shell=True)
            self.nlp = spacy.load('en', disable=['parser', 'tagger', 'ner'])
        finally:
            pass
        if word2idx is None and df is not None:
            print('Building Counter...')
            word_counter = self.build_counter()
            print('Building Vocab...')
            self.word2idx, self.idx2word = self.build_vocab(word_counter, max_vocab_size)
        else:
            self.word2idx, self.idx2word = word2idx, idx2word
        print('*'*100)
        print('Dataset info:')
        if df is not None:
            print(f'Number of Tweets: {self.df.shape[0]}')
            print(f'Vocab Size: {len(self.word2idx)}')
        print('*'*100)

    def __len__(self):
        return self.df.shape[0]

    def __getitem__(self, idx):
        sent = self.df.SentimentText[idx]
        tokens = [w.text.lower() for w in self.nlp(self.tweet_clean(sent))]
        vec = self.vectorize(tokens, self.word2idx)
        return vec, self.df.Sentiment[idx]

    def tweet_clean(self, text):
        """Very basic text cleaning. This function can be built upon for
           better preprocessing
        """
        text = re.sub(r'[\s]+', ' ', text) # replace multiple white spaces with single space
#         text = re.sub(r'@[A-Za-z0-9]+', ' ', text) # remove @ mentions
        text = re.sub(r'https?:/\/\S+', ' ', text) # remove links
        text = re.sub(r'[^A-Za-z0-9]+', ' ', text) # remove non alphanumeric character
        return text.strip()

    def build_counter(self):
        """Tokenize the tweets using spacy and build vocabulary
        """
        words_counter = Counter()
        for sent in tqdm(self.df.SentimentText.values):
            words_counter.update(w.text.lower() for w in self.nlp(self.tweet_clean(sent)))
        return words_counter

    def build_vocab(self, words_counter, max_vocab_size):
        """Add pad and unk tokens and build word2idx and idx2word dictionaries
        """
        word2idx = {'<PAD>': PAD, '<UNK>': UNK}
        word2idx.update({word:i+2 for i, (word, count) in tqdm(enumerate(words_counter.most_common(max_vocab_size)))})
        idx2word = {idx: word for word, idx in tqdm(word2idx.items())}
        return word2idx, idx2word

    def vectorize(self, tokens, word2idx):
        """Convert tweet to vector
        """
        vec = [word2idx.get(token, UNK) for token in tokens]
        return vec

    def save_vocabulary(self, PATH='./models/sd', filenames=('w2i','i2w')):
        """ Save the word2idx & idx2word dicts
        """
        f = open(PATH+filenames[0]+".json","w")
        f.write(json.dumps(self.word2idx))
        f.close()
        print("wrote word2idx to {}".format(PATH+filenames[0]+".json"))
        f = open(PATH+filenames[1]+".json","w")
        f.write(json.dumps(self.idx2word))
        f.close()
        print("wrote word2idx to {}".format(PATH+filenames[1]+".json"))
        return

    def load_vocabulary(self, PATH='./models/sd', filenames=('w2i','i2w')):
        f = open(PATH+filenames[0]+".json","r")
        self.word2idx = json.load(f)
        f.close()
        f = open(PATH+filenames[1]+".json","r")
        self.idx2word = json.load(f)
        f.close()
        print('!'*100)
        print(f'Updated Vocab Sizes (word2idx, idx2word): {len(self.word2idx)}, {len(self.idx2word)}')
        print('!'*100)
        return

    def load_json_as_df(self, json_obj):
        assert 'SentimentText' in json.loads(json_obj), "key `SentimentText` should appear in hash [dict] argument"
        assert 'Sentiment' in json.loads(json_obj), "key `Sentiment` should appear in hash [dict] argument"
        self.df = pd.read_json(json_obj)
        self.df.SentimentText.progress_apply(lambda x: x.strip())
        print('!'*100)
        print('Notice Updated Dataset info:')
        print(f'Number of %Tweets% / data entries: {self.df.shape[0]}')
        print('!'*100)
        return
    @staticmethod
    def get_pdl(ds, collator=collate_fn):
        """Get the Pytorch DataLoader for a SentimentDataset
            used for predictions
        """
        return DataLoader(ds, collate_fn=collator)

# *********************************** #
# * Max Pooling and Average Pooling * #
#   with Concat pooling GRU model     #
# *********************************** #

resolve_model_output = lambda a : 0 if a[0] == 0 and a[1] != 0 else (1 if a[0] != 0 and a[1] == 0 else None)
class ConcatPoolingGRUAdaptive(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
        super().__init__()
        self.vocab_size = vocab_size
        self.embedding_dim = embedding_dim
        self.n_hidden = n_hidden
        self.n_out = n_out
        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
        self.emb_drop = nn.Dropout(0.3)
        self.gru = nn.GRU(self.embedding_dim, self.n_hidden, dropout=0.3)
        self.out = nn.Linear(self.n_hidden*3, self.n_out)
        self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

    def forward(self, seq, lengths):
        self.h = self.init_hidden(seq.size(1))
        embs = self.emb_drop(self.emb(seq))
        embs = pack_padded_sequence(embs, lengths)
        gru_out, self.h = self.gru(embs, self.h)
        gru_out, lengths = pad_packed_sequence(gru_out)

        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)
        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)

        outp = self.out(torch.cat([self.h[-1],avg_pool,max_pool],dim=1))
        return F.log_softmax(outp, dim=-1) # it will return log of softmax

    def init_hidden(self, batch_size):
        return torch.zeros((1, batch_size,self.n_hidden), requires_grad=True).to(self.device)

    def import_dict(self, PATH='./models/text_gru_concat_sentiment_10.pth'):
        """# Load the saved state into the model
        """
        self.load_state_dict(torch.load(PATH))

    @staticmethod
    def prediction(model, dl):
        """ static method used on `ConcatPoolingGRUAdaptive` class to
            predict sentiment
        """
        sentiment = []
        model.eval() # run if you only want to use it for inference
        with torch.no_grad():
            for x, y, lens in dl:
                x = x.to(model.device) # , y.to(model.device)
                y_pred = model(x, lens)
                pred = torch.round(y_pred.squeeze()).tolist()
                print(pred)
                sentiment.append(resolve_model_output(pred))
            return sentiment


# Christopher Sandoval 13660
# 29/03/2019
# SR6: Transformations

# Funcion para multiplicar matrices
def matrix_mult(a,b):
    zip_b = zip(*b)
    zip_b = list(zip_b)
    return [[sum(ele_a*ele_b for ele_a, ele_b in zip(row_a, col_b))
             for col_b in zip_b] for row_a in a]


def fab(n):
    if n<1:
        print("输入有误!")

    if n==1 or n==2 :
        return 1
    else:
        return fab(n-1) + fab(n-2)

n = int(input("输入月数:\n"))
print("%d个月后共有%d个小兔崽子诞生" % (n,fab(n)))

递归求斐波那契数列.py


# function declaration
def is_palindrome(text):
    data = [i.lower() for i in text if i.isalpha()]

    return data == data[::-1]

# reading inputs
txt = input()

# function call
print(is_palindrome(txt))

013 Functions/13.5 Functions with return value: Part 2/#13.5-7.py


import numpy as np
from paddle.io import Dataset, DataLoader
import paddle
import paddle.nn as nn
import math
import functools
from paddle.nn import Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, Linear

# 处理图片数据：裁切、水平翻转、调整图片数据形状、归一化数据
def data_transform(img, resize_w, resize_h, load_size=286, pos=[0, 0, 256, 256], flip=True, is_image=True):
    if is_image:
        resized = img.resize((resize_w, resize_h), Image.BICUBIC)
    else:
        resized = img.resize((resize_w, resize_h), Image.NEAREST)
    croped = resized.crop((pos[0], pos[1], pos[2], pos[3]))
    fliped = ImageOps.mirror(croped) if flip else croped
    fliped = np.array(fliped) # transform to numpy array
    expanded = np.expand_dims(fliped, 2) if len(fliped.shape) < 3 else fliped
    transposed = np.transpose(expanded, (2, 0, 1)).astype('float32')
    if is_image:
        normalized = transposed / 255. * 2. - 1.
    else:
        normalized = transposed
    return normalized

# 定义CoCo数据集对象
class COCODateset(Dataset):
    def __init__(self, opt):
        super(COCODateset, self).__init__()
        inst_dir = opt.dataroot+'train_inst/'
        _, _, inst_list = next(os.walk(inst_dir))
        self.inst_list = np.sort(inst_list)
        self.opt = opt

    def __getitem__(self, idx):
        ins = Image.open(self.opt.dataroot+'train_inst/'+self.inst_list[idx])
        img = Image.open(self.opt.dataroot+'train_img/'+self.inst_list[idx].replace(".png", ".jpg"))
        img = img.convert('RGB')

        w, h = img.size
        resize_w, resize_h = 0, 0
        if w < h:
            resize_w, resize_h = self.opt.load_size, int(h * self.opt.load_size / w)
        else:
            resize_w, resize_h = int(w * self.opt.load_size / h), self.opt.load_size
        left = random.randint(0, resize_w - self.opt.crop_size)
        top = random.randint(0, resize_h - self.opt.crop_size)
        flip = False

        img = data_transform(img, resize_w, resize_h, load_size=opt.load_size,
            pos=[left, top, left + self.opt.crop_size, top + self.opt.crop_size], flip=flip, is_image=True)
        ins = data_transform(ins, resize_w, resize_h, load_size=opt.load_size,
            pos=[left, top, left + self.opt.crop_size, top + self.opt.crop_size], flip=flip, is_image=False)

        return img, ins, self.inst_list[idx]

    def __len__(self):
        return len(self.inst_list)

def data_onehot_pro(instance, opt):
    shape = instance.shape
    nc = opt.label_nc + 1 if opt.contain_dontcare_label \
        else opt.label_nc
    shape[1] = nc
    semantics = paddle.nn.functional.one_hot(instance.astype('int64'). \
        reshape([opt.batchSize, opt.crop_size, opt.crop_size]), nc). \
        transpose((0, 3, 1, 2))

    # edge
    edge = np.zeros(instance.shape, 'int64')
    t = instance.numpy()
    edge[:, :, :, 1:] = edge[:, :, :, 1:] | (t[:, :, :, 1:] != t[:, :, :, :-1])
    edge[:, :, :, :-1] = edge[:, :, :, :-1] | (t[:, :, :, 1:] != t[:, :, :, :-1])
    edge[:, :, 1:, :] = edge[:, :, 1:, :] | (t[:, :, 1:, :] != t[:, :, :-1, :])
    edge[:, :, :-1, :] = edge[:, :, :-1, :] | (t[:, :, 1:, :] != t[:, :, :-1, :])
    edge = paddle.to_tensor(edge).astype('float32')

    semantics = paddle.concat([semantics, edge], 1)
    return semantics

# 设置除 spade 以外的归一化层
def build_norm_layer(norm_type='instance'):
    """Return a normalization layer

    Args:
        norm_type (str) -- the name of the normalization layer: batch | instance | none

    For BatchNorm, we do not use learnable affine parameters and track running statistics (mean/stddev).
    For InstanceNorm, we do not use learnable affine parameters. We do not track running statistics.
    """
    if norm_type == 'batch':
        norm_layer = functools.partial(
            nn.BatchNorm2D,
            weight_attr=False,
            bias_attr=False)
    elif norm_type == 'syncbatch':
        norm_layer = functools.partial(
            nn.SyncBatchNorm,
            weight_attr=False,
            bias_attr=False)
    elif norm_type == 'instance':
        norm_layer = functools.partial(
            nn.InstanceNorm2D,)
    elif norm_type == 'spectral':
        norm_layer = functools.partial(Spectralnorm)
    elif norm_type == 'none':

        def norm_layer(x):
            return Identity()
    else:
        raise NotImplementedError('normalization layer [%s] is not found' %
                                  norm_type)
    return norm_layer

def simam(x, e_lambda=1e-4):
    b, c, h, w = x.shape
    n = w * h - 1
    x_minus_mu_square = (x - x.mean(axis=[2, 3], keepdim=True)) ** 2
    y = x_minus_mu_square / (4 * (x_minus_mu_square.sum(axis=[2, 3], keepdim=True) / n + e_lambda)) + 0.5
    return x * nn.functional.sigmoid(y)

class Dict(dict):
    __setattr__ = dict.__setitem__
    __getattr__ = dict.__getitem__


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import load_data
import clean_data
import transform_data
import eda

# In[3]:

datos=load_data.carga_archivo('consumo-agua.csv')

# In[4]:

load_data.observaciones_variables(datos)

# In[5]:

clean_data.estandariza_variables(datos)

# In[6]:

transform_data.tipo_variables(datos)

# In[7]:

transform_data.separar_variable(datos,'geo_point','latitud','longitud',',')
transform_data.eliminar_variable(datos,'geo_point')
transform_data.eliminar_variable(datos,'geo_shape')
transform_data.cambiar_tipo_variable(datos,'latitud','float64')
transform_data.cambiar_tipo_variable(datos,'longitud','float64')
transform_data.cambiar_tipo_variable(datos,'bimestre','str')
transform_data.cambiar_tipo_variable(datos,'gid','str')
transform_data.cambiar_minusculas_variable(datos,'colonia')
transform_data.cambiar_minusculas_variable(datos,'alcaldia')
transform_data.cambiar_minusculas_variable(datos,'indice_des')
transform_data.quitar_acentos(datos)

# In[8]:

datos.head()

# In[9]:

transform_data.tipo_variables(datos)

# In[10]:

eda.tabla_estadisticos_descriptivos_variables_numericas(datos)

# In[11]:

eda.tabla_estadisticos_descriptivos_variables_categoricas(datos)

# In[12]:

eda.grafico_histograma_categorica(datos,"alcaldia","Alcaldías","Conteos","Histograma de Alcaldías")

# In[13]:

eda.grafico_histograma_categorica(datos,"bimestre","Bimestre","Conteos","Histograma de Bimestre")

# In[14]:

eda.grafico_histograma_categorica(datos,"indice_des","Índice de Desarrollo","Conteos","Histograma de Índice de Desarrollo")

# In[15]:

eda.grafico_barplot_orden_decreciente(datos,'alcaldia','consumo_total','Alcaldía','consumo de agua total','Consumo de agua promedio por manzana en cada alcaldía')

# In[16]:

eda.tabla_ranking_n(datos,15,"colonia","alcaldia",'consumo_total')

# In[17]:

eda.grafico_barplot_orden_en_barras(datos,'indice_des','consumo_total',['popular','bajo','medio','alto'],'Índice de desarrollo','Consumo de agua total','Consumo de agua promedio por índice de desarrollo')

# In[18]:

eda.grafico_tipo_uso(datos,'total','consumos totales','tipos de uso','Consumos por tipo de uso del inmueble')

# In[19]:

eda.grafico_tipo_uso(datos,'prom','consumos promedio','tipos de uso','Consumos por tipo de uso del inmueble')

# In[20]:

eda.grafico_strip(datos,'bimestre','consumo_total','Bimestre de 2019','Consumo de agua total','Consumo de agua promedio por bimestre')

# In[21]:

eda.matriz_correlacion(datos)


# Get usernames from user collection and add the field to the recipes collectoin

recipe_comments = [
    {
        u"$unwind": {
            u"path": u"$comments",
            u"preserveNullAndEmptyArrays": False
        }
    },
    {
        u"$lookup": {
            u"from": u"users",
            u"localField": u"comments.user_id",
            u"foreignField": u"_id",
            u"as": u"comment_by"
        }
    },
    {
        u"$unwind": {
            u"path": u"$comment_by",
            u"preserveNullAndEmptyArrays": False
        }
    },
    {
        u"$addFields": {
            u"comments.comment_by": u"$comment_by.username"
        }
    },
    {
        u"$project": {
            u"comment_by": 0.0,
        }
    },
    {
        u"$project": {
            u"comments": 1.0
        }
    }
]


import random
numeros = []
pares = []
def sorteia():
    for c in range(1,6):
        numeros.append(random.randint(1,10))
    print(f'Os numeros sorteados foram {numeros}')
def somapar():
    for c in numeros:
        if c % 2 == 0:
            pares.append(c)
    print(f'Os números pares são: {pares} e a soma deles é {sum(pares)}', end=' ')

sorteia()
somapar()


import numpy as np
import tensorflow as tf
import tensorflow.contrib.rnn as rnn
import matplotlib.pyplot as plt

TIME_STEPS=10
BATCH_SIZE=128
HIDDEN_UNITS1=30
HIDDEN_UNITS=1
LEARNING_RATE=0.001
EPOCH=50

TRAIN_EXAMPLES=11000
TEST_EXAMPLES=1100

#------------------------------------Generate Data-----------------------------------------------#
#generate data
def generate(seq):
    X=[]
    y=[]
    for i in range(len(seq)-TIME_STEPS):
        X.append([seq[i:i+TIME_STEPS]])
        y.append([seq[i+TIME_STEPS]])
    return np.array(X,dtype=np.float32),np.array(y,dtype=np.float32)

#s=[i for i in range(30)]
#X,y=generate(s)
#print(X)
#print(y)

seq_train=np.sin(np.linspace(start=0,stop=100,num=TRAIN_EXAMPLES,dtype=np.float32))
seq_test=np.sin(np.linspace(start=100,stop=110,num=TEST_EXAMPLES,dtype=np.float32))

#plt.plot(np.linspace(start=0,stop=100,num=10000,dtype=np.float32),seq_train)

#plt.plot(np.linspace(start=100,stop=110,num=1000,dtype=np.float32),seq_test)
#plt.show()

X_train,y_train=generate(seq_train)
#print(X_train.shape,y_train.shape)
X_test,y_test=generate(seq_test)

#reshape to (batch,time_steps,input_size)
X_train=np.reshape(X_train,newshape=(-1,TIME_STEPS,1))
X_test=np.reshape(X_test,newshape=(-1,TIME_STEPS,1))

#draw y_test
plt.plot(range(1000),y_test[:1000,0],"r*")
#print(X_train.shape)
#print(X_test.shape)

#-----------------------------------------------------------------------------------------------------#

#--------------------------------------Define Graph---------------------------------------------------#
graph=tf.Graph()
with graph.as_default():

    #------------------------------------construct LSTM------------------------------------------#
    #place hoder
    X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,1),name="input_placeholder")
    y_p=tf.placeholder(dtype=tf.float32,shape=(None,1),name="pred_placeholder")

    #lstm instance
    lstm_cell1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1)
    lstm_cell=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS)

    multi_lstm=rnn.MultiRNNCell(cells=[lstm_cell1,lstm_cell])

    #自己初始化state
    #第一层state
    lstm_layer1_c=tf.zeros(shape=(BATCH_SIZE,HIDDEN_UNITS1))
    lstm_layer1_h=tf.zeros(shape=(BATCH_SIZE,HIDDEN_UNITS1))
    layer1_state=rnn.LSTMStateTuple(c=lstm_layer1_c,h=lstm_layer1_h)

    #第二层state
    lstm_layer2_c = tf.zeros(shape=(BATCH_SIZE, HIDDEN_UNITS))
    lstm_layer2_h = tf.zeros(shape=(BATCH_SIZE, HIDDEN_UNITS))
    layer2_state = rnn.LSTMStateTuple(c=lstm_layer2_c, h=lstm_layer2_h)

    init_state=(layer1_state,layer2_state)
    print(init_state)

    #自己展开RNN计算
    outputs = list()                                    #用来接收存储每步的结果
    state = init_state
    with tf.variable_scope('RNN'):
        for timestep in range(TIME_STEPS):
            if timestep > 0:
                tf.get_variable_scope().reuse_variables()
            # 这里的state保存了每一层 LSTM 的状态
            (cell_output, state) = multi_lstm(X_p[:, timestep, :], state)
            outputs.append(cell_output)
    h = outputs[-1]

    #---------------------------------define loss and optimizer----------------------------------#
    mse=tf.losses.mean_squared_error(labels=y_p,predictions=h)
    #print(loss.shape)
    optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=mse)

    init=tf.global_variables_initializer()

#-------------------------------------------Define Session---------------------------------------#
with tf.Session(graph=graph) as sess:
    sess.run(init)
    for epoch in range(1,EPOCH+1):
        results = np.zeros(shape=(TEST_EXAMPLES, 1))
        train_losses=[]
        test_losses=[]
        print("epoch:",epoch)
        for j in range(TRAIN_EXAMPLES//BATCH_SIZE):
            _,train_loss=sess.run(
                    fetches=(optimizer,mse),
                    feed_dict={
                            X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE],
                            y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE]
                        }
            )
            train_losses.append(train_loss)
        print("average training loss:", sum(train_losses) / len(train_losses))

        for j in range(TEST_EXAMPLES//BATCH_SIZE):
            result,test_loss=sess.run(
                    fetches=(h,mse),
                    feed_dict={
                            X_p:X_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE],
                            y_p:y_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE]
                        }
            )
            results[j*BATCH_SIZE:(j+1)*BATCH_SIZE]=result
            test_losses.append(test_loss)
        print("average test loss:", sum(test_losses) / len(test_losses))
        plt.plot(range(1000),results[:1000,0])
    plt.show()


import random

def roll_dice():
    """
    Simulates rolling a six-sided dice.
    """
    return random.randint(1, 6)

def main():
    print("Welcome to the Dice Simulator Game!")
    player_name = input("Enter your name: ")

    while True:
        print("\nOptions:")
        print("1. Roll the dice")
        print("2. Quit")
        choice = input("Enter your choice (1 or 2): ")

        if choice == "1":
            dice_roll = roll_dice()
            print(f"\n{player_name}, you rolled a {dice_roll}!")
        elif choice == "2":
            print("Thanks for playing!")
            break
        else:
            print("Invalid choice. Please try again.")

if __name__ == "__main__":
    main()


from django.urls import path
from weather.geoapps.views import show_geoapp, show_goeadmin, show_api_v1, show_api_v2, show_api_v3, show_api_history, show_api_station

app_name = 'weather.geoapps'

urlpatterns = [
    path('', show_geoapp, name='home'),
    path('dashboard/show/geoapp', show_goeadmin, name='show'),
    path('api/v1', show_api_v1, name='apiv1'),
    path('api/v2', show_api_v2, name='apiv2'),
    path('api/v3', show_api_v3, name='apiv3'),
    path('api/v4', show_api_history, name='history'),
    path('api/v5', show_api_station, name='station'),
]


# -*- coding: utf-8 -*-

from __future__ import unicode_literals
import sys, os

# il 13 sett 2023 copiando da https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/conf.py ho dato questa configurazione
import re

# il 13 sett 2023 copiando da https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/conf.py ho dato questa configurazione
from sphinx_rtd_theme import __version__ as theme_version
from sphinx_rtd_theme import __version_full__ as theme_version_full

# dal conf.py di tansignari
import recommonmark
from recommonmark.transform import AutoStructify

on_rtd = os.environ.get('READTHEDOCS') == 'True'

sys.path.append(os.path.abspath(os.pardir))

__version__ = '1.0'

# -- General configuration -----------------------------------------------------

# source_suffix = '.rst' eliminato dal conf.py di tansignari #
master_doc = 'index'
project = 'Ciro Spataro'
copyright = '= licenza CC BY Cirospat'

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

extlinks = {}

# -- Options for HTML output ---------------------------------------------------

# html_theme = 'default'   (eliminato il 13 sett 2023 in quanto la parte grafica di RTD si era rotta
# il 13 sett 2023 copiando da https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/conf.py ho dato questa configurazione
html_theme = 'sphinx_rtd_theme'

html_static_path = ['static']

def setup(app):
    # overrides for wide tables in RTD theme
    app.add_stylesheet('theme_overrides.css') # path relative to static

# dal conf.py di tansignari
from recommonmark.parser import CommonMarkParser

source_parsers = {
    '.md': 'recommonmark.parser.CommonMarkParser',
}

source_suffix = ['.rst', '.md']

extensions = ['sphinx.ext.ifconfig','sphinx_markdown_tables']

# per le extensions guarda anche https://sphinx-rtd-theme.readthedocs.io/en/stable/configuring.html

"""
  You might want to uncomment the “latex_documents = []” if you use CKJ characters in your document.
  Because the pdflatex raises exception when generate Latex documents with CKJ characters.
"""
#latex_documents = []

# latex_logo = "static/cirospat.jpg"
# html_logo = "static/cirospat.jpg"
latex_logo = "static/ciro.jpeg"
html_logo = "static/ciro.jpeg"

# Adding Custom CSS or JavaScript to a Sphinx Project: al seguente link ci sono esempi
# https://docs.readthedocs.io/en/latest/guides/adding-custom-css.html

templates_path = ['_templates']


import os
import pdfplumber
import pytesseract
import logging
from PIL import Image
import traceback

# Ensure pytesseract uses the correct installation path to Tesseract
pytesseract.pytesseract.tesseract_cmd = r'/usr/bin/tesseract'

# Initialize logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

def prompt_user_for_directories():
    """
    Prompt the user for source and target directory paths.
    """
    source_dir = input("Please enter the source directory path: ")
    target_dir = input("Please enter the target directory path: ")
    return source_dir, target_dir

def extract_text_from_page(page):
    """
    Extract text from a page.
    """
    text = page.extract_text()
    if text is None:
        text = ""
    return text

def extract_text_from_image(image):
    """
    Use OCR to extract text from a PIL image.
    """
    return pytesseract.image_to_string(image)

def extract_text_from_pdf(pdf_path, target_dir):
    """
    Extract text from a PDF file. The content from all pages, including text and OCR results from images,
    is combined and written to a single text file in the target directory.
    """
    with pdfplumber.open(pdf_path) as pdf:
        # Get the basename for the output file
        base_filename = os.path.splitext(os.path.basename(pdf_path))[0]
        logging.info(f"Extracting text from PDF: {base_filename}.pdf")

        all_text_content = []  # List to store all text content

        for page_num, page in enumerate(pdf.pages):
            text_content = extract_text_from_page(page)
            text_content += " \n\n"  # Add space between page text and image text

            # Extract text from images on the page
            image = page.to_image().original.convert('RGB')
            ocr_text = extract_text_from_image(image)
            if ocr_text:  # Append only if OCR text is extracted
                text_content += "Image Text:\n" + ocr_text + "\n\n"

            all_text_content.append(text_content)

        # Combine all text into one string
        combined_text = "\n\n".join(all_text_content)

        # Write the combined text to a single file named after the PDF
        output_filename = os.path.join(target_dir, f"{base_filename}.txt")
        with open(output_filename, 'w', encoding='utf-8') as output_file:
            output_file.write(combined_text)
            logging.info(f"Saved extracted text to {output_filename}")

def main():
    source_dir, target_dir = prompt_user_for_directories()

    # Validate source and target directories
    if not os.path.exists(source_dir) or not os.path.isdir(source_dir):
        logging.error(f"Source directory does not exist or is not a directory: {source_dir}")
        return

    if not os.path.exists(target_dir):
        try:
            os.makedirs(target_dir)
            logging.info(f"Created target directory: {target_dir}")
        except Exception as e:
            logging.error(f"Unable to create target directory: {e}")
            return

    # Attempt to read and list all files in the source directory for debugging
    try:
        file_list = os.listdir(source_dir)
        logging.info(f"All files found in source directory: {file_list}")
    except Exception as e:
        logging.error(f"Unable to read the source directory: {e}")
        return

    # Process PDF files in the source directory
    pdf_files = [f for f in file_list if f.lower().endswith('.pdf')]
    logging.info(f"Number of PDF files found: {len(pdf_files)}")
    if len(pdf_files) == 0:
        logging.info("No PDF files found in source directory.")
        return

    for filename in pdf_files:
        pdf_path = os.path.join(source_dir, filename)
        try:
            extract_text_from_pdf(pdf_path, target_dir)
            logging.info(f"Successfully extracted text from {filename}")
        except Exception as e:
            logging.error(f"Failed to extract text from {filename}: {e}")
            traceback.print_exc()  # Print the full traceback

    logging.info("Extraction completed. Check the target directory for the output files.")

if __name__ == "__main__":
    main()


from django.shortcuts import render
from django.http import HttpResponse
from django.views import View
from django.views.generic import TemplateView, CreateView

from .models import SiteStatus
from .forms import  SiteStatusForm

# Create your views here.

class HomeView(TemplateView):
    template_name = "home.html"

    def get_context_data(self, *args, **kwargs):
        context = super(HomeView, self).get_context_data(*args, *kwargs)
        mylist = {"0", "1", "2", "3", "4", 5, 6, 7, 8}
        context = {"var": "string of words", "yy": mylist}
        return context

class AboutView(TemplateView):
    template_name = "about.html"

class ContactView(TemplateView):
    template_name = "contact.html"

class SiteStatusView(CreateView):
    form_class = SiteStatusForm
    template_name = "sitestatus_form.html"


import sage.all
from sage.graphs.graph_generators import graphs

import pathlib
import multiprocessing as mp
import matplotlib.pyplot as plt
from collections import Counter

from src.experiments_helpers import *

import os
import sys

import time

max_n_cycles = 4
max_vertices = 30
N = 10
max_cycle_len = 7
max_oriented_path_len = 5

def experiments_iterator(p, j):
    for i in range(N):
        max_path_len = int(max_cycle_len/2)+1
        G = random_multiple_cycles_connected(n_cycles=j,
                                             max_vertices=max_vertices,
                                             max_cycle_len=max_cycle_len,
                                             max_path_len=max_path_len,
                                             p=p,
                                             min_cycle_len=3)
        G, l = random_orientation(G, max_oriented_path_len)
        print("(%.1f, %d): generated %s for i=%d" % (p, j, G.dig6_string(), i))
        yield G, l

PATH = str(pathlib.Path(__file__).parent.absolute()) + "/../results/"
PLOTS_PATH = str(pathlib.Path(__file__).parent.absolute()) + "/../plots/"

if not os.path.exists(PATH):
    os.mkdir(PATH)

if not os.path.exists(PLOTS_PATH):
    os.mkdir(PLOTS_PATH)

def generate_and_calculate():
    for p in [1, 0.5, 0]:
        processes = []
        for j in range(1, max_n_cycles + 1):
            out_file = "%.1f:%d.out" % (p, j)
            process = mp.Process(target=check_compressibility_many,
                                 args=(experiments_iterator(p, j), ),
                                 kwargs={"upper_bound": 8,
                                         "save_results": PATH + out_file})
            processes.append(process)
            process.start()

def plot_hist(ax, data, title, save_file, xlabel=True):
    bins = np.array([0, 1, 2, 3, 4, 5])

    _, bins, patches = ax.hist(data, bins=bins, rwidth=0.5)

    xlabels = ['', '1', '2', '3', '4+', '']
    N_labels = len(xlabels)
    ax.set_xlim([1, 5])
    ax.set_xticks(np.arange(N_labels) + 0.5)
    ax.set_xticklabels(xlabels)

    ax.set_title(title)
    if xlabel:
        ax.set_xlabel("Różnica pomiędzy kompresyjnością, a długością "
                      "najdłuższej ścieżki skierowanej.")
    ax.set_ylabel("Liczba grafów")

def plots_p():
    titles = {1: "Tylko ścieżki zewnętrzne (typ 1)",
              0.5: "Kombinacja ścieżek zewnętrznych i wewnętrznych (typ 2)",
              0: "Tylko ścieżki wewnętrzne (typ 3)"}
    fig, axs = plt.subplots(3, 1, figsize=(6, 9))
    for i, p in enumerate([1, 0.5, 0]):
        diff_tab = []
        comp = []
        for j in range(1, max_n_cycles + 1):
            in_file = PATH + "%.1f:%d.out" % (p, j)
            for line in open(in_file, 'r'):
                _, compressibility, path_len = line.split()
                compressibility = int(compressibility)
                path_len = int(path_len)
                if compressibility != -1:
                    comp.append(compressibility)
                    diff = compressibility - path_len
                    if diff > 4:
                        diff = 4
                else:
                    comp.append(9)
                    diff = 4
                diff_tab.append(diff)
        plot_hist(axs[i], diff_tab, titles[p], "hist:%.1f.png" % p)

    for ax in axs.flat:
        ax.label_outer()
    fig.savefig(PLOTS_PATH + "hist_p_comparison.png")
    fig.show()

def plots_p_1_diff_cycles():
    fig, axs = plt.subplots(2, 2, figsize=(6, 6))
    for j in range(1, max_n_cycles + 1):
        diff_tab = []
        in_file = PATH + "%.1f:%d.out" % (1, j)
        for line in open(in_file, 'r'):
            _, compressibility, path_len = line.split()
            compressibility = int(compressibility)
            path_len = int(path_len)
            if compressibility != -1:
                diff = compressibility - path_len
                if diff > 4:
                    diff = 4
            else:
                diff = 4
            diff_tab.append(diff)

        cykl = "cykl" if j == 1 else "cykle"
        plot_hist(axs[int((j-1)/2), (j-1) % 2], diff_tab, "%d %s" % (j, cykl),
                  "hist:1.0:%d.png" % j, False)

    for ax in axs.flat:
        ax.label_outer()
    fig.savefig(PLOTS_PATH + "hist_j_comparison.png")
    fig.show()

def plot_density():
    densities = []
    compr_minus_path = []
    for p in [1, 0.5, 0]:
        for j in range(1, max_n_cycles + 1):
            in_file = PATH + "%.1f:%d.out" % (p, j)
            for line in open(in_file, 'r'):
                graph, compressibility, path_len = line.split()
                compressibility = int(compressibility)
                path_len = int(path_len)
                graph = DiGraph(graph)
                density = len(graph.edges()) / len(graph.vertices())
                densities.append(density)
                if compressibility != -1:
                    diff = compressibility - path_len
                    if diff > 4:
                        diff = 4
                else:
                    diff = 4
                compr_minus_path.append(diff)
    plt.scatter(densities, compr_minus_path)
    plt.xlabel("Gęstość")
    plt.ylabel("Kompresyjność - długość najdłuższej ścieżki")

    plt.yticks(np.arange(1, 5), ['1', '2', '3', '4+'])

    plt.savefig(PLOTS_PATH + "density.png")

    plt.show()

def plot_triangles():
    triangles_num = []
    compr_minus_path = []
    for p in [1, 0.5, 0]:
        for j in range(1, max_n_cycles + 1):
            in_file = PATH + "%.1f:%d.out" % (p, j)
            for line in open(in_file, 'r'):
                graph, compressibility, path_len = line.split()
                compressibility = int(compressibility)
                path_len = int(path_len)
                graph = DiGraph(graph).to_undirected()
                triangles_num.append(graph.triangles_count())
                if compressibility != -1:
                    diff = compressibility - path_len
                    if diff > 4:
                        diff = 4
                else:
                    diff = 4
                compr_minus_path.append(diff)

    plt.scatter(triangles_num, compr_minus_path)
    plt.xlabel("Liczba trójkątów")
    plt.ylabel("Kompresyjność - długość najdłuższej ścieżki")

    plt.yticks(np.arange(1, 5), ['1', '2', '3', '4+'])

    plt.savefig(PLOTS_PATH + "triangles.png")

    plt.show()

def measure_time(n, m, N):
    times = []
    for j in range(14, m + 1):
        print(j)
        total = 0
        for i in range(N):
            G = graphs.RandomGNM(n, j)
            DiG, _ = random_orientation(G, 9)
            start = time.time()
            compressibility_number(DiG, upper_bound=9)
            end = time.time()
            total += end - start
        print(total/N)
        times.append(total / N)
    return times

def plot_time():
    n = 10
    m = 13
    N = 10
    times = measure_time(n, m, N)
    plt.plot(list(range(1, m + 1)), times)
    plt.xlabel("Liczba krawędzi")
    plt.ylabel("Średni czas obliczania kompresyjności (s)")
    plt.title("Czas obliczania kompresyjności dla losowych grafów")
    plt.savefig(PLOTS_PATH + "time.png")
    plt.show()

if __name__ == "__main__":
    #generate_and_calculate()
    plots_p()
    plots_p_1_diff_cycles()
    plot_density()
    plot_triangles()
    plot_time()


# Using graphillion
from graphillion import GraphSet
import graphillion.tutorial as tl

def A333466(n):
    if n == 1: return 1
    universe = tl.grid(n - 1, n - 1)
    GraphSet.set_universe(universe)
    cycles = GraphSet.cycles()
    for i in [1, n, n * (n - 1) + 1, n * n]:
        cycles = cycles.including(i)
    return cycles.len()

print([A333466(n) for n in range(1, 10)])


from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _

def numbers_only_validator(value):
    if not all([x.isdigit() for x in str(value)]):
        raise ValidationError(_('Please, enter digits only.'), code='invalid')

def phone_number_validator(value):
    allowed_chars = ['+', ' ']
    for c in str(value):
        if not c.isdigit() and c not in allowed_chars:
            raise ValidationError(_('The allowed characters are: +, space and digits.'), code='invalid')


"""Collection of path, file, and directory utilities.

Intended to add a self containing, consistent, and robust layer over the
standard os module, and thus `os` should not be needed outside this module.
There are four main areas of file/directory functionality covered:
introspection, generic removal/creation, validation, and path parsing.

Notes
-----
* All path parameters for every function are normalized (made path independent).
* Functions with names that start with 'ensure' are meant to be used prior to
  file input/output, to ensure failures are caught ahead of time. This is very
  important since virtually nothing in the API will work correctly if data is not
  retrieved correctly.
* Minimal usage of neighboring functions in this module to reduce call-stack
  complexity, thus os.path functions are used repetitively.
"""
import os
import errno
import codecs
import shutil
import inspect
import importlib
import contextlib
from typing import Sequence

# --------------------------------- file and directory inspection ----------------------------------
def is_existent_file(file_path: str) -> bool:
    """Return True if given file exists, else False.

    See Also
    --------
    * Function `ensure_file_exists` for the exception raising counterpart
    """
    try:
        ensure_file_exists(file_path)
        return True
    except FileNotFoundError:
        return False

def is_existent_directory(dir_path: str) -> bool:
    """Return True if given path is dir exists, else False.

    See Also
    --------
    * Function `ensure_directory_exists` for the exception raising counterpart
    """
    try:
        ensure_directory_exists(dir_path)
        return True
    except FileNotFoundError:
        return False

def is_empty_file(file_path: str) -> bool:
    """Return True if file is empty, else False. Raises if file not present."""
    ensure_file_exists(file_path)
    return os.stat(os.path.normpath(file_path.strip(' '))).st_size == 0

def get_path_of_python_source(obj: object) -> str:
    """Return path to the python source code of given object.

    Raises
    ------
    ValueError
        * If inspected path is not a python file
    FileNotFoundError
        * If inspected path does not exist (extremely unlikely)

    Notes
    -----
    * Given object is a package if and only if its name is same as its parent
      directory's, and its source file is an __init__.py. In that case, we
      return the parent directory of the source file (exact package path)
    """
    file_path = os.path.normpath(inspect.getfile(object=obj))
    if os.path.splitext(file_path)[-1].lower() != '.py':
        raise ValueError('Only python source objects can be inspected for path.')
    if not os.path.isfile(file_path):
        raise FileNotFoundError(errno.ENOENT, 'Python source cannot be found ', file_path)

    if file_path.endswith('__init__.py') and os.path.dirname(file_path).endswith(obj.__name__):
        return os.path.normpath(os.path.dirname(file_path))
    return file_path

# -------------------------------- file and directory manipulation ---------------------------------
def create_directory(dir_path: str) -> None:
    """Make single directory (no intermediates) at given path if creatable.

    See Also
    --------
    * For requirements of a successful directory creation, and the corresponding
      exceptions raised, visit function `ensure_directory_is_creatable`.
    """
    dir_path_ = os.path.normpath(dir_path)
    ensure_directory_is_creatable(dir_path_)
    os.mkdir(path=dir_path_)

def create_temp_file():
    """Create temp file."""
    pass

def remove_directory(dir_path: str, ignore_errors=False) -> None:
    """Remove directory and all children at given path, with an option of ignoring any Errors."""
    dir_path_ = os.path.normpath(dir_path.strip(' '))
    if ignore_errors:
        with contextlib.suppress(OSError):
            shutil.rmtree(path=dir_path_, ignore_errors=True)
    else:
        try:
            ensure_directory_exists(dir_path_)
            shutil.rmtree(path=dir_path_, ignore_errors=False)
        except Exception:
            raise OSError(f'Directory \'{dir_path_}\' failed to be removed.')

def remove_file(file_path: str, ignore_errors=False) -> None:
    """Remove file at given path, with an option of ignoring any Errors."""
    file_path_ = os.path.normpath(file_path.strip(' '))
    if ignore_errors:
        with contextlib.suppress(OSError):
            os.remove(file_path_)
    else:
        try:
            ensure_file_exists(file_path_)
            os.remove(file_path_)
        except Exception:
            raise OSError(f'File \'{file_path_}\' failed to be removed.')

@contextlib.contextmanager
def change_directory(dir_path: str) -> str:
    """Temporarily change directory to given path, similar to 'cd' in terminal.

    Examples
    --------
    >>> import os
    >>> from stem_center_analytics import PROJECT_DIR

    >>> previous_dir = os.getcwd()
    >>> with change_directory(PROJECT_DIR): print(os.getcwd() == PROJECT_DIR)
    True
    >>> previous_dir == os.getcwd()
    True
    """
    new_cwd = os.path.normpath(dir_path)
    old_cwd = os.path.normpath(os.getcwd())
    ensure_directory_exists(new_cwd)
    try:
        os.chdir(new_cwd)
        yield new_cwd
    finally:
        os.chdir(old_cwd)

# ---------------------------------------- error checking ------------------------------------------
def ensure_successful_imports(path: str, names: Sequence[str]) -> None:
    """Valid if all object names are successfully imported at given path location.

    Parameters
    ----------
    path : str
        Path at which to import the given names. If file path given,
        names are imported from its parent directory
    names : array-like of str
        Names of objects to import in the order given. This can be the name of
        any importable object, but to ensure successful imports favor
        full, absolute import names such as 'package.module.func', over '.func'

    Raises
    ------
    ImportError
        * If any module in names failed to be imported correctly

    Notes
    -----
    * Imports are lost (goes out of scope) once the function returns
    """
    path_ = os.path.normpath(path.strip(' '))
    dir_to_import_at = path_ if os.path.isdir(path_) else os.path.normpath(os.path.dirname(path_))

    # import given names from the inferred directory location
    unsuccessful_imports = []
    with change_directory(dir_to_import_at):
        for name in names:
            try:
                importlib.import_module(name)
            except ImportError:
                unsuccessful_imports.append(name)
    if unsuccessful_imports:
        raise ImportError(f'Failed to import modules - {tuple(unsuccessful_imports)}.')

def ensure_file_exists(file_path: str) -> None:
    """Valid if path exists as a file.

    Raises
    ------
    FileNotFoundError
        * If path is not an existing file
    ValueError
        * If file does not end with one of `file_type` extensions
    """
    file_path_ = os.path.normpath(file_path.strip(' '))
    if not os.path.isfile(file_path_):
        raise FileNotFoundError(errno.ENOENT, 'No such file', file_path_)

def ensure_directory_exists(dir_path: str) -> None:
    """Valid if path exists as a directory.

    Raises
    ------
    FileNotFoundError
        * If path is a child of a non-existent parent directory
    """
    dir_path_ = os.path.normpath(dir_path.strip(' '))
    if not os.path.isdir(dir_path_):
        raise FileNotFoundError(errno.ENOENT, 'No such directory', dir_path_)

def ensure_file_is_creatable(file_path: str) -> None:
    """Creatable if vacant path in existing directory..

    Raises
    ------
    FileExistsError
        * If file path already exists
    FileNotFoundError
        * If file path is a child of a non-existent parent directory
    ValueError
        * If file does not end with one of `file_type` extensions
    """
    file_path_ = os.path.normpath(file_path.strip(' '))
    parent_dir_path_ = os.path.normpath(os.path.dirname(file_path_))
    if os.path.exists(file_path_):
        raise FileExistsError(errno.ENOENT, 'Cannot create file at non-vacant location', file_path_)
    if not os.path.isdir(parent_dir_path_):
        raise FileNotFoundError(errno.ENOENT, 'Cannot create file in non-existent parent directory',
                                parent_dir_path_)

def ensure_directory_is_creatable(dir_path: str) -> None:
    """Directory is creatable if path is vacant in existing directory.

    Raises
    ------
    FileExistsError
        * If directory path already exists
    FileNotFoundError
        * If path is a child of a non-existent parent directory
    """
    dir_path_ = os.path.normpath(dir_path.strip(' '))
    parent_dir_path_ = os.path.normpath(os.path.dirname(dir_path_))
    if os.path.exists(dir_path_):
        raise FileExistsError(errno.ENOENT, 'Cannot create directory at non-vacant location',
                              dir_path_)
    if not os.path.isdir(parent_dir_path_):
        raise FileNotFoundError(errno.ENOENT,
                                'Cannot create a sub directory in a non-existent parent directory',
                                parent_dir_path_)

def ensure_path_is_absolute(file_path: str) -> None:
    """Valid if file path is absolute.

    Raises
    ------
    ValueError
        * If file path is not absolute

    Notes
    -----
    * In contrast to most other functions in `os_lib` starting with 'ensure',
      given path is NOT checked for existence
    """
    if not os.path.isabs(os.path.normpath(file_path.strip(' '))):
        raise ValueError('File path \'{}\' is not absolute.')

def ensure_valid_file_type(file_path: str, file_type: str) -> None:
    """Valid if file is one of the given file types.

    Raises
    ------
    ValueError
        * If file name has more than one extension
        * If file name doesn't have extension of `file_type`

    Notes
    -----
    * In contrast to most other functions in `os_lib` starting with 'ensure',
      given path is NOT checked for existence
    """
    file_type_ = file_type.lstrip('.')
    file_name, file_extension = get_basename(file_path), get_extension(file_path).lstrip('.')
    if file_name.count('.') != 1:
        raise ValueError(f'File \'{file_name}\' is invalid - file name must have a single extension.')
    if file_extension != file_type_:
        raise ValueError(f'File \'{file_name}\' is invalid - file must be of type \'{file_type_}\'.')

def ensure_valid_file_encoding(file_path: str, encoding: str) -> None:
    """Valid if file at given path can be successfully decoded with given encoding."""
    file_path_ = os.path.normpath(file_path.strip(' '))
    encoding_ = encoding.strip().lower()

    try:
        with codecs.open(file_path_, mode='r', encoding=encoding_, errors='strict'): pass
    except LookupError:
        raise LookupError(f'Unknown encoding - \'{encoding_}\'.') from None
    except UnicodeError:
        raise UnicodeDecodeError(f'File \'{file_path_}\' cannot be decoded as \'{encoding_}\'.')

# ------------------------------------- path name manipulation -------------------------------------
def join_path(path: str, *args: str) -> str:
    """Return path joined with given items.

    Notes
    -----
    * Will break if items contain separators
    """
    base_path_ = os.path.normpath(path.strip(' '))
    path_components = [arg.strip(' ') for arg in args]
    return os.path.normpath(os.path.join(base_path_, *path_components))

def normalize_path(path: str) -> str:
    """Return platform independent path with leading/trailing whitespace removed."""
    return os.path.normpath(path.strip(' '))

def get_extension(path: str) -> str:
    """Return path's (last) extension, including leading '.' separator."""
    return os.path.splitext(os.path.normpath(path.strip(' ')))[-1].lower()

def get_parent_dir(path: str) -> str:
    """Return immediate parent directory of given path."""
    return os.path.normpath(os.path.dirname(os.path.normpath(path.strip(' '))))

def get_basename(path: str, include_extension: bool=True) -> str:
    """Return unix-style basename (eg: '/foo/bar/' => 'bar')."""
    # normalize path, and return the last component
    name = os.path.split(os.path.normpath(path.strip(' ')))[-1]
    return name if include_extension else os.path.splitext(name)[0]


#!/usr/bin/python
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils import common_koji

ANSIBLE_METADATA = {
    'metadata_version': '1.0',
    'status': ['preview'],
    'supported_by': 'community'
}

DOCUMENTATION = '''
---
module: koji_external_repo

short_description: Create and manage Koji external repos
description:
   - Create and manage Koji external repos
options:
   name:
     description:
       - The name of the Koji external repo to create and manage.
     required: true
   url:
     description:
       - The URL to the Koji external repo.
       - Note, this uses "$arch", not the common "$basearch" you may find in a
         typical Yum repository file.
       - For idempotency, please ensure your url always ends with a "/"
         character. If you leave it out, Koji Hub will automatically add a "/"
         slash when storing this value in the database, and every subsequent
         Ansible run will appear to be "changing" the external repo's URL.
     required: true
requirements:
  - "python >= 2.7"
  - "koji"
'''

EXAMPLES = '''
- name: create a koji tag with an external repo.
  hosts: localhost
  tasks:
    - name: Create an external repo for CentOS "CR"
      koji_external_repo:
        name: centos7-cr
        url: http://mirror.centos.org/centos/7/cr/$arch/
        state: present

    - name: Create a koji tag that uses the CentOS CR repo
      koji_tag:
        name: storage7-ceph-nautilus-el7-build
        state: present
        external_repos:
        - repo: centos7-cr
          priority: 5
'''

RETURN = ''' # '''

def ensure_external_repo(session, name, check_mode, url):
    """
    Ensure that this external repo exists in Koji.

    :param session: Koji client session
    :param name: Koji external repo name
    :param check_mode: don't make any changes
    :param url: URL to this external repo
    """
    repoinfo = session.getExternalRepo(name)
    result = {'changed': False, 'stdout_lines': []}
    if not repoinfo:
        if check_mode:
            result['stdout_lines'].append('would create repo %s' % name)
            result['changed'] = True
            return result
        common_koji.ensure_logged_in(session)
        repoinfo = session.createExternalRepo(name, url)
        result['stdout_lines'].append('created repo id %d' % repoinfo['id'])
        result['changed'] = True
        return result
    if repoinfo['url'] != url:
        result['stdout_lines'].append('set url to %s' % url)
        result['changed'] = True
        if not check_mode:
            common_koji.ensure_logged_in(session)
            session.editExternalRepo(info=repoinfo['id'], url=url)
    return result

def delete_external_repo(session, name, check_mode):
    """ Ensure that this external_repo is deleted from Koji. """
    repoinfo = session.getExternalRepo(name)
    result = dict(
        stdout='',
        changed=False,
    )
    if repoinfo:
        result['stdout'] = 'deleted external repo %s' % name
        result['changed'] = True
        if not check_mode:
            common_koji.ensure_logged_in(session)
            session.deleteExternalRepo(name)
    return result

def run_module():
    module_args = dict(
        koji=dict(),
        name=dict(required=True),
        state=dict(choices=['present', 'absent'], default='present'),
        url=dict(),
    )
    module = AnsibleModule(
        argument_spec=module_args,
        supports_check_mode=True
    )

    if not common_koji.HAS_KOJI:
        module.fail_json(msg='koji is required for this module')

    check_mode = module.check_mode
    params = module.params
    profile = params['koji']
    name = params['name']
    state = params['state']
    url = params['url']

    session = common_koji.get_session(profile)

    if state == 'present':
        if not url:
            module.fail_json(msg='you must set a url for this external_repo')
        result = ensure_external_repo(session, name, check_mode, url)
    elif state == 'absent':
        result = delete_external_repo(session, name, check_mode)

    module.exit_json(**result)

def main():
    run_module()

if __name__ == '__main__':
    main()


import tweepy
from keys import *
from youtubeData import *
from datetime import *
import matplotlib.pyplot as plt
import numpy as np

#set up today's date and time to be used
today = datetime.now()
date = today.strftime("%x")
time = today.strftime("%X")

def main():
    auth = tweepy.OAuthHandler(twitter_APIkey,
        twitter_APISecretKey)
    auth.set_access_token(twitter_AccessToken,
        twitter_AccessTokenSecret)

    api = tweepy.API(auth)

    header = date + "\n"

    subs = getSubs(iZoneChannelID) + " subs for izone(test3)"
    viewCountVioleta = getViews(violetaVideoID)
    violetaviews = u'\U0001F48E' + "Violeta views: " + viewCountVioleta
    lverviews = u'\U0001F339' + "La Vie en Rose views: " + getViews(lverID)
    print(subs + "\n" + violetaviews)
    api.update_status(header + subs + "\n" + violetaviews + "\n" + lverviews)
    '''
    for status in api.user_timeline():
        print(status.id)
    '''
    data = open("data.txt", "a")
    writeData(data,"violetaViews",viewCountVioleta)
    data1 = [date, (datetime.today() + timedelta(days=1)).strftime("%x")]
    data2 = [1, 2]
    fig, ax = plt.subplots(1, 1)
    ax.set_facecolor("#3C3C3C")
    fig.set_facecolor("#C3B0DE")
    createGraph(ax, data1, data2, {'color': '#B6E3BC'})
    plt.savefig('test.png', facecolor="#C3B0DE")
    #plt.show()
    all_tweets = api.user_timeline()
    media = api.media_upload("test.png")
    print(media.media_id)
    api.update_status(status="test pic", media_ids=[media.media_id], in_reply_to_status_id=all_tweets[0].id)
    data.close()

#format data as date, time, numbers

def writeData(file, tag, data):
    file.write("*"+tag + "\n")
    file.write(date + "\n")
    file.write(time + "\n")
    file.write(data + "\n")
    file.write("*end" + "\n")

def createGraph(ax, x, y, param_dict):
    out = ax.plot(x, y, **param_dict)
    return out

if __name__ == '__main__':
    main()


import sys
from collections import deque,defaultdict
input = sys.stdin.readline

for _ in range(int(input())):
    V,E = map(int,input().split(" "))
    d = defaultdict(list)
    for _ in range(E):
        a,b = map(int,input().split(" "))
        d[a].append(b)
        d[b].append(a)

    v = [ -1 for _ in range(V+1) ] # 0: Red , 1: Blue

    answer = True
    for start in range(1,V+1):
        if v[start] == -1:
            dq = deque()
            dq.append((start,0))
            v[start] = 0
            while dq:
                node,color = map(int,dq.popleft())
                for next_node in d[node]:
                    if v[next_node]==color:
                        answer = False
                        break
                    elif v[next_node]==-1:
                        v[next_node]= (color+1)%2 # 다른색으로
                        dq.append((next_node,v[next_node]))
        if answer==False:
            break

    print("YES" if answer else "NO")