# General
import os

# Tools/utils
import itertools
import multiprocessing
from tqdm.notebook import tqdm
from tqdm import tqdm as tqdm_cli
from functools import reduce  # for aggregate functions
from itertools import chain  # for aggregate functions

# Data management
import math
import numpy as np
import pandas as pd
import networkx as nx
import igraph as ig
import leidenalg as la
from community import community_louvain

# Visualization
import matplotlib.pyplot as plt
import seaborn as sns
import pygraphviz as pgv
import colorcet as cc
from matplotlib.colors import ListedColormap
from wordcloud import WordCloud, STOPWORDS
from termcolor import colored  # colored text output

from sklearn.preprocessing import MinMaxScaler

stopwords = STOPWORDS.union({
    'regulation', 'activity', 'positive', 'negative', 'catabolic', 'process', 'protein', 'complex',
    'binding', 'response', 'gene', 'genes', 'encoding', 'defining', 'GeneID', 'regulated',
})

def get_tf_targ_ctx(df):
    tf_target_dict = {'TF': [], 'target': [], 'importance': []}
    tf_target_info = (
        df.droplevel(axis=0, level=1).droplevel(axis=1, level=0)['TargetGenes']
          .map(set)  # transform each list into set
          .groupby('TF').agg(lambda x: reduce(lambda a, b: a.union(b), x))  # combine all targets per TF
    )
    for tf, target_info in tf_target_info.iteritems():
        tf_target_dict['TF'] += [tf for target_name, score in target_info]
        tf_target_dict['target'] += [target_name for target_name, score in target_info]
        tf_target_dict['importance'] += [score for target_name, score in target_info]
    return pd.DataFrame(tf_target_dict)

def netgraph_community_layout(G, node_to_community, community_scale=1., node_scale=2., seed=42):
    """
    Compute the node positions for a modular graph.
    """

    # assert that there multiple communities in the graph; otherwise abort
    communities = set(node_to_community.values())
    if len(communities) < 2:
        warnings.warn("Graph contains a single community. Unable to compute a community layout. Computing spring layout instead.")
        return nx.spring_layout(G, weight='importance', **kwargs)

    community_size = _get_community_sizes(node_to_community)
    community_centroids = _get_community_positions(G, node_to_community, community_scale, seed=seed)
    relative_node_positions = _get_node_positions(G, node_to_community, node_scale, seed=seed)

    # combine positions
    node_positions = dict()
    for node, community in node_to_community.items():
        xy = community_centroids[node]
        delta = relative_node_positions[node] * community_size[community]
        node_positions[node] = xy + delta

    return node_positions

def _get_community_sizes(node_to_community):
    """
    Compute the area of the canvas reserved for each community.
    """

    def _invert_dict(mydict):
        """Invert a dictionary such that values map to keys."""
        inverse = dict()
        for key, value in mydict.items():
            inverse.setdefault(value, set()).add(key)
        return inverse

    scale = (1, 1)

    total_nodes = len(node_to_community)
    max_radius = np.linalg.norm(scale) / 2
    scalar = max_radius / total_nodes
    community_to_nodes = _invert_dict(node_to_community)
    community_size = {community : len(nodes) * scalar for community, nodes in community_to_nodes.items()}

    return community_size

def _get_community_positions(G, node_to_community, community_scale, seed, simple=True):
    """
    Compute a centroid position for each community.
    """

    # create a weighted graph, in which each node corresponds to a community,
    # and each edge weight to the number of edges between communities
    between_community_edges = _find_between_community_edges(G, node_to_community)

    communities = set(node_to_community.values())
    hypergraph = nx.DiGraph()
    hypergraph.add_nodes_from(communities)

    if not simple:
        for (ci, cj), edges in between_community_edges.items():
            hypergraph.add_edge(ci, cj, weight=len(edges))

    # find layout for communities
    pos_communities = nx.spring_layout(hypergraph, scale=community_scale, seed=seed)

    # set node positions to position of community
    pos = dict()
    for node, community in node_to_community.items():
        pos[node] = pos_communities[community]

    return pos

def _find_between_community_edges(G, node_to_community, fixed_community=None):
    """Convert the graph into a weighted network of communities."""
    edges = dict()

    for (ni, nj) in G.edges():
        ci = node_to_community[ni]
        cj = node_to_community[nj]

        if fixed_community is not None:
            if fixed_community != ci and fixed_community != cj:
                continue

        if ci != cj:
            try:
                edges[(ci, cj)] += [(ni, nj)]
            except KeyError:
                edges[(ci, cj)] = [(ni, nj)]

    return edges

def _get_node_positions(G, node_to_community, node_scale, seed):
    """
    Positions nodes within communities.
    """
    communities = dict()
    for node, community in node_to_community.items():
        try:
            communities[community] += [node]
        except KeyError:
            communities[community] = [node]

    pos = dict()
    for ci, nodes in communities.items():
        subgraph = G.subgraph(nodes)
        pos_subgraph = nx.spring_layout(subgraph, weight='importance', scale=node_scale, seed=seed)
        pos.update(pos_subgraph)

    return pos

def squeeze_graph(G, partition, approximate_size=4000):
    """
    Squeeze graph by picking only top nodes (according to number of connections) in each partition. This
    step is needed to speed up the networkx visualization and show only the general POV on the graph.
    """

    #### STEP 1 - filtering nodes

    # Getting the number of partitions
    num_partitions = len(set(partition.values()))

    # Getting partition parameters
    partition_sizes = {i: len([1 for node, k in partition.items() if k == i]) for i in range(num_partitions)}
    min_partition_size = min(partition_sizes.values())

    # Normalizing partition size: divide each partition size by the minimal partition size
    normalized_partition_size = {i: (size // min_partition_size) for i, size in partition_sizes.items()}

    # Getting scale factor - to get approximately size of the graph close to approximate_size
    scale_factor = math.ceil(approximate_size / sum(normalized_partition_size.values()))
    squeezed_partition = {i: (size * scale_factor) for i, size in normalized_partition_size.items()}

    top_nodes = []
    for i, num_nodes in squeezed_partition.items():
        # Getting partition graph
        partition_i = G.subgraph([node for node, k in partition.items() if k == i])

        # Finding inter-community edges
        intercommunity_edges = _find_between_community_edges(G, partition, i)

        # Calculating node importance according to number of inter-community edges
        node_importance = {}
        for (part_1, part_2), edges in intercommunity_edges.items():
            for node_1, node_2 in edges:
                curr_node = node_1 if part_1 == i else node_2
                if curr_node in node_importance:
                    node_importance[curr_node] += 1
                else:
                    node_importance[curr_node] = 1

        # Getting top nodes in the partition according to maximum number of inter-community edge (node_importance)
        top_nodes += list(dict(sorted(node_importance.items(), key=lambda x: x[1], reverse=True)[:squeezed_partition[i]]).keys())

    filtered_partition = {node: i for node, i in partition.items() if node in top_nodes}
    filtered_G = G.subgraph(top_nodes)

    #### STEP 2 - filtering edges

    # Setting up the size of the squeezed graph (number of edges)
    keep_num_edges = 20000
    edges_to_keep = \
        list(
            dict(
                sorted(
                    {
                        (st, end): data['importance'] for st, end, data in filtered_G.edges(data=True)
                    }.items(), key=lambda x: x[1], reverse=True)[:keep_num_edges]
            ).keys()
    )
    squeezed_G = filtered_G.edge_subgraph(edges_to_keep)
    squeezed_partition = {node: i for node, i in filtered_partition.items() if node in squeezed_G.nodes()}

    return squeezed_G, squeezed_partition

def get_elipsis_mask():
        h, w = 600, 800
        center = (int(w/2), int(h/2))
        radius_x = w // 2
        radius_y = h // 2

        Y, X = np.ogrid[:h, :w]
        mask = ((X - center[0])**2/radius_x**2 + (Y - center[1])**2/radius_y**2 >= 1)*255

        return mask

def plot_cloud(G, partition, squeezed_pos, ax, anno_db, filter_genes=True,
               limit_anno_until=50, display_func=False, if_betweenness=True,
               k=3000):
    """
    Plot word cloud that indicates the function(s) of each gene cluster.
    """

    # Loading the gene functional annotation
    gene_func = load_gene_func_db(anno_db, reload=False, as_series=True)

    # Reversing partition dict -> {group_1: [gene_1, gene_2, ...], group_2: [gene_3, gene_4, ...], ...}
    partition_genes_ = {}
    for gene, i in partition.items():
        if i not in partition_genes_.keys():
            partition_genes_[i] = [gene]
        else:
            partition_genes_[i] += [gene]

    # If display gene function in the word clouds
    if display_func:

        # Whether to filter the genes on which we compute the word cloud (most important genes)
        if filter_genes:
            compute_centrality = nx.betweenness_centrality if if_betweenness else nx.closeness_centrality
            distance_metric = {'weight': 'distance'} if if_betweenness else {'distance': 'distance'}
            partition_genes = {}
            t = tqdm(partition_genes_.items())
            for i, genes in t:
                t.set_description(f'Processing cluster {i}, size={G.subgraph(genes).order()}')
                top_len = min(limit_anno_until, len(genes))
                top_gene_scores = dict(
                    sorted(
                        compute_centrality(
                            G.subgraph(genes), k=min(G.subgraph(genes).order(), k), **distance_metric
                        ).items(),
                        key=lambda x: x[1], reverse=True
                    )[:top_len]
                )
                # Renormalizing centrality scores between 1 and 100, and rounding them to use later when
                # displaying wordclouds (higher score - higher "frequency" or word size)
                norm_top_gene_scores = dict(
                    zip(
                        top_gene_scores.keys(), list(map(lambda x: int(x), scale(list(top_gene_scores.values()), 1, 100)))
                    )
                )
                partition_genes[i] = norm_top_gene_scores
            print('Filtered genes for generating the function word cloud..')
        else:
            partition_genes = {{gene_: 1 for gene_ in gene_list} for i, gene_list in partition_genes_.items()}

        # Computing functional annotation for each cluster as a concatenated list of annotations
        # Each annotation is weighted by its duplication gene_score times (e.g. a gene has score = 2 ->
        # the functional annotation is duplicated and have bigger font in WordCloud)
        partition_funcs = {
            i: ' '.join(
                chain.from_iterable([
                   gene_func[gene_func.index == gene].to_list()*gene_score
                        for gene, gene_score in gene_score_list.items()
            ])) for i, gene_score_list in partition_genes.items()
        }

        # Generating word counts from aggregated gene annotation texts -> obtaining main (most frequent) function tokens
        word_counts = {i: WordCloud(max_words=30, min_font_size=15, stopwords=stopwords).process_text(text) for i, text in partition_funcs.items()}
        word_counts = {
            i: (freqs if freqs else {'no found function': 1}) for i, freqs in word_counts.items()
        }  # dealing with no word case
        wordclouds = {
            i: WordCloud(
                max_words=30, min_font_size=15, stopwords=stopwords, background_color='white', mask=get_elipsis_mask()
            ).generate_from_frequencies(freqs) for i, freqs in word_counts.items()
        }

    # Display main genes in decreasing order of importance (top `top_len` genes)
    else:

        compute_centrality = nx.betweenness_centrality if if_betweenness else nx.closeness_centrality
        distance_metric = {'weight': 'distance'} if if_betweenness else {'distance': 'distance'}
        partition_genes = {}
        t = tqdm(partition_genes_.items())
        for i, genes in t:
            t.set_description(f'Processing cluster {i}, size={G.subgraph(genes).order()}')
            top_len = min(limit_anno_until, len(genes))
            top_gene_scores = dict(
                sorted(
                    compute_centrality(
                        G.subgraph(genes), k=min(G.subgraph(genes).order(), k), **distance_metric
                    ).items(),
                    key=lambda x: x[1], reverse=True
                )[:top_len]
            )
            # Renormalizing centrality scores between 1 and 100, and rounding them to use later when
            # displaying wordclouds (higher score - higher "frequency" or word size)
            norm_top_gene_scores = dict(
                zip(
                    top_gene_scores.keys(), list(map(lambda x: int(x), scale(list(top_gene_scores.values()), 1, 100)))
                )
            )
            partition_genes[i] = norm_top_gene_scores
        print('Obtained top genes for generating the gene word cloud..')

        wordclouds = {
            i: WordCloud(
                max_words=30, min_font_size=15, background_color='white', mask=get_elipsis_mask()
            ).generate_from_frequencies(gene_score_dict) for i, gene_score_dict in partition_genes.items()
        }

    # Plotting
    partition_coords = {}
    for gene, coords in squeezed_pos.items():
        if partition[gene] not in partition_coords:
            partition_coords[partition[gene]] = [coords]
        else:
            partition_coords[partition[gene]] += [coords]
    for i, coords in partition_coords.items():
        x, y = zip(*coords)
        min_x, max_x = min(x), max(x)
        min_y, max_y = min(y), max(y)
        ax.imshow(wordclouds[i], interpolation='bilinear', extent=[min_x, max_x, min_y, max_y])

    return ax

def process_communities(data, pat=None, algo='leiden', filter_quantile=0.95, if_betweenness=True,
                        limit_anno_until=50, k=5000, save_top_intercommunity_links_until=20,
                        other_functions_until=20, save_top_new_found_cluster_links=20, seed=42):
    """
    Process graph by finding its communities, annotate its communities, and save everything into .tsv format.
    """

    from joblib import Parallel, delayed

    def highlight_TFs(word, font_size, position, orientation, font_path, random_state):
        TF_color = (255, 0, 0)  # red
        if word in lambert_TF_names or word in dorothea_TF_names:
            return TF_color
        else:
            r, g, b, alpha = plt.get_cmap('viridis')(font_size / 120)
            return (int(r * 255), int(g * 255), int(b * 255))

    print('\nPerforming community analysis..\n\n')

    # Setting pathways to files
    _PROJ_PATH = '/gpfs/projects/bsc08/bsc08890'
    _FMETA = os.path.join(_PROJ_PATH, 'data/GSE145926_RAW/metadata.tsv')
    _DATA_HOME = os.path.join(_PROJ_PATH, 'res/covid_19')

    # Loading sample meta data, reordering patients
    full_meta = pd.read_csv(_FMETA, sep='\t', index_col=0)

    # Prepare everything to save the figs and dataframe
    if data == 'all_data':
        data = 'raw_data'
    elif 'raw_data_' not in data:
        data = f'raw_data_{data}'
    else:
        pass

    if pat is None or pat == 'all_data':

        # Cell-type aggregated data
        data_folder = 'all_data' if data == 'raw_data' else data.replace('raw_data_', '')

        figs_as = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'figs', 'grnboost2', f'raw_data')

        data_to = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'data', 'grnboost2', f'{algo}_communities')
        data_as = os.path.join(data_to, f'raw_data_communities_info.pickle')

    elif pat in ['C', 'M', 'S']:

        # Patient-type aggregated data
        data_folder = 'all_data' if data == 'raw_data' else data.replace('raw_data_', '')

        figs_as = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'figs', 'grnboost2',
                               f'raw_data_{pat}_type')

        data_to = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'data', 'grnboost2', f'{algo}_communities')
        data_as = os.path.join(data_to, f'raw_data_{pat}_type_communities_info.pickle')

    else:

        # Loading patient-specific data
        figs_as = os.path.join(_DATA_HOME, pat, 'figs', 'grnboost2', f'{data}')

        data_to = os.path.join(_DATA_HOME, pat, 'data', 'grnboost2', f'{algo}_communities')
        data_as = os.path.join(data_to, f'{data}_communities_info.pickle')

    os.makedirs(data_to, exist_ok=True)
    os.makedirs(os.path.dirname(figs_as), exist_ok=True)

    # Loading lists of TFs from Lambert 2018 and DoRothEA, in the latter case we will keep only confident regulons
    lambert_TF_names = pd.read_csv(os.path.join(_PROJ_PATH, 'data/TF_lists/lambert2018.txt'), header=None)[0].to_list()
    dorothea_TF_names = list(
        pd.read_csv(os.path.join(_PROJ_PATH, 'data/TF_lists/dorothea_regulons.tsv'), sep='\t') \
            .loc[lambda x: x['confidence'].isin(['A', 'B', 'C'])]['tf'].unique()
    )

    # Loading the graph
    G = get_nx_graph(data=data, data_type='all', pat=pat, get_filtered=filter_quantile)
    print(f"Loaded the graph: {colored('pat', 'green')}='{colored(pat, 'red')}', "
          f"{colored('data', 'green')}='{colored(data, 'red')}', "
          f"{colored('data_type', 'green')}='{colored('all', 'red')}'\n")

    ###### FINDING COMMUNITIES IN THE GRAPH #######

    print('Finding communities in the graph..')

    if algo == 'louvain':
        partition = community_louvain.best_partition(G.to_undirected(), weight='importance', random_state=seed)
    elif algo == 'leiden':
        G_igraph = ig.Graph.from_networkx(G.to_undirected())
        la_partition = la.find_partition(G_igraph, la.ModularityVertexPartition, weights='importance', seed=seed)
        partition = {G_igraph.vs[node]['_nx_name']: i for i, cluster_nodes in enumerate(la_partition) for node in cluster_nodes}
    else:
        raise NotImplementedError

    num_partitions = len(set(partition.values()))
    print(f'Number of partitions using {algo} algorithm: {colored(num_partitions, "cyan")}\n')

    ###### FINDING HIGH-CENTRALITY GENES IN THE WHOLE GRAPH

    print('Finding high-centrality genes in the whole graph..')

    num_workers = max(multiprocessing.cpu_count() // 2, 1)
    whole_G_central_genes = dict(
        sorted(betweenness_centrality_parallel(G, processes=num_workers).items(), key=lambda x: x[1], reverse=True)[:limit_anno_until]
    )
    print(f'Computed the {"betweenness" if if_betweenness else "closeness"} centrality for all genes in the graph\n')

    ###### FINDING HIGH-CENTRALITY GENES AND CORRESPONDING FUNCTIONS IN EACH COMMUNITY USING GO ANNOTATION ######

    print('Finding high-centrality genes/functions in each cluster..')

    # Loading the gene functional annotation
    anno_db_tags = ['GO', 'KEGG', 'immunological', 'hallmark']
    gene_func_dbs = {tag: load_gene_func_db(tag, as_series=True) for tag in anno_db_tags}

    # Reversing partition dict -> {group_1: [gene_1, gene_2, ...], group_2: [gene_3, gene_4, ...], ...}
    partition_genes_ = {}
    for gene, i in partition.items():
        if i not in partition_genes_.keys():
            partition_genes_[i] = [gene]
        else:
            partition_genes_[i] += [gene]

    # Whether to filter the genes on which we compute the word cloud (most important genes)
    compute_centrality = nx.betweenness_centrality if if_betweenness else nx.closeness_centrality
    distance_metric = {'weight': 'distance'} if if_betweenness else {'distance': 'distance'}
    all_partition_genes = {}
    norm_partition_genes = {}
    t = tqdm_cli(partition_genes_.items(), ascii=True)
    for i, genes in t:
        t.set_description(f'Processing cluster {i}, size={G.subgraph(genes).order()}')
        gene_scores = dict(
            sorted(
                compute_centrality(
                    G.subgraph(genes), k=min(G.subgraph(genes).order(), k), normalized=True, **distance_metric
                ).items(),
                key=lambda x: x[1], reverse=True
            )
        )
        all_partition_genes[i] = gene_scores
        central_gene_scores = {gene: gene_scores[gene] for k, gene in enumerate(gene_scores.keys()) if k < limit_anno_until}

        # Renormalizing centrality scores between 1 and 100, and rounding them to use later when
        # displaying wordclouds (higher score - higher "frequency" or word size)
        norm_partition_genes[i] = dict(
            zip(
                central_gene_scores.keys(),
                list(map(lambda x: int(x), scale(list(central_gene_scores.values()), 1, 100)))
            )
        )
    print('Computed centrality scores for each gene in each community\n')

    print('Finding functional annotations for each cluster..')

    # Computing functional annotation for each cluster as a concatenated list of annotations
    # Each annotation is weighted by its duplication gene_score times (e.g. a gene has score = 2 ->
    # the functional annotation is duplicated and have bigger font in WordCloud)
    # We also do it for different functional annotations like GO, KEGG, Hallmark, etc..
    partition_funcs = {
        tag:
            {
                i: ' '.join(
                    chain.from_iterable([
                       gene_func[gene_func.index == gene].to_list()*gene_score
                            for gene, gene_score in gene_score_list.items()
                ])) for i, gene_score_list in norm_partition_genes.items()
            } for tag, gene_func in gene_func_dbs.items()
    }

    print('Computed functional annotations for each cluster\n')

    ###### PLOTTING GENE AND FUNC COMMUNITY CLOUDS ######

    print('Plotting clusters..')

    # Getting positions of squeezed graph - we do not plot every gene on the figure
    squeezed_G, squeezed_partition = squeeze_graph(G, partition)
    print('Computed a squeezed graph representation..')

    squeezed_pos = netgraph_community_layout(squeezed_G, squeezed_partition, seed=seed)  # nx.nx_agraph.pygraphviz_layout(G.to_undirected(), prog="sfdp")  # nx.nx.spring_layout(G, seed=seed, k=0.2, iterations=20)
    partition_coords = {}
    for gene, coords in squeezed_pos.items():
        if partition[gene] not in partition_coords:
            partition_coords[partition[gene]] = [coords]
        else:
            partition_coords[partition[gene]] += [coords]
    print('Computed node positions of the squeezed graph representation..')

    cmap = ListedColormap(sns.color_palette(cc.glasbey_bw, n_colors=num_partitions).as_hex())

    for plot_type in ['genes'] + list(map(lambda x: f"func_{x}", anno_db_tags)):

        if plot_type.startswith('func'):
            # Getting current functional annotation
            curr_partition_funcs = partition_funcs[plot_type[plot_type.find('_') + 1:]]

        f, ax = plt.subplots(figsize=(20, 35))

        if plot_type == 'genes':
            wordclouds = {
                i: WordCloud(
                    max_words=30, min_font_size=15, background_color='white', mask=get_elipsis_mask()
                ).generate_from_frequencies(gene_score_dict).recolor(color_func=highlight_TFs)
                    for i, gene_score_dict in norm_partition_genes.items()
            }
        else:
            word_counts = {
                i: WordCloud(max_words=30, min_font_size=15, stopwords=stopwords).process_text(text) for i, text in curr_partition_funcs.items()
            }
            word_counts = {
                i: (freqs if freqs else {'no found function': 1}) for i, freqs in word_counts.items()
            }  # dealing with no word case
            wordclouds = {
                i: WordCloud(
                    max_words=30, min_font_size=15, stopwords=stopwords, background_color='white', mask=get_elipsis_mask()
                ).generate_from_frequencies(freqs) for i, freqs in word_counts.items()
            }

        # Plotting clouds
        for i, coords in partition_coords.items():
            x, y = zip(*coords)
            min_x, max_x = min(x), max(x)
            min_y, max_y = min(y), max(y)
            ax.imshow(wordclouds[i], interpolation='bilinear', extent=[min_x, max_x, min_y, max_y])
        print(f'Finished plotting {plot_type} word cloud..')

        nx.draw(squeezed_G, squeezed_pos, ax=ax, arrowstyle="->", arrowsize=20,
                connectionstyle=f'arc3, rad = 0.25', edge_color='gray', width=0.4,
                node_color='k', node_size=50, alpha=0.02)
        nx.draw_networkx_nodes(squeezed_G, squeezed_pos, ax=ax, node_size=100,
                               nodelist=list(squeezed_partition.keys()),
                               node_color=list(squeezed_partition.values()),
                               cmap=cmap, alpha=0.005)
        print(f'Finished plotting {plot_type} nodes..')

        ax.set_title(f'Found communities ({pat}, "all", {data}), '
                     f'annotation - {plot_type}',
                     fontsize=30)
        plt.axis('off')

        plt.savefig(f'{figs_as}_{plot_type}.png', bbox_inches='tight', dpi=400)

    print('Finished plotting..\n')

    ###### SAVING DATAFRAME CONTAINING INFORMATION ABOUT EACH COMMUNITY ######

    def compute_community_info(i):
        """
        Parallel saving of the dataframe.
        """

        # Getting information for each community
        genes = list(all_partition_genes[i].keys())
        community_subgraph = G.subgraph(genes)

        communities_i = pd.Series(dtype='object')

        # Setting tqdm logs
        # t.set_description(f'Saving info about {i} cluster, size={community_subgraph.order()}')

        # Getting information about cluster genes
        central_genes_and_scores = {
            gene: all_partition_genes[i][gene] for k, gene in enumerate(genes) if k < limit_anno_until
        }

        non_lambert_TFs = [
            f'{gene} (rank={k})' for k, gene in enumerate(central_genes_and_scores.keys(), start=1) if gene not in lambert_TF_names
        ]
        non_dorothea_TFs = [
            f'{gene} (rank={k})' for k, gene in enumerate(central_genes_and_scores.keys(), start=1) if gene not in dorothea_TF_names
        ]

        # Filling dataframe with the information
        communities_i['num_nodes'] = community_subgraph.number_of_nodes()
        communities_i['num_edges'] = community_subgraph.number_of_edges()
        communities_i['all_sorted_genes'] = '; '.join(
            f'{gene} (score={score})' for gene, score in all_partition_genes[i].items()
        )
        communities_i['sorted_central_genes_scores'] = '; '.join(
            f'{gene} (score={score:.2f})' for gene, score in central_genes_and_scores.items()
        )
        communities_i['non_lambert_2018_TF_central_genes'] = '; '.join(non_lambert_TFs)
        communities_i['non_dorothea_TF_central_genes'] = '; '.join(non_dorothea_TFs)
        communities_i['whole_G_central_genes_scores'] = '; '.join(
            f'{gene} (score={score:.2f})' for gene, score in whole_G_central_genes.items()
        )

        # Filling information about newly found gene-gene links (based on absence in KEGG and Hallmark)
        top_cluster_links = set()

        iter_i = 0

        for st, end, edge_info in sorted(community_subgraph.edges(data=True),
                                         key=lambda t: t[2]['importance'],
                                         reverse=True):

            # If the current (reverse directed) link was not encountered previously..
            if (end, st) not in [(uniq_st, uniq_end) for uniq_st, uniq_end, _ in top_cluster_links]:
                top_cluster_links.add((st, end, edge_info['importance']))
                iter_i += 1
            if iter_i == save_top_new_found_cluster_links:
                break

        for anno_tag in ['KEGG', 'hallmark']:

            curr_db = load_gene_func_db(anno_tag)
            tmp_list = []

            # if `st` gene and `end` gene have non-overlapping annotations..
            for st, end, imp in top_cluster_links:
                st_anno_IDs = set(curr_db[curr_db.index == st]['ID'])
                end_anno_IDs = set(curr_db[curr_db.index == end]['ID'])
                if len(st_anno_IDs.intersection(end_anno_IDs)) == 0 and \
                        (len(st_anno_IDs) != 0 or len(end_anno_IDs) != 0):
                    tmp_list.append(f"{st} ({' & '.join(st_anno_IDs)}) <-> {end} ({' & '.join(end_anno_IDs)})")

            communities_i[f'new_gene_gene_links_{anno_tag}'] = '; '.join(tmp_list)

        # Filling information about cluster functions
        for tag, gene_func in gene_func_dbs.items():

            curr_partition_funcs = partition_funcs[tag]

            # Filling main functions - non duplicates at the top
            main_functions = list(dict.fromkeys([  # dropping duplicates, but preserving order
                func for gene in central_genes_and_scores.keys()
                    for func in gene_func[gene_func.index == gene].to_list()
            ]))
            gene_with_main_functions = [
                ','.join(
                    gene_func[gene_func == func].loc[lambda x: x.index.isin(genes)].index.to_list()
                ) for func in main_functions
            ]
            main_functions = [
                f'>>> {func} <<<: {gene}' for gene, func in zip(gene_with_main_functions, main_functions)
            ]
            communities_i[f'main_functions_{tag}'] = '; '.join(main_functions)  # saving..

            # Saving functions corresponding to each gene
            central_functions_per_gene = [
                f">>> {gene} <<<: {' & '.join(gene_func[gene_func.index == gene].to_list())}" for gene in central_genes_and_scores.keys()
            ]
            communities_i[f'sorted_central_functions_{tag}'] = '; '.join(central_functions_per_gene)  # saving..

            # Saving most frequent function words
            freq_words = WordCloud(
                max_words=30, min_font_size=15, stopwords=stopwords
            ).process_text(curr_partition_funcs[i])
            freq_words = dict(
                sorted(freq_words.items(), key=lambda x: x[1], reverse=True)
            ) if freq_words else {'no found function': 1}  # dealing with no word case
            communities_i[f'most_frequent_function_words_{tag}'] = '; '.join(freq_words.keys())  # saving

            # Saving other functions present in this cluster
            other_functions = list(dict.fromkeys([  # dropping duplicates, but preserving order
                func for gene in genes if gene not in central_genes_and_scores.keys()
                    for func in gene_func[gene_func.index == gene].to_list() if func not in main_functions
            ]))[:other_functions_until]
            genes_with_other_functions = [
                ','.join(
                    gene_func[gene_func == func].loc[lambda x: x.index.isin(genes)].index.to_list()
                ) for func in other_functions
            ]
            other_functions = [
                f'>>> {func} <<<: {gene}' for gene, func in zip(genes_with_other_functions, other_functions)
            ]
            communities_i[f'other_functions_{tag}'] = '; '.join(other_functions)  # saving

        # Filling information about top inter-community links
        # t_sub = tqdm(range(num_partitions), ascii=True, leave=False)
        for k in range(num_partitions):  # t_sub:
            # t_sub.set_description(f'Extracting top inter-community links with {k}')

            if i != k:
                genes_in_k = list(all_partition_genes[k].keys())

                # Getting the subgraph that contains central genes in community_i and all genes in comunity_k
                G_central_i_k = G.subgraph(list(central_genes_and_scores.keys()) + genes_in_k)
                # Getting the subgraph that contains all genes from community_i and community_k
                G_i_k = G.subgraph(genes + genes_in_k)

                # Creating two helper sets that allow us to keep only unique links
                links_central_i_k = set()
                links_i_k = set()

                iter_i = 0

                # Getting out top links from the second subgraph
                for st, end, edge_info in sorted(G_central_i_k.edges(data=True),
                                                 key=lambda t: t[2]['importance'],
                                                 reverse=True):
                    # If the current (reverse directed) link was not encountered previously..
                    if (end, st) not in [(uniq_st, uniq_end) for uniq_st, uniq_end, _ in links_central_i_k] and \
                            ((st in genes and end not in genes) or (end in genes and st in genes)):
                        links_central_i_k.add((st, end, edge_info['importance']))
                        iter_i += 1
                    if iter_i == save_top_intercommunity_links_until:
                        break

                iter_i = 0

                # Getting out top links from the second subgraph
                for st, end, edge_info in sorted(G_i_k.edges(data=True),
                                                 key=lambda t: t[2]['importance'],
                                                 reverse=True):
                    # If the current (reverse directed) link was not encountered previously..
                    if (end, st) not in [(uniq_st, uniq_end) for uniq_st, uniq_end, _ in links_i_k] and \
                            ((st in genes and end not in genes) or (end in genes and st in genes)):
                        links_i_k.add((st, end, edge_info['importance']))
                        iter_i += 1
                    if iter_i == save_top_intercommunity_links_until:
                        break

                # Adding top links to the dataframe
                communities_i[f'top_links_scores_central_genes<->community_{k}'] = \
                    '; '.join(f'{st} <-> {end} (score={score:.2f})' for st, end, score in links_central_i_k)
                communities_i[f'top_links_scores_with_community_{k}'] = \
                    '; '.join([f'{st} <-> {end} (score={score:.2f})' for st, end, score in links_i_k])

        return communities_i

    print('Saving info dataframe..')

    t = tqdm_cli(range(num_partitions), ascii=True)

    # Getting dataframe
    result = Parallel(n_jobs=num_workers)(delayed(compute_community_info)(i) for i in t)
    communities_df = pd.concat(result, axis=1).T.reindex(
        columns=[
            'num_nodes', 'num_edges',
            'main_functions_GO', 'main_functions_KEGG', 'main_functions_immunological', 'main_functions_hallmark',
            'non_lambert_2018_TF_central_genes', 'non_dorothea_TF_central_genes',
            'new_gene_gene_links_KEGG', 'new_gene_gene_links_hallmark',
            'whole_G_central_genes_scores',
            'other_functions_GO', 'other_functions_KEGG', 'other_functions_immunological', 'other_functions_hallmark',
            'sorted_central_genes_scores',
            'sorted_central_functions_GO', 'sorted_central_functions_KEGG', 'sorted_central_functions_immunological', 'sorted_central_functions_hallmark',
            'most_frequent_function_words_GO', 'most_frequent_function_words_KEGG', 'most_frequent_function_words_immunological', 'most_frequent_function_words_hallmark',
            'all_sorted_genes'] +
            [f'top_links_scores_central_genes<->community_{i}' for i in range(num_partitions)] +
            [f'top_links_scores_with_community_{i}' for i in range(num_partitions)
            ]
    )

    # Saving dataframe
    communities_df.to_pickle(data_as)
    print(f"Saved the data to {data_as}!\n")

def run_enrichr(data, is_communities=False, is_positive_markers=True, group_types = 'all', on_targets=False, choose_fixed_tf=None,
                data_type='all', top_n=50, algo='leiden', enrichr_library='MSigDB_Hallmark_2020'):
    """
    Run enrichment analysis with Enrichr.
    """

    import json
    import requests
    import sys
    import io

    out_folder = 'community_ana' if is_communities else 'cohort_ana'

    if is_communities == True:

        print('Running EnrichR on communities..')

        algo = 'leiden'
        _DATA_HOME = '/gpfs/projects/bsc08/bsc08890/res/covid_19'

        if data_type == 'all':
            community_data = pd.read_pickle(os.path.join(
                _DATA_HOME, 'cell_types', data, 'data', 'grnboost2', f'{algo}_communities',
                f'raw_data_communities_info.pickle'
            ))
        else:
            community_data = pd.read_pickle(os.path.join(
                _DATA_HOME, 'cell_types', data, 'data', 'grnboost2', f'{algo}_communities',
                f'raw_data_{data_type}_type_communities_info.pickle'
            ))

        df = pd.concat([
            pd.DataFrame({
                'cluster': f'cluster_{i}',
                'gene': [el[: el.find(' ')] for el in vals.split('; ')][:top_n]
            }) for i, vals in community_data['all_sorted_genes'].iteritems()
        ], axis=0).reset_index(drop=True)

    else:

        if on_targets:

            print('Running EnrichR on targets between 3 group types..')

            types = ['C', 'M', 'S']

            df = pd.concat([
                pd.read_csv(
                    f'/gpfs/home/bsc08/bsc08890/tmp/cohort_ana/tmp_enrichr_{data}_{t}_{choose_fixed_tf}_target_list.tsv',
                    header=None, names=['gene']
                ).assign(cluster=t) for t in types
            ], axis=0)

        else:

            if group_types == 'all':
                print('Running EnrichR on TFs between 3 group types..')
                df = pd.read_csv(f'/gpfs/home/bsc08/bsc08890/tmp/tf_markers_df_{data}.tsv', sep='\t')
            else:
                print('Running EnrichR on 2 group types..')
                if group_types == 'M_S':
                    group_types = 'S_M'
                if group_types == 'C_M':
                    group_types = 'M_C'
                if group_types == 'C_S':
                    group_types = 'S_C'
                df_1 = pd.read_csv(f'/gpfs/home/bsc08/bsc08890/tmp/tf_markers_df_{group_types}_{data}.tsv', sep='\t')
                df_1['gene'] = df_1.index
                df_2 = df_1.copy()
                df_2['avg_log2FC'] = - df_2['avg_log2FC']
                df_1['cluster'], df_2['cluster'] = group_types.split('_')

                df = pd.concat([df_1, df_2], axis=0)

            if is_positive_markers:
                df = df[(df['p_val_adj'] < 0.05) & (df['avg_log2FC'] > 1)]
            else:
                df = df[(df['p_val_adj'] < 0.05) & (df['avg_log2FC'] < -1)]

    cluster_dfs = {}
    for cl in df['cluster'].unique():

        print(f'Processing {cl}..')

        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/addList'
        genes_str = '\n'.join(df[df['cluster'] == cl]['gene'])
        description = f"{data}_{data_type}_{cl}"

        if is_communities == True:
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_{data_type}_{cl}.tsv'
        elif on_targets:
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_{data_type}_{choose_fixed_tf}_target_{cl}.tsv'
        elif group_types == 'all':
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_{data_type}_{cl}.tsv'
        else:
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_2_groups_{cl}.tsv'

        payload = {
          'list': (None, genes_str),
          'description': (None, description)
        }
        response = requests.post(ENRICHR_URL, files=payload)

        if not response.ok:
            raise Exception('Error analyzing gene list')

        job_id = json.loads(response.text)

        ################################################################################
        # Get enrichment results
        #
        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/export'
        query_string = '?userListId=%s&filename=%s&backgroundType=%s'
        user_list_id = str(job_id['userListId'])
        gene_set_library = str(enrichr_library)
        url = ENRICHR_URL + query_string % (user_list_id, filename, gene_set_library)

        response = requests.get(url, stream=True)

        print('        Enrichr API : Downloading file of enrichment results: Job Id:', job_id)
        with open(filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)

        print(f'        Saved to {filename}')

        cluster_dfs[cl] = pd.read_csv(filename, sep='\t')

    return cluster_dfs

def betweenness_centrality_parallel(G, processes=None):
    """Parallel betweenness centrality  function"""
    from multiprocessing import Pool

    def chunks(l, n):
        """Divide a list of nodes `l` in `n` chunks"""
        l_c = iter(l)
        while 1:
            x = tuple(itertools.islice(l_c, n))
            if not x:
                return
            yield x

    p = Pool(processes=processes)
    node_divisor = len(p._pool) * 4
    node_chunks = list(chunks(G.nodes(), int(G.order() / node_divisor)))
    num_chunks = len(node_chunks)
    bt_sc = p.starmap(
        nx.betweenness_centrality_subset,
        zip(
            [G] * num_chunks,
            node_chunks,
            [list(G)] * num_chunks,
            [True] * num_chunks,
            ['distance'] * num_chunks
        ),
    )

    # Reduce the partial solutions
    bt_c = bt_sc[0]
    for bt in bt_sc[1:]:
        for n in bt:
            bt_c[n] += bt[n]
    return bt_c


# -*- coding: utf-8 -*-

import xml.etree.ElementTree as ET
from xml.etree import ElementTree as etree
from xml.dom import minidom
import untangle

def xml_generator(input_filename, input_foldername, exif_list, root_path):
    root = ET.Element('annotation')

    source = ET.SubElement(root, 'source')
    image_date = ET.SubElement(source, 'date')
    image_date.text = str(exif_list[0])
    folder_name = ET.SubElement(source, 'folder')
    folder_name.text = input_foldername
    file_name = ET.SubElement(source, 'filename')
    file_name.text = input_filename

    gpsinfo = ET.SubElement(root, 'gpsinfo')
    gps_altitude = ET.SubElement(gpsinfo, 'GPSAltitude')
    gps_altitude.text = str(exif_list[1])
    gps_latitude = ET.SubElement(gpsinfo, 'GPSLatitude')
    gps_latitude.text = str(exif_list[2])
    gps_latitude_ref = ET.SubElement(gpsinfo, 'GPSLatitudeRef')
    gps_latitude_ref.text = str(exif_list[3])
    gps_longitude = ET.SubElement(gpsinfo, 'GPSLongitude')
    gps_longitude.text = str(exif_list[4])
    gps_longitude_ref = ET.SubElement(gpsinfo, 'GPSLongitudeRef')
    gps_longitude_ref.text = str(exif_list[5])

    '''
    There should be position annotation inside 'object' tag
    '''
    #ann_obj = ET.SubElement(root, 'object')

    xml_string = etree.tostring(root)
    tree = minidom.parseString(xml_string)
    xml_string = tree.toxml()

    save_path = '%s/ob_%s/%s.xml' % (root_path, input_foldername, input_filename[:-4])

    f=open(save_path,'wb')
    f.write(tree.toprettyxml(encoding='utf-8'))
    f.close()

def xml_parsing(input_xml_file):
    obj = untangle.parse(input_xml_file)

    date_time = obj.annotation.source.date.cdata
    GPSAltitude = obj.annotation.gpsinfo.GPSAltitude.cdata
    GPSLatitude = obj.annotation.gpsinfo.GPSLatitude.cdata
    GPSLatitudeRef = obj.annotation.gpsinfo.GPSLatitudeRef.cdata
    GPSLongitude = obj.annotation.gpsinfo.GPSLongitude.cdata
    GPSLongitudeRef = obj.annotation.gpsinfo.GPSLongitudeRef.cdata

    xml_info_keys = ['Date', 'GPSAltitude', 'GPSLatitude', 'GPSLatitudeRef', 'GPSLongitude', 'GPSLongitudeRef']
    xml_info_value = [date_time, GPSAltitude, GPSLatitude, GPSLatitudeRef, GPSLongitude, GPSLongitudeRef]
    xml_info_dict = dict(zip(xml_info_keys, xml_info_value))
    return xml_info_dict

#im = '/Users/xiang/ml_ann/ann_tools_eric/dataset/ob_curr/00001.xml'
#xml_parsing(im)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Author: Xiaoy LI
# Description:
# run_machine_comprehension.py
# Please Notice that the data should contain
# multi answers
# need pay MORE attention when loading data

import os
import argparse
import numpy as np
import random
import torch

from data_loader.model_config import Config
from data_loader.mrc_data_loader import MRCNERDataLoader
from data_loader.mrc_data_processor import Conll03Processor, MSRAProcessor, Onto4ZhProcessor, Onto5EngProcessor, GeniaProcessor, ACE2004Processor, ACE2005Processor, ResumeZhProcessor
from layer.optim import AdamW, lr_linear_decay, BertAdam
from model.bert_mrc import BertQueryNER
from data_loader.bert_tokenizer import BertTokenizer4Tagger
from metric.mrc_ner_evaluate  import flat_ner_performance, nested_ner_performance

def args_parser():
    # start parser
    parser = argparse.ArgumentParser()

    # requires parameters
    parser.add_argument("--config_path", default="/home/lixiaoya/", type=str)
    parser.add_argument("--data_dir", default=None, type=str)
    parser.add_argument("--bert_model", default=None, type=str,)
    parser.add_argument("--task_name", default=None, type=str)
    parser.add_argument("--max_seq_length", default=128, type=int)
    parser.add_argument("--train_batch_size", default=32, type=int)
    parser.add_argument("--dev_batch_size", default=32, type=int)
    parser.add_argument("--test_batch_size", default=32, type=int)
    parser.add_argument("--checkpoint", default=100, type=int)
    parser.add_argument("--learning_rate", default=5e-5, type=float)
    parser.add_argument("--num_train_epochs", default=5, type=int)
    parser.add_argument("--warmup_proportion", default=0.1, type=float)
    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
    parser.add_argument("--gradient_accumulation_steps", type=int, default=1)
    parser.add_argument("--seed", type=int, default=3006)
    parser.add_argument("--output_dir", type=str, default="/home/lixiaoya/output")
    parser.add_argument("--data_sign", type=str, default="msra_ner")
    parser.add_argument("--weight_start", type=float, default=1.0)
    parser.add_argument("--weight_end", type=float, default=1.0)
    parser.add_argument("--weight_span", type=float, default=1.0)
    parser.add_argument("--entity_sign", type=str, default="flat")
    parser.add_argument("--n_gpu", type=int, default=1)
    parser.add_argument("--dropout", type=float, default=0.2)
    parser.add_argument("--entity_threshold", type=float, default=0.5)
    parser.add_argument("--num_data_processor", default=1, type=int, help="number of data processor.")
    parser.add_argument("--data_cache", default=True, action='store_false')
    parser.add_argument("--export_model", default=True, action='store_false')
    parser.add_argument("--do_lower_case", default=False, action='store_true', help="lower case of input tokens.")
    parser.add_argument('--fp16', default=False, action='store_true', help="Whether to use MIX 16-bit float precision instead of 32-bit")
    parser.add_argument("--amp_level", default="O2", type=str, help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3'].See details at https://nvidia.github.io/apex/amp.html")
    parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus")

    args = parser.parse_args()

    args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps

    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)

    os.makedirs(args.output_dir, exist_ok=True)

    return args

def load_data(config):

    print("-*-"*10)
    print("current data_sign: {}".format(config.data_sign))

    if config.data_sign == "conll03":
        data_processor = Conll03Processor()
    elif config.data_sign == "zh_msra":
        data_processor = MSRAProcessor()
    elif config.data_sign == "zh_onto":
        data_processor = Onto4ZhProcessor()
    elif config.data_sign == "en_onto":
        data_processor = Onto5EngProcessor()
    elif config.data_sign == "genia":
        data_processor = GeniaProcessor()
    elif config.data_sign == "ace2004":
        data_processor = ACE2004Processor()
    elif config.data_sign == "ace2005":
        data_processor = ACE2005Processor()
    elif config.data_sign == "resume":
            data_processor = ResumeZhProcessor()
    else:
        raise ValueError("Please Notice that your data_sign DO NOT exits !!!!!")

    label_list = data_processor.get_labels()
    tokenizer = BertTokenizer4Tagger.from_pretrained(config.bert_model, do_lower_case=config.do_lower_case)

    dataset_loaders = MRCNERDataLoader(config, data_processor, label_list, tokenizer, mode="train", allow_impossible=True)
    train_dataloader = dataset_loaders.get_dataloader(data_sign="train", num_data_processor=config.num_data_processor)
    dev_dataloader = dataset_loaders.get_dataloader(data_sign="dev", num_data_processor=config.num_data_processor)
    test_dataloader = dataset_loaders.get_dataloader(data_sign="test", num_data_processor=config.num_data_processor)
    num_train_steps = dataset_loaders.get_num_train_epochs()

    return train_dataloader, dev_dataloader, test_dataloader, num_train_steps, label_list

def load_model(config, num_train_steps, label_list):
    device = torch.device("cuda")
    n_gpu = config.n_gpu
    model = BertQueryNER(config, )
    model.to(device)
    if config.n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # prepare optimzier
    param_optimizer = list(model.named_parameters())

    no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight", 'gamma', 'beta']
    optimizer_grouped_parameters = [
    {"params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], "weight_decay": 0.01},
    {"params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], "weight_decay": 0.0}]

    optimizer = AdamW(optimizer_grouped_parameters, lr=config.learning_rate, betas=(0.9, 0.98), eps=1e-6, weight_decay=0.01)
    # optimizer = BertAdam(optimizer_grouped_parameters, lr=config.learning_rate, warmup=config.warmup_proportion,
    #                     t_total=num_train_steps, max_grad_norm=config.clip_grad)
    sheduler = None

    if config.fp16:
        try:
            from apex import amp
        except ImportError:
            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
        model, optimizer = amp.initialize(model, optimizer, opt_level=config.amp_level)

    # Distributed training (should be after apex fp16 initialization)
    if config.local_rank != -1:
        model = torch.nn.parallel.DistributedDataParallel(
            model, device_ids=[config.local_rank], output_device=config.local_rank, find_unused_parameters=True
            )

    return model, optimizer, sheduler, device, n_gpu

def train(model, optimizer, sheduler,  train_dataloader, dev_dataloader, test_dataloader, config, \
    device, n_gpu, label_list):

    dev_best_acc = 0
    dev_best_precision = 0
    dev_best_recall = 0
    dev_best_f1 = 0
    dev_best_loss = 10000000000000

    test_acc_when_dev_best = 0
    test_pre_when_dev_best = 0
    test_rec_when_dev_best = 0
    test_f1_when_dev_best = 0
    test_loss_when_dev_best = 1000000000000000

    model.train()
    for idx in range(int(config.num_train_epochs)):
        tr_loss = 0
        nb_tr_examples, nb_tr_steps = 0, 0
        print("#######"*10)
        print("EPOCH: ", str(idx))
        if idx != 0:
            lr_linear_decay(optimizer)
        for step, batch in enumerate(train_dataloader):
            batch = tuple(t.to(device) for t in batch)
            input_ids, input_mask, segment_ids, start_pos, end_pos, span_pos, span_label_mask, ner_cate = batch

            loss = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask, \
                start_positions=start_pos, end_positions=end_pos, span_positions=span_pos, span_label_mask=span_label_mask)

            if config.n_gpu > 1:
                loss = loss.mean()

            if config.fp16:
                from apex import amp
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
                torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), config.max_grad_norm)
            else:
                loss.backward()
                torch.nn.utils.clip_grad_norm_(model.parameters(), config.max_grad_norm)

            optimizer.step()
            model.zero_grad()

            tr_loss += loss.item()
            nb_tr_examples += input_ids.size(0)
            nb_tr_steps += 1

            if nb_tr_steps % config.checkpoint == 0:
                print("-*-"*15)
                print("current training loss is : ")
                print(loss.item())
                model, tmp_dev_loss, tmp_dev_acc, tmp_dev_prec, tmp_dev_rec, tmp_dev_f1 = eval_checkpoint(model, dev_dataloader, config, device, n_gpu, label_list, eval_sign="dev")
                print("......"*10)
                print("DEV: loss, acc, precision, recall, f1")
                print(tmp_dev_loss, tmp_dev_acc, tmp_dev_prec, tmp_dev_rec, tmp_dev_f1)

                if tmp_dev_f1 > dev_best_f1 :
                    dev_best_acc = tmp_dev_acc
                    dev_best_loss = tmp_dev_loss
                    dev_best_precision = tmp_dev_prec
                    dev_best_recall = tmp_dev_rec
                    dev_best_f1 = tmp_dev_f1

                    # export model
                    if config.export_model:
                        model_to_save = model.module if hasattr(model, "module") else model
                        output_model_file = os.path.join(config.output_dir, "bert_finetune_model_{}_{}.bin".format(str(idx),str(nb_tr_steps)))
                        torch.save(model_to_save.state_dict(), output_model_file)
                        print("SAVED model path is :")
                        print(output_model_file)

                    model = model.cuda().to(device)
                    model, tmp_test_loss, tmp_test_acc, tmp_test_prec, tmp_test_rec, tmp_test_f1 = eval_checkpoint(model, test_dataloader, config, device, n_gpu, label_list, eval_sign="test")
                    print("......"*10)
                    print("TEST: loss, acc, precision, recall, f1")
                    print(tmp_test_loss, tmp_test_acc, tmp_test_prec, tmp_test_rec, tmp_test_f1)

                    test_acc_when_dev_best = tmp_test_acc
                    test_pre_when_dev_best = tmp_test_prec
                    test_rec_when_dev_best = tmp_test_rec
                    test_f1_when_dev_best = tmp_test_f1
                    test_loss_when_dev_best = tmp_test_loss
                    model = model.cuda().to(device)

                print("-*-"*15)

    print("=&="*15)
    print("Best DEV : overall best loss, acc, precision, recall, f1 ")
    print(dev_best_loss, dev_best_acc, dev_best_precision, dev_best_recall, dev_best_f1)
    print("scores on TEST when Best DEV:loss, acc, precision, recall, f1 ")
    print(test_loss_when_dev_best, test_acc_when_dev_best, test_pre_when_dev_best, test_rec_when_dev_best, test_f1_when_dev_best)
    print("=&="*15)

def eval_checkpoint(model_object, eval_dataloader, config, device, n_gpu, label_list, eval_sign="dev"):
    # input_dataloader type can only be one of dev_dataloader, test_dataloader

    eval_loss = 0
    start_pred_lst = []
    end_pred_lst = []
    span_pred_lst = []
    start_scores_lst = []
    end_scores_lst = []
    mask_lst = []
    start_gold_lst = []
    span_gold_lst = []
    end_gold_lst = []
    eval_steps = 0
    ner_cate_lst = []

    for input_ids, input_mask, segment_ids, start_pos, end_pos, span_pos, span_label_mask, ner_cate in eval_dataloader:
        input_ids = input_ids.to(device)
        input_mask = input_mask.to(device)
        segment_ids = segment_ids.to(device)
        start_pos = start_pos.to(device)
        end_pos = end_pos.to(device)
        span_pos = span_pos.to(device)
        span_label_mask = span_label_mask.to(device)

        with torch.no_grad():
            model_object.eval()
            tmp_eval_loss = model_object(input_ids, segment_ids, input_mask, start_pos, end_pos, span_pos, span_label_mask)
            start_labels, end_labels, span_scores = model_object(input_ids, segment_ids, input_mask)

        start_pos = start_pos.to("cpu").numpy().tolist()
        end_pos = end_pos.to("cpu").numpy().tolist()
        span_pos = span_pos.to("cpu").numpy().tolist()

        start_label = start_labels.detach().cpu().numpy().tolist()
        end_label = end_labels.detach().cpu().numpy().tolist()
        span_scores = span_scores.detach().cpu().numpy().tolist()
        span_label = span_scores
        input_mask = input_mask.to("cpu").detach().numpy().tolist()

        ner_cate_lst += ner_cate.numpy().tolist()
        eval_loss += tmp_eval_loss.mean().item()
        mask_lst += input_mask
        eval_steps += 1

        start_pred_lst += start_label
        end_pred_lst += end_label
        span_pred_lst += span_label

        start_gold_lst += start_pos
        end_gold_lst += end_pos
        span_gold_lst += span_pos

    if config.entity_sign == "flat":
        eval_accuracy, eval_precision, eval_recall, eval_f1 = flat_ner_performance(start_pred_lst, end_pred_lst, span_pred_lst, start_gold_lst, end_gold_lst, span_gold_lst, ner_cate_lst, label_list, threshold=config.entity_threshold, dims=2)
    else:
        eval_accuracy, eval_precision, eval_recall, eval_f1 = nested_ner_performance(start_pred_lst, end_pred_lst, span_pred_lst, start_gold_lst, end_gold_lst, span_gold_lst, ner_cate_lst, label_list, threshold=config.entity_threshold, dims=2)

    average_loss = round(eval_loss / eval_steps, 4)
    eval_f1 = round(eval_f1 , 4)
    eval_precision = round(eval_precision , 4)
    eval_recall = round(eval_recall , 4)
    eval_accuracy = round(eval_accuracy , 4)
    model_object.train()

    return model_object, average_loss, eval_accuracy, eval_precision, eval_recall, eval_f1

def merge_config(args_config):
    model_config_path = args_config.config_path
    model_config = Config.from_json_file(model_config_path)
    model_config.update_args(args_config)
    model_config.print_config()
    return model_config

def main():
    args_config = args_parser()
    config = merge_config(args_config)
    train_loader, dev_loader, test_loader, num_train_steps, label_list = load_data(config)
    model, optimizer, sheduler, device, n_gpu = load_model(config, num_train_steps, label_list)
    train(model, optimizer, sheduler, train_loader, dev_loader, test_loader, config, device, n_gpu, label_list)

if __name__ == "__main__":
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
@AUTHOR:Joselyn Zhao
@CONTACT:zhaojing17@foxmail.com
@HOME_PAGE:joselynzhao.top
@SOFTWERE:PyCharm
@FILE:main.py
@TIME:2019/6/13 10:32
@DES:
'''

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
old_v = tf.logging.get_verbosity()
tf.logging.set_verbosity(tf.logging.ERROR)
from tensorflow.examples.tutorials.mnist import input_data
from Lenet import  *
from PIL import Image

mnist = input_data.read_data_sets('../../../data/mnist', one_hot=True)
x_test = np.reshape(mnist.test.images, [-1, 28, 28, 1])
x_test = np.pad(x_test, ((0, 0), (2, 2), (2, 2), (0, 0)),
                'constant')  # print("Updated Image Shape: {}".format(X_train[0].shape))
tf.logging.set_verbosity(old_v)

iteratons = 1000
batch_size = 64
ma = 0
sigma = 0.1
lr = 0.01

def get_sample100(label):
    sample100_x=[]
    sample100_y=[]
    count = 0
    for i in range(len(mnist.test.images)):
        if mnist.test.labels[i][label]==1:
            count+=1
            sample100_y.append(mnist.test.labels[i])
            sample100_x.append(mnist.test.images[i])
        if count>=100:
            break
    return sample100_x,sample100_y

def train_lenet(lenet):
    with tf.Session() as sess:  #这个session需要关闭么？
        sess.run(tf.global_variables_initializer())

        tf.summary.image("input",lenet.x,3)
        merged_summary = tf.summary.merge_all()

        writer = tf.summary.FileWriter("LOGDIR/4/",sess.graph)  # 保存到不同的路径下
        # writer.add_graph(sess.graph)
        for ii in range(iteratons):
            batch_xs,batch_ys = mnist.train.next_batch(batch_size)
            batch_xs = np.reshape(batch_xs,[-1,28,28,1])
            batch_xs = np.pad(batch_xs,((0, 0), (2, 2), (2, 2), (0, 0)), 'constant')
            sess.run(lenet.train_step,feed_dict ={lenet.x:batch_xs,lenet.y_:batch_ys})
            if ii % 50 == 1:
                acc,s = sess.run([lenet.accuracy,merged_summary],feed_dict ={lenet.x:x_test,lenet.y_:mnist.test.labels})
                writer.add_summary(s,ii)
                print("%5d: accuracy is: %4f" % (ii, acc))
        sample100_x,sample100_y = get_sample100(4) #随便选了一个label 输入0-9的值
        sample100_x = np.reshape(sample100_x,[-1,28,28,1])
        sample100_x = np.pad(sample100_x, ((0, 0), (2, 2), (2, 2), (0, 0)), 'constant')
        x_min = tf.reduce_min(lenet.fc2)
        x_max = tf.reduce_max(lenet.fc2)
        fc2 = (lenet.fc2 - x_min) / (x_max - x_min)
        fc2 = sess.run(fc2,feed_dict={lenet.x:sample100_x,lenet.y_:sample100_y})
        plt.imshow(fc2)
        plt.show()

        print('[accuracy,loss]:', sess.run([lenet.accuracy], feed_dict={lenet.x:x_test,lenet.y_:mnist.test.labels}))

if __name__ =="__main__":
    act = "sigmoid"
    lenet = Lenet(ma,sigma,lr,act)
    train_lenet(lenet)


import matplotlib.pyplot as plt
# from mpl_toolkits.axes_grid1 import ImageGrid
# import numpy as np
from os import listdir
from os import chdir
from os import path
from PIL import Image
# import matplotlib.gridspec as gridspec
import argparse

parser = argparse.ArgumentParser(description="generate plot for report")
parser.add_argument("--input_dir", required=True, help="Input ROS bag.")
parser.add_argument("--rows", required=True, help="numer of rows in figure")
parser.add_argument("--cols", required=True, help="number of columns in figure")

args = parser.parse_args()

# chdir('/Volumes/macOS Big Sur/Users/pmvanderburg/matplotlib_test/')
chdir(args.input_dir)
files = listdir(args.input_dir)
files.sort()

for i, f in enumerate(files):
    if f!='.DS_Store':
        print(i,f)
    else:
        del files[i]

images = [Image.open(f) for f in files]
print(len(images))
max_rows = 7
max_cols = 3
# max_rows = 3
# max_cols = 2

methods=['Input image',
         '640x480 N+FT',
         '832x256 K+FT',
         '640x480 N',
         '832x256 N',
         '640x480 K',
         '832x256 K']

fig, axes = plt.subplots(nrows=7, ncols=3, figsize=(9,10),sharex=True, sharey=True)
for idx, image in enumerate(images):
    # print(files[idx])
    print(idx)
    row = idx % max_rows
    col = idx // max_rows
    print(row,' row')
    print(col,' col')
    # if col>0:
    # axes[row, col].axis("off")
    axes[row,col].spines['bottom'].set_color('#ffffff')
    axes[row,col].spines['top'].set_color('#ffffff')
    axes[row,col].spines['right'].set_color('#ffffff')
    axes[row,col].spines['left'].set_color('#ffffff')

    if image.size==(1280, 720):
        image = image.resize((640,480))

    axes[row, col].imshow(image, cmap="gray", aspect="auto")

    axes[row, 0].set_ylabel(methods[row])

plt.subplots_adjust(wspace=.05, hspace=.05)
plt.xticks([])
plt.yticks([])
# fig.savefig(path.join)
plt.show()


import re
import io

keywords=["int","void","main","print"]

operators = { '=': 'Assignment Operator','+': 'Additon Operator', '-' : 'Substraction Operator', '/' : 'Division Operator', '*': 'Multiplication Operator'}
optr_keys = operators.keys()

symbols = {';':'semi_colon','{' : 'left_brace', '}':'right_brace', '(':'left_parenthesis',')':'right_parenthesis' ,'[':'left_sqbracket',']':'right_sqbracket'}
symbol_keys = symbols.keys()

the_ch = " "
the_line = 1
token_list = []
error = []

#get the next character from the input file
def next_ch():

    global the_ch, the_line
    the_ch = input_file.read(1)
    if the_ch == '\n':
        the_line += 1
    return the_ch

#handle identifiers and numbers
def identifier_or_number(line_no):

    text = ""
    while the_ch.isalnum() or the_ch == '_' or the_ch =='.':
        text += the_ch
        next_ch()

    if len(text) == 0:
        error_msg = "Unrecognized character "+the_ch+" found in line : "+str(line_no)
        error.append(error_msg)
        next_ch()
        return '' , '' , ''

    elif text in keywords:
        token_list.append(text)
        return line_no, text, "Keyword"

    elif text in re.findall('[_a-zA-Z][_a-zA-Z0-9]*',text):
        token_list.append('id')
        return line_no , text , 'Identifier'

    elif text in re.findall('[0-9]+[.]?[0-9]*',text):
        token_list.append('num')
        return line_no , text , 'Number'

    elif text not in re.findall('[_a-zA-Z ][_a-zA-Z0-9 ]*',text):
        error_msg=text+" is an invalid identifier found in line : "+str(line_no)
        error.append(error_msg)
        return '','',''

#return the next token type
def getToken():

    while the_ch.isspace():
        next_ch()

    line_no = the_line

    if len(the_ch) == 0:
        token_list.append('$')
        return line_no, '$' , 'Enf_of_input'

    elif the_ch in symbol_keys:
        token = the_ch
        token_list.append(token)
        sym = symbols[token]
        next_ch()
        return line_no, token , sym

    elif the_ch in optr_keys:
        token = the_ch
        token_list.append(token)
        opr = operators[token]
        next_ch()
        return line_no, token , opr

    else:
        return identifier_or_number(line_no)

#opening input file
f = open("input.txt", "r")
i = f.read()

program = re.sub('//.*?\n|/\*.*?\*/', '', i, flags=re.S) #removes all comment lines from input file
input_file = io.StringIO(program) #converting string to file object

print("\nOutput of Lexical Analyser\n--------------------------\n")
print("%5s %7s %9s" % ("Line No.", "Token","Meaning"))
print("----------------------------")

while True:
    t = getToken()
    line  = t[0]
    token = t[1]
    meaning  = t[2]

    if token != '' and token != '$':
        print("%5s  %9s  %-15s" % (line, token, meaning))
    elif token == '$':
        break

#display error msg if found any
if len(error) != 0:
    print("\n\n-------------",len(error),"ERROR FOUND ---------------\n")
    for msg in error:
        print(msg)
    print("\n-------------------------------------------")
    exit(0)

print("\nThere are total",line,"lines in program")
print("Tokens:",token_list)


import datetime
import os

# From https://www.quora.com/Whats-the-best-spaced-repetition-schedule.  Not really scientific, but it seems decent.
schedule = [1, 2, 3, 5, 8, 13, 21, 34, 55, 89]

if __name__ == '__main__':
    today = datetime.date.today()
    schedule_days = [today - datetime.timedelta(days=i) for i in schedule]

    schedule_days = [i.strftime("%m/%d") for i in schedule_days]
    output_string = ", ".join(schedule_days)

    # God, applescript is literally the worst :'(.  Why does this exist?  Who thought this was a good idea?
    # Also, you'll have to change this for a different OS, but this should work on pretty much all OSX versions
    os.system("osascript -e 'display notification \"%s\" with title \"Spaced Repetition Reminder\"\'" % output_string)


#compare parameter between abc-smc
import seaborn as sns
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sys
from scipy import stats
from matplotlib.colors import LogNorm, Normalize
from scipy.signal import argrelextrema

filename=["ACDC_X2","ACDC_Y2","ACDC_Z2"]#,"ACDC_all"]
#filename=['ACDC_X2']
filename=['ACDC_X2','ACDC_X21ind']
n=['final']
#n=['1','2','3','4','5','6','7','8','9','10','11','12','final']#'13','14','15','final']
#n=['1','2','3','4','5','6','7','8','9','10','11','12','13','final']#,'12','13','14','final']#,'15']#,'final']

path='C:/Users/Administrator/Desktop/Modeling/AC-DC/'
path='/users/ibarbier/AC-DC/'
sys.path.insert(0, path + filename[0])

import model_equation as meq

parlist=meq.parlist

namelist=[]
for i,par in enumerate(parlist):
    namelist.append(parlist[i]['name'])

par0 = {
    'K_ARAX':-3.5,#0.01,
    'n_ARAX':2,
    'K_XY':-2.5,
    'n_XY':2,
    'K_XZ':-1.55,#-1.25
    'n_XZ':2,
    'beta_X':1,
    'alpha_X':0,
    'delta_X':1,

    'K_ARAY':-3.5,
    'n_ARAY':2,
    'K_YZ':-3.5,
    'n_YZ':2,
    'beta_Y':1,
    'alpha_Y':0,
    'delta_Y':1,

    'K_ZX':-2.5,
    'n_ZX':2,
    'beta_Z':1,
    'alpha_Z':0,
    'delta_Z':1,

    'beta/alpha_X':2,
    'beta/alpha_Y':2,
    'beta/alpha_Z':2

}

def pars_to_dict(pars,parlist):
### This function is not necessary, but it makes the code a bit easier to read,
### it transforms an array of pars e.g. p[0],p[1],p[2] into a
### named dictionary e.g. p['k0'],p['B'],p['n'],p['x0']
### so it is easier to follow the parameters in the code
    dict_pars = {}
    for ipar,par in enumerate(parlist):
        dict_pars[par['name']] = pars[ipar]
    return dict_pars

def load(number= n,filename=filename,parlist=parlist):
    namelist=[]
    for i,par in enumerate(parlist):
      namelist.append(parlist[i]['name'])
    number=str(number)
    filepath = path+filename+'/smc/pars_' + number + '.out'
    dist_path =  path+filename+'/smc/distances_' + number + '.out'
    raw_output= np.loadtxt(filepath)
    dist_output= np.loadtxt(dist_path)
    df = pd.DataFrame(raw_output, columns = namelist)
    df['dist']=dist_output
    df=df.sort_values('dist',ascending=False)
    distlist= sorted(df['dist'])
    p=[]
    for dist in distlist:
        p_0=df[df['dist']==dist]
        p0=[]
        for n in namelist:
          p0.append(p_0[n].tolist()[0])
        p0=pars_to_dict(p0,parlist)
        p.append(p0)
    return p, df

def get_stats(filename,namelist):
    stats_df = pd.DataFrame( columns = ['par','file','mean','sd','mode'])
    parl = np.append(namelist,'dist')
   # for fi,fnm in enumerate(filename):
    fnm=filename[0]
    p,df= load(n[0],fnm,parlist)
    mean=np.mean(df).tolist()
    sd=np.std(df).tolist()
    mode=stats.mode(df)[0][0]
    new_row={'par':parl,'file':[fnm]*len(parl),'mean':mean,'sd':sd,'mode':mode}
    df2=pd.DataFrame(new_row)
    stats_df =stats_df.append(df2)
    return stats_df

def bar_plot(filename,namelist, t="mean"):
    stats_df=get_stats(filename,namelist)
    # set width of bars
    barWidth = 0.20
    # Set position of bar on X axis
    r1 = np.arange(len(parl))

    #mean
    if t=="mean":
        for i,nm in enumerate(filename):
            v=stats_df[stats_df['method']==nm]
            plt.bar((r1+barWidth*i),v['mean'],yerr=v['sd'], capsize=2,width=barWidth, label=nm)

        plt.xlabel('par', fontweight='bold')
        plt.xticks([r + barWidth for r in range(len(parl))], parl)
        plt.legend()
        plt.show()

    #mode
    if t == "mode":
        for i,nm in enumerate(filename):
            v=stats_df[stats_df['method']==nm]
            plt.bar((r1+barWidth*i),v['mode'],width=barWidth, label=nm)

        plt.xlabel('par', fontweight='bold')
        plt.xticks([r + barWidth for r in range(len(parl))], parl)
        plt.legend()
        plt.show()

def plot_compare(n,filename,namelist):

    parl = np.append(namelist,'dist')
    index=1
    size=round(np.sqrt(len(parl)))
    for i,name in enumerate(parl):
        plt.subplot(size,size,index)
        plt.tight_layout()
        for fi,fnm in enumerate(filename):
              p,df= load(n,fnm,namelist1)
              sns.kdeplot(df[name],bw_adjust=.8,label=fnm)
        #plt.ylim(0,1)
        if i < (len(parl)-2):
            plt.xlim((parlist[i]['lower_limit'],parlist[i]['upper_limit']))
        index=index+1
        if index==5:
            plt.legend(bbox_to_anchor=(1.05, 1))

    #sns.kdeplot(df['K_XZ'])
    plt.savefig(str(filename)+str(n)+"_compareplot.pdf", bbox_inches='tight')
    plt.show()

#plot_compare(n[0],filename,namelist)

def plot_alltime(filename,namelist):
    parl = np.append(namelist,'dist')
    index=1
    for i,name in enumerate(parl):
        plt.subplot(4,4,index)
        plt.tight_layout()
        for ni,nmbr in enumerate(n):
            p,df= load(nmbr,filename[0],parlist)
            sns.kdeplot(df[name],bw_adjust=.8,label=nmbr)
        #plt.ylim(0,1)
        if i < (len(parl)-2):
            plt.xlim((parlist[i]['lower_limit'],parlist[i]['upper_limit']))
        index=index+1
        #if index==5:
    plt.legend(bbox_to_anchor=(1.05, 1))
    plt.show()

#plot_alltime(['ACDC_X2'],namelist)

def plotdistpar(filename,namelist):
    index=1
    for ni,nb in enumerate(n):
        p,df= load(nb,filename[0],parlist)
        for i,name in enumerate(namelist):
            plt.subplot(len(n),len(namelist),index)
           # plt.tight_layout()
            plt.scatter(df['dist'],df[name],s=1)
            mean=np.mean(df[name]).tolist()
            mode=stats.mode(df[name])[0][0]

            plt.plot([0,40],[mean,mean],'r',label="mean")
            plt.plot([0,40],[mode,mode],'g',label="meode")

            plt.ylim((parlist[i]['lower_limit'],parlist[i]['upper_limit']))
            plt.ylabel(name)
            index=index+1
    plt.legend(bbox_to_anchor=(1.05, 1))
    plt.show()

'''
ARA=np.logspace(-4.5,-2.,10,base=10)
p,df= load(n[0],filename[0],parlist)

stdf=get_stats(filename,namelist)
pmean=pars_to_dict(stdf['mean'])
pmode=pars_to_dict(stdf['mode'])

for i,p in enumerate([p[0],pmean,pmode,p[999]]):

    X,Y,Z=meq.model(ARA,p)
    df_X=pd.DataFrame(X,columns=ARA)
    df_Y=pd.DataFrame(Y,columns=ARA)
    df_Z=pd.DataFrame(Z,columns=ARA)

    plt.subplot(4,3,(1+3*i))
    sns.heatmap(df_X, cmap="Reds")
    plt.subplot(4,3,(2+3*i))
    sns.heatmap(df_Y, cmap ='Blues')
    plt.subplot(4,3,(3+3*i))
    sns.heatmap(df_Z, cmap ='Greens')

plt.show()

X,Y,Z=meq.model(ARA,pmode)
plt.plot(X[:,0],label="DCoff")
plt.plot(X[:,3],label="AC1")
plt.plot(X[:,6],label="AC2")
plt.plot(X[:,9],label="DCon")
plt.plot([200,200],[0,1000],'--')

plt.legend(bbox_to_anchor=(1.05, 1))
plt.tight_layout()
plt.show()
'''

#####1indvs2ind
def plotdesnity1vs2():
    p2,df2= load('final','ACDC_X2',parlist)
    parlist1=parlist.copy()
    del parlist1[7:9]
    p1,df1= load('final','ACDC_X21ind',parlist1)

    namelist=[]
    for i,par in enumerate(parlist1):
        namelist.append(par['name'])

    parl = np.append(namelist,'dist')
    index=1
    for i,name in enumerate(parl):
        plt.subplot(4,4,index)
        plt.tight_layout()

        sns.kdeplot(df1[name],bw_adjust=.8,label='X_1ind')
        sns.kdeplot(df2[name],bw_adjust=.8,label='X_2ind')
            #plt.ylim(0,1)
        if i < (len(parl)-2):
            plt.xlim((parlist1[i]['lower_limit'],parlist1[i]['upper_limit']))
        index=index+1
        if index==5:
            plt.legend(bbox_to_anchor=(1.05, 1))

        #sns.kdeplot(df['K_XZ'])
    plt.savefig("1vs2ind"+str(n[0])+"_compareplot.pdf", bbox_inches='tight')
    #plt.show()

plotdesnity1vs2()

def ind1vs2indmeanandmode():
    p2,df2= load('final','ACDC_X',parlist)
    df2=df2.drop(columns=['K_ARAY', 'n_ARAY'])
    mean_df2=np.mean(df2)
    sd_df2=np.std(df2)
    mode_df2=stats.mode(df2)[0][0]
    parlist1=parlist.copy()
    del parlist1[7:9]
    p1,df1= load('12','ACDC_1ind',parlist1)
    mean_df1=np.mean(df1)
    sd_df1=np.std(df1)
    mode_df1=stats.mode(df1)[0][0]

    namelist=[]
    for i,par in enumerate(parlist1):
        namelist.append(par['name'])
    parl = np.append(namelist,'dist')
    # set width of bars
    barWidth = 0.30
    # Set position of bar on X axis
    r1 = np.arange(len(parl))
    plt.bar((r1+barWidth*0),mean_df1,yerr=sd_df1, capsize=2,width=barWidth, label="1ind")
    plt.bar((r1+barWidth*1),mean_df2,yerr=sd_df2, capsize=2,width=barWidth, label="2ind")
    plt.xlabel('par', fontweight='bold')
    plt.xticks([r + barWidth for r in range(len(parl))], parl)
    plt.legend()
    plt.show()
    plt.bar((r1+barWidth*0),mode_df1,width=barWidth, label="1ind")
    plt.bar((r1+barWidth*1),mode_df2,width=barWidth, label="2ind")
    plt.xlabel('par', fontweight='bold')
    plt.xticks([r + barWidth for r in range(len(parl))], parl)
    plt.legend()
    plt.show()

def calculateSS(ARA,parUsed):
    #sort ss according to their stabilitz
    #create stability list of shape : arabinose x steady x X,Y,Z
    unstable=np.zeros((len(ARA),3,3))
    stable=np.zeros((len(ARA),3,3))
    oscillation=np.zeros((len(ARA),3,3))
    unstable[:]=np.nan
    stable[:]=np.nan
    oscillation[:]=np.nan

    for ai,a in enumerate(ARA):
        ss=meq.findss(a,parUsed)
        if len(ss) > 3:
            print("error: more than 3 steadystates")
        else:
            d = b = c=0 # can replace a,b,c by si, but allow to have osccilation on the same level
            for si,s in enumerate(ss):
                e=meq.stability(a,parUsed,[s])[0][0]
                if all(e<0):
                        stable[ai][d]=s
                        d+=1
                if any(e>0):
                    pos=e[e>0]
                    if len(pos)==2:
                        if pos[0]-pos[1] == 0:
                            oscillation[ai][b]=s
                            b+=1
                        else:
                            unstable[ai][c]=s
                            c+=1
                    else:
                        unstable[ai][c]=s
                        c+=1
    return unstable,stable,oscillation

#chose parameter
def bifurcation(parUsed=None):
    p,df= load('final','ACDC_X2',parlist)
    #parUsed=par0
    if parUsed == None:
        parUsed=p[0]
    ARA=np.logspace(-4.5,-2.,20,base=10)
    ss=meq.findss(ARA[0],parUsed)[0]
    #print(ss)
    init=[ss[0],ss[1],ss[2]]
    X,Y,Z=meq.model(ARA,parUsed,totaltime=100,init=init)
    df_X=pd.DataFrame(X[500:],columns=ARA)
    sns.heatmap(df_X, cmap="Reds", norm=LogNorm())
    plt.show()

    xss,yss,zss = calculateSScurve(ARA,parUsed)

    maxX=[]
    minX=[]
    maxY=[]
    minY=[]
    maxZ=[]
    minZ=[]
   # X,Y,Z=meq.model(ARA,parUsed,totaltime=400)
    delta=10e-5
    for i in np.arange(0,len(ARA)):
        min_x=[np.nan,np.nan,np.nan]
        max_x=[np.nan,np.nan,np.nan]
        ss=meq.findss(ARA[i],parUsed)
        for si,s in enumerate(ss):
            init=[s[0]+delta,s[1]+delta,s[2]+delta]
            X,Y,Z=meq.model(ARA,parUsed,totaltime=100,init=init)
           # print(max(X[200:,i]))

            max_x[si]=max(X[200:,i])
            min_x[si]=min(X[200:,i])

        maxX.append(max_x)
        minX.append(min_x)

       # minX.append(min(X[200:,i]))
        maxY.append(max(Y[200:,i]))
        minY.append(min(Y[200:,i]))
        maxZ.append(max(Z[200:,i]))
        minZ.append(min(Z[200:,i]))
    plt.subplot(3,1,1)
    plt.plot(ARA,xss,'--o')
    plt.plot(ARA,maxX,'-b')
    plt.plot(ARA,minX,'-g')
    #plt.fill_between(ARA,maxX,minX,alpha=0.2,facecolor='red')
    plt.yscale("log")
    plt.xscale("log")
    plt.subplot(3,1,2)
    plt.plot(ARA,yss,'--b')
   # plt.plot(ARA,maxY,'-b')
   # plt.plot(ARA,minY,'-b')
   # plt.fill_between(ARA,maxY,minY,alpha=0.2,facecolor='blue')
    plt.yscale("log")
    plt.xscale("log")
    plt.subplot(3,1,3)
    plt.plot(ARA,zss,'--g')
  #  plt.plot(ARA,maxZ,'-g')
  #  plt.plot(ARA,minZ,'-g')
  #  plt.fill_between(ARA,maxZ,minZ,alpha=0.2,facecolor='green')
    plt.yscale("log")
    plt.xscale("log")
    plt.show()

def getlimitcycle(ARA,ssl,par,tt=500):
    M=np.ones((len(ARA),3,3))*np.nan
    m=np.ones((len(ARA),3,3))*np.nan
    delta=10e-5
    transient=500
    for ai,a in enumerate(ARA):
            ss=ssl[ai]
            for si,s in enumerate(ss):
                if any(np.isnan(s)) == False:
                    init=[s[0]+delta,s[1]+delta,s[2]+delta]
                    X,Y,Z=meq.model([a],par,totaltime=tt,init=init)
                    M[ai,si,0]=max(X[transient:])
                    M[ai,si,1]=max(Y[transient:])
                    M[ai,si,2]=max(Z[transient:])
                    m[ai,si,0]=min(X[transient:])
                    m[ai,si,1]=min(Y[transient:])
                    m[ai,si,2]=min(Z[transient:])

                    max_list=argrelextrema(X[transient:], np.greater)
                    maxValues=X[transient:][max_list]
                    min_list=argrelextrema(X[transient:], np.less)
                    minValues=X[transient:][min_list]

                    maximaStability = abs(maxValues[-2]-minValues[-2])-(maxValues[-3]-minValues[-3])
                    if maximaStability > 0.01:
                        print("limit cycle not achieved for ARA["+str(ai)+"]:" + str(a) + " at st.s:"+ str(s))

    return M,m

def bifurcation_plot(n,filename):
    p,df= load(n,filename,parlist)
    ARA=np.logspace(-4.5,-2.,200,base=10)
    un,st,osc=calculateSS(ARA,p[1])
    M,m=getlimitcycle(ARA,osc,p[1],tt=500)
    for i,col in enumerate(['r','b','g']):
        plt.subplot(3,1,i+1)
        plt.plot(ARA,un[:,:,i],'--'+col)
        plt.plot(ARA,st[:,:,i],'-'+col)
        plt.plot(ARA,osc[:,:,i],'--'+col)
        plt.fill_between(ARA,M[:,0,i],m[:,0,i],alpha=0.2,facecolor=col)
        plt.fill_between(ARA,M[:,1,i],m[:,1,i],alpha=0.2,facecolor=col)
        plt.fill_between(ARA,M[:,2,i],m[:,2,i],alpha=0.2,facecolor=col)
        plt.yscale("log")
        plt.xscale("log")
    plt.show()

#bifurcation(p[1])


# coding: utf-8

# In[1]:

from pandas import DataFrame, read_csv
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import Normalizer
from sklearn import metrics

from sklearn.cluster import KMeans, MiniBatchKMeans

import sys
from time import time

import numpy as np

# In[2]:

df = pd.read_csv('lyrics.csv')
df.head(10)

# In[3]:

df['lyrics'].replace('', np.nan, inplace=True)
df.dropna(subset=['lyrics'], inplace=True)
ind_drop = df[df['genre'].apply(lambda x: x.startswith('Other'))].index
df = df.drop(ind_drop)

# In[4]:

ind_drop = df[df['genre'].apply(lambda x: x.startswith('Not Available'))].index
df = df.drop(ind_drop)

# In[5]:

ind_drop = df[df['lyrics'].apply(lambda x: x.startswith('INSTRUMENTAL'))].index
df = df.drop(ind_drop)
df.drop(columns=['index'])

ind_drop = df[df['lyrics'].apply(lambda x: x.startswith('instrumental'))].index
df = df.drop(ind_drop)
df.drop(columns=['index'])

# In[6]:

genre=df['genre'].values
lyrics=df['lyrics'].values
true_k = len(np.unique(genre))
print(np.unique(genre), "The total number of genres are", true_k)

#shaping:
lyrics = np.array(lyrics)[:,None]
print(lyrics.shape)
genre = np.array(genre)[:,None]
print(genre.shape)

# In[7]:

data = np.append(lyrics,genre,axis=1)
data.shape
print(data)

# In[8]:

np.random.shuffle(data)

data_test = data[10001:20001,]
data = data[:10000,]

# In[9]:

data_lyrics=data[:,0]
data_genre=data[:,1]

data_lyrics_test = data_test[:,0]
data_genre_test = data_test[:,1]

# print(data_lyrics)
# print(data_genre.shape)

# In[10]:

vectorizer = TfidfVectorizer(
                max_df=0.75, # max doc freq (as a fraction) of any word to include in the vocabulary
                min_df=0.3,   # min doc freq (as doc counts) of any word to include in the vocabulary
                max_features=10000,           # max number of words in the vocabulary
                stop_words='english',         # remove English stopwords
                use_idf=True )

# In[11]:

labels={'Country':1, 'Electronic':2, 'Folk':3, 'Hip-Hop':4, 'Indie':5, 'Jazz':6,
       'Metal':7, 'Pop':8, 'R&B':9, 'Rock':10}

print("Extracting features from the training dataset using a sparse vectorizer")
t0 = time()
vectorizer.fit(data_lyrics)

X = vectorizer.transform(data_lyrics)
Y = [labels[i] for i in data_genre]

X_test = vectorizer.transform(data_lyrics_test)
Y_test = [labels[i] for i in data_genre_test]

n_features = X.shape[1]
print("done in %fs" % (time() - t0))
print("n_samples: %d, n_features: %d" % X.shape)

# In[12]:

doc_ind = 1  # Index of an example document
xi = X[doc_ind,:].todense()
term_ind = xi.argsort()[:, ::-1]
xi_sort = xi[0,term_ind]
terms = vectorizer.get_feature_names()

for i in range(n_features):
    term = terms[term_ind[0,i]]
    tfidf = xi[0,term_ind[0,i]]
    print('{0:20s} {1:f} '.format(term, tfidf))

# In[13]:

km = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1,
                verbose=True)

# In[14]:

print("Clustering sparse data with %s" % km)
t0 = time()
km.fit(X)
print("done in %0.3fs" % (time() - t0))
print()

# In[15]:

order_centroids = km.cluster_centers_.argsort()[:, ::-1]
for i in range(true_k):
    print("Cluster %d:" % i, end='')
    for ind in order_centroids[i, :10]:
        print(' %s' % terms[ind], end='')
    print()

# In[16]:

labels={'Country':1, 'Electronic':2, 'Folk':3, 'Hip-Hop':4, 'Indie':5, 'Jazz':6,
       'Metal':7, 'Pop':8, 'R&B':9, 'Rock':10}
print(labels.values)
# genre_names
# data_genre
genre_labels=[]
#print(genre_labels.shape)
for j,i in enumerate(data_genre):
    x=labels[i]
    #print(x)
    np.append(genre_labels,x)
    genre_labels.append(x)
#print(genre_labels)

# In[17]:

print((Y_test == km.predict(X_test)).sum() / len(Y_test))

# In[18]:

labelkm = km.labels_
print(labelkm.shape)
print(type(labelkm))

# In[19]:

#print(data_genre)
labelkm = km.labels_
from sklearn.metrics import confusion_matrix
C = confusion_matrix(genre_labels,labelkm)

Csum = np.sum(C,axis=0)
Cnorm = C / Csum[None,:]
print(Cnorm)
print(np.array_str(C, precision=3, suppress_small=True))
plt.imshow(C, interpolation='none')
plt.colorbar()


import torch
import pandas as pd
import os
from shutil import copy
from utils import fix_randomness, save_to_df, _logger, report_results, get_nonexistant_path,  copy_Files
from dataloader.dataloader import data_generator
from trainer.training_evaluation import cross_domain_test
from datetime import datetime
from itertools import product
from args import args
import wandb
start_time = datetime.now()
device = torch.device(args.device)
da_method = args.da_method
save_dir = args.save_dir
data_type = args.selected_dataset
data_path = f"./data/{data_type}"
base_model_type = args.base_model
experiment_description = args.experiment_description
if not os.path.exists(save_dir):
    os.mkdir(save_dir)

exec(f'from trainer.{da_method} import cross_domain_train')
exec(f'from config_files.{data_type}_Configs import Config as Configs')
exec(f'from models.models import {base_model_type} as base_model')
configs = Configs()

# os.environ["WANDB_MODE"] = "dryrun"
os.environ["WANDB_SILENT"] = 'true'
os.environ['CUDA_LAUNCH_BLOCKING'] = "1"
# torch.backends.cudnn.enabled = False # another solution for lstm lunch faiulure issue

def main_train_cd():

    # find out the domains IDs
    data_files = os.listdir(data_path)
    data_files = [i for i in data_files if "train" in i]
    sources = [i[6] for i in data_files]
    src_tgt_product = [sources, sources]

    simple_column_names = ['Run ID',
                           'source_loss', 'source_acc',
                            'target_loss', 'target_acc',]
    column_names_mean = ['Scenario',
                         'Source_only_loss_mean', 'Source_only_acc_mean',
                         f'{da_method}_loss_mean', f'{da_method}_acc_mean',
                         f'Source_only_loss_std', 'Source_only_acc_std',
                         f'{da_method}_loss_std', f'{da_method}_acc_std']

    simple_df= pd.DataFrame(columns=simple_column_names)
    mean_df = pd.DataFrame(columns=column_names_mean)
    # Logging
    # cwd = os.getcwd()
    # exp_log_dir = os.path.join(r"D:\Autoregressive Domain Adaptation for Time series data\Last",save_dir, experiment_description, f"{da_method}_{data_type}_{args.run_description}")
    exp_log_dir = os.path.join(os.getcwd(),save_dir, experiment_description, f"{da_method}_{data_type}_{args.run_description}")

    exp_log_dir = get_nonexistant_path(exp_log_dir)
    # os.makedirs(exp_log_dir, exist_ok=True)
    # copy(f"/home/mohamed/SLARADA/config_files/{data_type}_configs.py", f"{exp_log_dir}/{data_type}_configs.py")
    # copy(f"/home/mohamed/SLARADA/trainer/{da_method}.py", f"{exp_log_dir}/{da_method}_script.py")
    # copy("/home/mohamed/SLARADA/args.py",  f"{exp_log_dir}/args.py")
    copy_Files(exp_log_dir, data_type, da_method)
    # loop through domains
    # loop through domains
    counter = 0
    src_counter = 0
    for src_id, tgt_id in product(*src_tgt_product):
    # for src_id in ['a', 'b', 'c']:
    #     for tgt_id in ['a', 'b','c']:
            if src_id != tgt_id:
                # prepare save directory
                # specify number of consecutive runs
                for run_id in range(args.num_runs):
                    fix_randomness(run_id)

                    # Logging
                    log_dir = os.path.join(exp_log_dir, src_id + "_to_" + tgt_id + "_run_"+ str(run_id))
                    os.makedirs(log_dir, exist_ok=True)
                    log_file_name = os.path.join(log_dir, f"logs_{datetime.now().strftime('%d_%m_%Y_%H_%M_%S')}.log")
                    logger = _logger(log_file_name)
                    logger.debug("=" * 45)
                    logger.debug(f'Dataset: {data_type}')
                    logger.debug(f'Method:  {da_method}')
                    logger.debug("=" * 45)
                    logger.debug(f'Source: {src_id} ---> Target: {tgt_id}')
                    logger.debug(f'Run ID: {run_id}')
                    logger.debug("=" * 45)

                    # Load datasets
                    src_train_dl, src_valid_dl, src_test_dl = data_generator(data_path, src_id, configs)
                    tgt_train_dl, tgt_valid_dl, tgt_test_dl = data_generator(data_path, tgt_id, configs)

                    if args.tensorboard:
                        wandb.init(project="SLARDA",   group = f'{da_method}_{data_type}', name=f'{src_id}_to_{tgt_id}_run_{run_id}', config=configs,
                                   sync_tensorboard=False, reinit=True, dir=r"./visualize/", )

                    source_model, target_model = cross_domain_train(src_train_dl, src_valid_dl, src_test_dl,
                                                                    tgt_train_dl, tgt_valid_dl, base_model,
                                                                    src_id, tgt_id,
                                                                    device, logger, configs)
                    scores = cross_domain_test(source_model, target_model, src_id, tgt_id,
                                               src_train_dl, tgt_train_dl, src_test_dl, tgt_test_dl,
                                               device, log_dir, logger)

                    run_name = f"domain_{src_id}_run_{run_id}"
                    outs = (run_name,) + scores
                    simple_df.loc[counter] = outs
                    counter += 1

                input_data = [f"{src_id}-->{tgt_id}"]
                input_data.extend(simple_df.iloc[-args.num_runs:, 1:].mean().array)
                input_data.extend(simple_df.iloc[-args.num_runs:, 1:].std().array)
                mean_df.loc[src_counter] = input_data
                src_counter += 1

    # Printing and saving final results
    print(simple_df.to_string())
    print(mean_df.to_string())
    printed_results = mean_df[['Scenario', 'Source_only_acc_mean', 'Source_only_acc_std', f'{da_method}_acc_mean', f'{da_method}_acc_std']]
    mean = mean_df[['Source_only_acc_mean', 'Source_only_acc_std', f'{da_method}_acc_mean', f'{da_method}_acc_std']].mean()
    printed_results.loc[len(printed_results)] = mean
    printed_results.at[len(printed_results)-1, 'Scenario'] = 'Average'

    logger.debug(f"Total training time is {datetime.now() - start_time}")

    logger.debug('=' * 45)
    logger.debug(f'Results using: {da_method}')
    logger.debug('=' * 45)
    logger.debug(mean_df.to_string())
    logger.debug(printed_results.to_string())
    print_res_name = os.path.basename(exp_log_dir)
    simple_df.to_excel(f'{exp_log_dir}/full_res_results_{print_res_name}.xlsx')
    printed_results.to_excel(f'{exp_log_dir}/printed_results_{print_res_name}.xlsx')

    if args.tensorboard:
        wandb.log({"Full_results": wandb.Table(dataframe=simple_df)})
        wandb.log({"Printed_results": wandb.Table(dataframe=printed_results)})

if __name__ == "__main__":
    wandb.config = configs
    main_train_cd()


#!/usr/bin/env python3
import numpy as np
import urllib.request
import cv2
import binascii
import lorem
import math
import matplotlib.pyplot as plt

def encode_as_binary_array(msg):
    """Encode a message as a binary string."""
    msg = msg.encode("utf-8")
    msg = msg.hex()
    msg = [msg[i:i + 2] for i in range(0, len(msg), 2)]
    msg = [bin(int(el, base=16))[2:] for el in msg]
    msg = ["0" * (8 - len(el)) + el for el in msg]
    return "".join(msg)

def decode_from_binary_array(array):
    """Decode a binary string to utf8."""
    array = [array[i:i+8] for i in range(0, len(array), 8)]
    if len(array[-1]) != 8:
        array[-1] = array[-1] + "0" * (8 - len(array[-1]))
    array = [hex(int(el, 2))[2:].zfill(2) for el in array]
    array = "".join(array)
    result = binascii.unhexlify(array)
    return result.decode("utf-8", errors="replace")

def hide_message(image, message, nbits=1):
    """Hide a message in an image (LSB).

    nbits: number of least significant bits
    """
    nbits = clamp(nbits, 1, 8)
    shape = image.shape
    image = np.copy(image).flatten()
    if len(message) > len(image) * nbits:
        raise ValueError("Message is to long :(")

    chunks = [message[i:i + nbits] for i in range(0, len(message), nbits)]
    for i, chunk in enumerate(chunks):
        byte = str(bin(image[i]))[2:].zfill(8)
        new_byte = byte[:-nbits] + chunk
        image[i] = int(new_byte, 2)

    return image.reshape(shape)

def clamp(n, minn, maxn):
    """Clamp the n value to be in range (minn, maxn)."""
    return max(min(maxn, n), minn)

def reveal_message(image, nbits=1, length=0):
    """Reveal the hidden message.

    nbits: number of least significant bits
    length: length of the message in bits.
    """
    nbits = clamp(nbits, 1, 8)
    shape = image.shape
    image = np.copy(image).flatten()
    length_in_pixels = math.ceil(length/nbits)
    if len(image) < length_in_pixels or length_in_pixels <= 0:
        length_in_pixels = len(image)

    message = ""
    i = 0
    while i < length_in_pixels:
        byte = str(bin(image[i]))[2:].zfill(8)
        message += byte[-nbits:]
        i += 1

    mod = length % -nbits
    if mod != 0:
        message = message[:mod]
    return message

print("Downloading image!")
path = 'https://picsum.photos/500/500'
resp = urllib.request.urlopen(path)
image = np.asarray(bytearray(resp.read()), dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
print("Image downloaded!")

message = lorem.text()*1000
secret = encode_as_binary_array(message)

resultImageRow1 = None
resultImageRow2 = None

nbitsList = range(1, 9)
nbitsMSE = []
for nbits in nbitsList:
    print(nbits)
    imageSecret = hide_message(image, secret[:int(image.size*0.8)], nbits)
    mse = ((imageSecret - image)**2).mean()
    nbitsMSE.append(mse)

    if nbits <= 4:
        resultImageRow1 = imageSecret if resultImageRow1 is None else np.hstack(
            [resultImageRow1, imageSecret])
    else:
        resultImageRow2 = imageSecret if resultImageRow2 is None else np.hstack(
            [resultImageRow2, imageSecret])

plt.plot(nbitsList, nbitsMSE)
plt.xlabel('nbits')
plt.ylabel('MSE')

cv2.namedWindow("Result", cv2.WINDOW_NORMAL)

cv2.imshow('Result', np.vstack([resultImageRow1, resultImageRow2]))
cv2.imwrite('ex2_encoded.png', np.vstack([resultImageRow1, resultImageRow2]))
cv2.waitKey(1)
plt.savefig('ex2_plot.png')
plt.show()
cv2.waitKey()

# Dla nbits=7,8 MSE zmalał, ponieważ widoczna jest większa część bazowego obrazu
# - wiadomość zapisano na mniejszej liczbie plkseli


from tornado.web import RequestHandler
from tornado.web import gen
from controller import favoriteTopicController
import json

# 收藏话题
class AddFavoriteTopic(RequestHandler):
    @gen.coroutine
    def post(self):
        session_id = self.get_argument('session_id')
        topicId = self.get_argument('topicId')
        data = favoriteTopicController.createFavoriteTopic(session_id, topicId)
        self.write(json.dumps(data))

# 取消收藏的话题
class RemoveFavoriteTopic(RequestHandler):
    @gen.coroutine
    def post(self):
        session_id = self.get_argument('session_id')
        topicId = self.get_argument('topicId')
        data = favoriteTopicController.removeFavoriteTopic(session_id, topicId)
        self.write(json.dumps(data))

# 获取用户收藏的话题
class GetFavoriteTopic(RequestHandler):
    @gen.coroutine
    def post(self):
        session_id = self.get_argument('session_id')
        x = self.get_argument('x')
        n = self.get_argument('n')
        data = favoriteTopicController.retrieveFavoriteTopic(session_id, x, n)
        self.write(json.dumps(data))