from argparse import ArgumentParser
from dataclasses import dataclass
import typing as ty
import os

import torch
from torch import Tensor
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.optim import Optimizer, lr_scheduler, Adam
from tqdm import tqdm
import numpy as np
from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score
import matplotlib.pyplot as plt

from paras.scripts.data_processing.structure_processing.make_voxel_dataset import VoxelDataset
import paras.data.structure_data

SUBSTRATE_PATH = os.path.join(os.path.dirname(paras.data.structure_data.__file__), "included_substrates.txt")

class Cnn3D(torch.nn.Module):
    pass

def parse_arguments():
    parser = ArgumentParser()
    parser.add_argument('-train', type=str, required=True, help="Domain list of training data points")
    parser.add_argument('-test', type=str, required=True, help="Domain list of test data points")
    parser.add_argument('-i', type=str, required=True, help="Directory to saved voxel features directory")
    parser.add_argument('-s', type=str, required=True, help="Directory to specificities file")
    parser.add_argument('-e', type=int, default=35, help="Number of epochs")
    parser.add_argument('-b', type=int, default=10, help="Batch size")
    parser.add_argument('-o', type=str, required=True, help="Output dir")
    parser.add_argument('-m', type=str, default=None, help="Directory to pre-trained model")
    parser.add_argument('-l', type=float, default=0.001, help="Start learning rate")
    parser.add_argument('-n', type=str, default=SUBSTRATE_PATH, help="Path to file containing included substrates")

    args = parser.parse_args()
    return args

@dataclass
class SubstrateMetrics:
    substrate: str

    tp_count: int = 0
    fp_count: int = 0
    fn_count: int = 0
    tn_count: int = 0

    def __post_init__(self):
        self.mispredictions = []

    def process_prediction(self, pred_y, true_y):
        if pred_y > 0.5 and true_y > 0.5:
            self.tp_count += 1
        elif pred_y <= 0.5 < true_y:
            self.fn_count += 1
        elif true_y <= 0.5 < pred_y:
            self.fp_count += 1
        else:
            self.tn_count += 1

    def count_mispredictions(self):
        counts = {}
        for misprediction in self.mispredictions:
            if misprediction not in counts:
                counts[misprediction] = 0
            counts[misprediction] += 1

        return counts

    def print_mispredictions(self):
        counts = self.count_mispredictions()
        print("Mispredictions")
        sorted_counts = []
        for misprediction, count in counts.items():
            sorted_counts.append((misprediction, count))
        sorted_counts.sort(key=lambda x: x[1], reverse=True)

        for misprediction, count in sorted_counts:
            print(f"{misprediction}: {count}")

    def print(self):
        print(f"Substrate: {self.substrate}")
        print(f"TP: {self.tp_count}")
        print(f"FP: {self.fp_count}")
        print(f"TN: {self.tn_count}")
        print(f"FN: {self.fn_count}\n")
        print('\n')
        self.print_mispredictions()
        print('\n')

def write_aa_metrics(out_folder, epoch, metrics, specificities):
    out_file = os.path.join(out_folder, f"epoch_{epoch}.txt")
    out_file_2 = os.path.join(out_folder, f"epoch_{epoch}_cm.txt")
    with open(out_file, 'w') as out:
        out.write("amino_acid\tTP\tFP\tTN\tFN\n")
        for aa in specificities:
            aa_metrics = metrics[aa]
            out.write(f"{aa}\t{aa_metrics.tp_count}\t{aa_metrics.fp_count}\t{aa_metrics.tn_count}\t{aa_metrics.fn_count}\n")
    with open(out_file_2, 'w') as out:
        out.write('Predicted\\Actual\t')
        out.write('\t'.join(specificities))
        out.write("\n")
        for aa in specificities:
            out.write(f"{aa}")
            aa_metrics = metrics[aa]
            misprediction_counts = aa_metrics.count_mispredictions()
            for aa_2 in specificities:
                if aa_2 not in misprediction_counts:
                    if aa_2 == aa:
                        out.write(f"\t{aa_metrics.tp_count}")
                    else:
                        out.write(f"\t{0}")
                else:
                    out.write(f"\t{misprediction_counts[aa_2]}")
            out.write("\n")

def plot_aa_metrics(out_folder, metrics, specificities):
    for specificity in specificities:
        out_file = os.path.join(out_folder, f"{specificity}.svg")
        x = []
        tp = []
        fp = []
        tn = []
        fn = []

        for epoch, aa_to_metrics in metrics.items():
            x.append(epoch)

            tp.append(aa_to_metrics[specificity].tp_count)
            fp.append(aa_to_metrics[specificity].fp_count)
            tn.append(aa_to_metrics[specificity].tn_count)
            fn.append(aa_to_metrics[specificity].fn_count)

        plt.plot(x, tp, label="TP")
        plt.plot(x, fp, label="FP")
        plt.plot(x, tn, label="TN")
        plt.plot(x, fn, label="FN")
        plt.legend()
        plt.savefig(out_file)
        plt.clf()

def write_metrics(out_file, epochs, train_losses, test_losses, accuracies, learning_rates, precisions, recalls, class_precisions, class_recalls, train_accuracies):
    with open(out_file, 'w') as out:
        out.write("epoch\ttrain_loss\ttest_loss\taccuracy\tlearning_rate\tprecision\trecall\tclass_precision\tclass_recall\ttrain_accuracy\n")
        for i, epoch in enumerate(epochs):
            out.write(f"{epoch}\t{train_losses[i]}\t{test_losses[i]}\t{accuracies[i]}\t{learning_rates[i]}\t{precisions[i]}\t{recalls[i]}\t{class_precisions[i]}\t{class_recalls[i]}\t{train_accuracies[i]}\n")

def plot_metrics(out_plots, epochs, train_losses, test_losses, accuracies, precisions, recalls, class_precisions, class_recalls, train_accuracies):
    precision_out = os.path.join(out_plots, "precision.png")
    recall_out = os.path.join(out_plots, "recall.png")
    class_precision_out = os.path.join(out_plots, "class_precision.png")
    class_recall_out = os.path.join(out_plots, "class_recall.png")
    accuracy_out = os.path.join(out_plots, "accuracy.png")
    loss_out = os.path.join(out_plots, "loss.png")

    plt.plot(epochs, precisions)
    plt.savefig(precision_out)
    plt.clf()

    plt.plot(epochs, recalls)
    plt.savefig(recall_out)
    plt.clf()

    plt.plot(epochs, class_precisions)
    plt.savefig(class_precision_out)
    plt.clf()

    plt.plot(epochs, class_recalls)
    plt.savefig(class_recall_out)
    plt.clf()

    plt.plot(epochs, train_accuracies, label="Train")
    plt.plot(epochs, accuracies, label="Test")
    plt.legend()
    plt.savefig(accuracy_out)
    plt.clf()

    plt.plot(epochs, train_losses, label="Train")
    plt.plot(epochs, test_losses, label="Test")
    plt.legend()
    plt.savefig(loss_out)
    plt.clf()

class Convolution3DSmall(torch.nn.Module):
    def __init__(
            self,
            n_classes: int,
            in_channels: int = 7,
            out_channels: int = 16,
            kernel_size: int = 3,
            pool_size: int = 2

    ) -> None:
        super().__init__()
        self.pool_size = pool_size
        self.kernel_size = tuple([kernel_size] * 3)
        self.conv1 = self.make_conv_layer_pool(in_channels, out_channels)
        self.conv2 = self.make_conv_layer_pool(out_channels, out_channels * 2)
        self.fc1 = self.make_fc_layer(2**3 * out_channels * 2, 128, dropout=True)
        self.fc2 = self.make_fc_layer(128, n_classes)

    def make_conv_layer(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU()
        )

        return conv_layer

    def make_conv_layer_pool(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU(),
            nn.AvgPool3d(self.pool_size, stride=self.pool_size, padding=0, count_include_pad=False)
        )

        return conv_layer

    def make_fc_layer(self, in_channels, out_channels, dropout=False):
        if dropout:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels),
                nn.Dropout(p=0.15)

            )
        else:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels)
            )
        return fc_layer

    def forward(self, x: Tensor) -> torch.Tensor:
        out = self.conv1(x)
        out = self.conv2(out)
        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = self.fc2(out)

        return out

class Convolution3D(torch.nn.Module):
    def __init__(
            self,
            n_classes: int,
            input_size: int,
            in_channels: int = 7,
            conv_layers_out: ty.Tuple = (32, 32, 32),
            conv_pool: ty.Tuple = (2, 0, 0),
            conv_padding: ty.Tuple = (1, 1, 0),
            conv_stride: ty.Tuple = (1, 1, 1),
            hidden_fc_layers_out: ty.Tuple = (512, 256, 128),
            fc_dropout: ty.Tuple = (0.15, 0.15, 0.15),
            kernel_size: ty.Tuple = (3, 3, 3),

    ) -> None:
        super().__init__()
        assert len(conv_layers_out) == len(conv_pool) == len(conv_padding) == len(conv_stride)
        assert len(hidden_fc_layers_out) == len(fc_dropout)

        self.conv_layers = []

        nr_channels = in_channels
        box_size = input_size
        out_channels = in_channels

        for i, conv_layer_out in enumerate(conv_layers_out):
            conv_layer = self.make_conv_layer(nr_channels, conv_layer_out,
                                              padding=conv_padding[i],
                                              stride=conv_stride[i],
                                              pool=conv_pool[i], kernel_size=kernel_size[i])
            box_size = self.calculate_box_size(box_size, conv_stride[i], conv_pool[i], conv_padding[i], kernel_size[i])
            self.conv_layers.append(conv_layer)
            out_channels = conv_layer_out

        out_channels = box_size ** 3 * out_channels

        self.hidden_layers = []

        for i, hidden_layer_out in enumerate(hidden_fc_layers_out):
            dropout = fc_dropout[i]
            hidden_layer = self.make_fc_layer(out_channels, hidden_layer_out, dropout=dropout)
            out_channels = hidden_layer_out
            self.hidden_layers.append(hidden_layer)

        self.output_layer = self.make_fc_layer(out_channels, n_classes, dropout=0.0)

    def calculate_box_size(self, previous_box, stride, pool, padding, kernel_size):
        box_size = ((previous_box - kernel_size + 2 * padding) / stride) + 1
        if pool:
            box_size = ((box_size - self.pool_size) / self.pool_size) + 1

        return box_size

    def make_conv_layer(self, in_channels, out_channels, padding=0, stride=1, pool=0, kernel_size=3):
        conv_layer = nn.Conv3d(in_channels, out_channels, kernel_size=tuple([kernel_size] * 3), padding=padding,
                               stride=tuple([stride] * 3))
        activation_layer = nn.ReLU()

        if not pool:
            full_conv_layer = nn.Sequential(conv_layer, activation_layer)

        else:
            pool_layer = nn.AvgPool3d(pool, stride=pool, padding=0, count_include_pad=False)
            full_conv_layer = nn.Sequential(conv_layer, activation_layer, pool_layer)

        return full_conv_layer

    @staticmethod
    def make_fc_layer(in_channels, out_channels, dropout=0.15):
        fc_layer = nn.Sequential(nn.Linear(in_channels, out_channels),
                                 nn.ReLU(),
                                 nn.BatchNorm1d(out_channels),
                                 nn.Dropout(p=dropout))
        return fc_layer

    def forward(self, x: Tensor) -> torch.Tensor:
        out = x
        for conv_layer in self.conv_layers:
            out = conv_layer(out)

        out = out.view(out.size(0), -1)
        for hidden_layer in self.hidden_layers:
            out = hidden_layer(out)

        out = self.output_layer(out)

        return out

class Convolution3DOld(torch.nn.Module):
    def __init__(
            self,
            n_classes: int,
            in_channels: int = 7,
            out_channels: int = 32,
            kernel_size: int = 3,
            pool_size: int = 2

    ) -> None:
        super().__init__()
        self.pool_size = pool_size
        self.kernel_size = tuple([kernel_size] * 3)
        self.conv1 = self.make_conv_layer(in_channels, out_channels, padding=1)
        self.conv2 = self.make_conv_layer(out_channels, out_channels, padding=1)
        self.conv3 = self.make_conv_layer(out_channels, out_channels, padding=1)
        self.fc1 = self.make_fc_layer(17**3 * out_channels, 512)
        self.fc2 = self.make_fc_layer(512, 256)
        self.fc3 = self.make_fc_layer(256, 128)
        self.fc4 = self.make_fc_layer(128, n_classes, dropout=False)

    def make_conv_layer(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU()
        )

        return conv_layer

    def make_conv_layer_pool(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU(),
            nn.AvgPool3d(self.pool_size, stride=self.pool_size, padding=0, count_include_pad=False)
        )

        return conv_layer

    def make_fc_layer(self, in_channels, out_channels, dropout=True):
        if dropout:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels),
                nn.Dropout(p=0.15)

            )
        else:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels))

        return fc_layer

    def forward(self, x: Tensor) -> torch.Tensor:
        out = self.conv1(x)
        out = self.conv2(out)
        out = self.conv3(out)

        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = self.fc2(out)
        out = self.fc3(out)
        out = self.fc4(out)

        return out

class Convolution3DSeq(torch.nn.Module):
    def __init__(
            self,
            n_classes: int,
            in_channels: int = 7,
            out_channels: int = 32,
            kernel_size: int = 3,
            pool_size: int = 2

    ) -> None:
        super().__init__()
        self.pool_size = pool_size
        self.kernel_size = tuple([kernel_size] * 3)
        self.conv1 = self.make_conv_layer(in_channels, out_channels, padding=1)
        self.conv2 = self.make_conv_layer(out_channels, out_channels, padding=1)
        self.conv3 = self.make_conv_layer(out_channels, out_channels, padding=1)
        self.fc1 = self.make_fc_layer(17**3 * out_channels, 512)
        self.fc2 = self.make_fc_layer(512, 256)
        self.fc3 = self.make_fc_layer(256, 128)
        self.fc4 = self.make_fc_layer(128, n_classes, dropout=False)

    def make_conv_layer(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU()
        )

        return conv_layer

    def make_conv_layer_pool(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU(),
            nn.AvgPool3d(self.pool_size, stride=self.pool_size, padding=0, count_include_pad=False)
        )

        return conv_layer

    def make_fc_layer(self, in_channels, out_channels, dropout=True):
        if dropout:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels),
                nn.Dropout(p=0.15)

            )
        else:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels))

        return fc_layer

    def forward(self, x: Tensor) -> torch.Tensor:
        out = self.conv1(x)
        out = self.conv2(out)
        out = self.conv3(out)

        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = self.fc2(out)
        out = self.fc3(out)
        out = self.fc4(out)

        return out

class Convolution3DLarge(torch.nn.Module):
    def __init__(
            self,
            n_classes: int,
            in_channels: int = 7,
            out_channels: int = 32,
            kernel_size: int = 3,
            pool_size: int = 2

    ) -> None:
        super().__init__()
        self.pool_size = pool_size
        self.kernel_size = tuple([kernel_size] * 3)
        self.conv1 = self.make_conv_layer_pool(in_channels, out_channels, padding=1)
        self.conv2 = self.make_conv_layer(out_channels, out_channels * 2, padding=1)
        self.conv3 = self.make_conv_layer(out_channels * 2, out_channels * 2, padding=1)
        self.fc1 = self.make_fc_layer(8**3 * out_channels * 2, 16384)
        self.fc2 = self.make_fc_layer(16384, 4096)
        self.fc3 = self.make_fc_layer(4096, 1024)
        self.fc4 = self.make_fc_layer(1024, 256, dropout=False)
        self.fc5 = self.make_fc_layer(256, 128, dropout=False)
        self.fc6 = self.make_fc_layer(128, n_classes, dropout=False)

    def make_conv_layer(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU()
        )

        return conv_layer

    def make_conv_layer_pool(self, in_channels, out_channels, padding=0):
        conv_layer = nn.Sequential(
            nn.Conv3d(in_channels, out_channels, kernel_size=self.kernel_size, padding=padding),
            nn.ReLU(),
            nn.AvgPool3d(self.pool_size, stride=self.pool_size, padding=0, count_include_pad=False)
        )

        return conv_layer

    def make_fc_layer(self, in_channels, out_channels, dropout=True):
        if dropout:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels),
                nn.Dropout(p=0.15)

            )
        else:
            fc_layer = nn.Sequential(
                nn.Linear(in_channels, out_channels),
                nn.ReLU(),
                nn.BatchNorm1d(out_channels))

        return fc_layer

    def forward(self, x: Tensor) -> torch.Tensor:
        out = self.conv1(x)
        out = self.conv2(out)
        out = self.conv3(out)
        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = self.fc2(out)
        out = self.fc3(out)
        out = self.fc4(out)
        out = self.fc5(out)
        out = self.fc6(out)

        return out

def train_loop(
        model: nn.Module,
        loader: DataLoader,
        optimizer: Optimizer,
        criterion: ty.Callable
) -> ty.Tuple[np.ndarray, np.ndarray]:
    """
    Train loop for Shapenet.
    Arguments
    ---------
    model (nn.Module): Neural net model.
    loader (DataLoader): Data loader.
    optimizer (Optimizer): Optimizer.
    criterion (ty.Callable): Loss function.
    Returns
    -------
    float: Train loss.
    """
    model.train()
    loader = tqdm(loader, leave=False)
    batch_losses = []
    accuracies = []
    for protein, specificities in loader:
        optimizer.zero_grad()
        out = model(protein)
        result = calculate_metrics(out.tolist(), specificities.tolist())
        loss_batch = criterion(out, specificities.type(torch.float))
        batch_loss_value = loss_batch.item()
        loss_batch.backward()
        optimizer.step()
        batch_losses.append(batch_loss_value)
        accuracies.append(result['accuracy'])
    loss_value = np.mean(batch_losses)
    mean_accuracy = np.mean(np.array(accuracies))

    return loss_value, mean_accuracy

def eval_loop(
        model: nn.Module,
        loader: DataLoader,
        criterion: ty.Callable,
        epoch: int,
        metrics: ty.Dict[int, ty.Dict[str, SubstrateMetrics]],
        specificity_list: ty.List[str]
) -> ty.Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
    """
    Evaluation loop for Shapenet.

    Arguments
    ---------
    model (nn.Module): Neural net model.
    loader (DataLoader): Data loader.
    criterion (ty.Callable): Loss function.

    Returns
    -------
    ty.Tuple[float, float]: Test loss and accuracy.
    """
    model.eval()
    loader = tqdm(loader, leave=False)
    batch_losses = []
    batch_nr = 0
    accuracies = []
    recalls = []
    class_recalls = []
    precisions = []
    class_precisions = []

    for protein, specificities in loader:
        batch_nr += 1
        with torch.no_grad():
            prediction = model(protein)
            loss_batch = criterion(prediction, specificities.type(torch.float))
            batch_loss_value = loss_batch.item()
            batch_losses.append(batch_loss_value)
            result = calculate_metrics(prediction.tolist(), specificities.tolist())
            store_aa_metrics(metrics, prediction, specificities, specificity_list, epoch)

            recalls.append(result['micro/recall'])
            precisions.append(result['micro/precision'])
            class_recalls.append(result['macro/recall'])
            class_precisions.append(result['macro/precision'])
            accuracies.append(result['accuracy'])

    loss_value = np.mean(batch_losses)
    mean_accuracy = np.mean(np.array(accuracies))
    mean_recall = np.mean(np.array(recalls))
    mean_precision = np.mean(np.array(precisions))
    mean_class_recall = np.mean(np.array(class_recalls))
    mean_class_precision = np.mean(np.array(class_precisions))

    return loss_value, mean_accuracy, mean_recall, mean_precision, mean_class_recall, mean_class_precision

def save_model(
        epoch: int,
        model: nn.Module,
        optimizer: Optimizer,
        save_dir: str,
) -> None:
    """
    Save model.

    Arguments
    ---------
    epoch (int): Epoch.
    model (nn.Module): Neural net model.
    optimizer (Optimizer): Optimizer.
    save_dir (str): Save directory.
    """
    state = {
        "epoch": epoch,
        "state_dict": model.state_dict(),
        "optimizer": optimizer.state_dict()
    }
    torch.save(state, os.path.join(save_dir, f"model_epoch_{epoch + 1}.pth.tar"))

def load_checkpoint(model, optimizer, filename):
    # Note: Input model & optimizer should be pre-defined.  This routine only updates their states.
    start_epoch = 0
    if os.path.isfile(filename):
        print("=> loading checkpoint '{}'".format(filename))
        checkpoint = torch.load(filename)
        start_epoch = checkpoint['epoch']
        model.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        print(f"=> loaded checkpoint '{filename}' (epoch {start_epoch})")
    else:
        print(f"=> no checkpoint found at '{filename}'")

    return model, optimizer, start_epoch

def main() -> None:
    """
    Driver code.
    """
    args = parse_arguments()
    if not os.path.exists(args.o):
        os.mkdir(args.o)

    train = VoxelDataset(args.i, args.train, args.s, SUBSTRATE_PATH)
    test = VoxelDataset(args.i, args.test, args.s, SUBSTRATE_PATH)

    if len(train) % args.b == 1:

        train_loader = DataLoader(train, shuffle=True, batch_size=args.b, drop_last=True)
    else:
        train_loader = DataLoader(train, shuffle=True, batch_size=args.b)

    if len(test) % args.b == 1:
        test_loader = DataLoader(test, shuffle=True, batch_size=args.b, drop_last=True)
    else:
        test_loader = DataLoader(test, shuffle=True, batch_size=args.b)

    model = Convolution3D(len(train.all_specificities))

    weights = torch.FloatTensor(train.get_weights())

    lr = args.l
    criterion = nn.BCEWithLogitsLoss(pos_weight=weights)
    optimizer = Adam(params=model.parameters(), lr=lr)

    start_epoch = 0

    if args.m:
        model, optimizer, start_epoch = load_checkpoint(model, optimizer, args.m)

    scheduler = lr_scheduler.ReduceLROnPlateau(
        optimizer=optimizer,
        mode='min',
        factor=0.5,
        patience=5,
        min_lr=0.00001
    )

    epochs, train_losses, test_losses, accuracies, learning_rates, precisions, recalls, class_precisions, class_recalls, train_accuracies = \
        [], [], [], [], [], [], [], [], [], []

    num_epochs = args.e
    aa_metrics_folder = os.path.join(args.o, "amino_acid_metrics")
    if not os.path.exists(aa_metrics_folder):
        os.mkdir(aa_metrics_folder)
    aa_metrics = {}
    for epoch in range(num_epochs):
        aa_metrics[epoch + 1 + start_epoch] = {}
        for spec in train.all_specificities:
            aa_metrics[epoch + 1 + start_epoch][spec] = SubstrateMetrics(spec)

    for epoch in range(num_epochs):

        lr = scheduler.optimizer.param_groups[0]['lr']
        train_loss, train_accuracy = train_loop(
            model=model,
            loader=train_loader,
            optimizer=optimizer,
            criterion=criterion
        )
        test_loss, accuracy, recall, precision, class_recall, class_precision = eval_loop(
            model=model,
            loader=test_loader,
            criterion=criterion,
            epoch=epoch + start_epoch + 1,
            metrics=aa_metrics,
            specificity_list=train.all_specificities
        )
        scheduler.step(test_loss)
        for aa, metrics in aa_metrics[epoch + start_epoch + 1].items():
            metrics.print()

        epochs.append(epoch + 1 + start_epoch)
        train_losses.append(train_loss)
        test_losses.append(test_loss)
        learning_rates.append(lr)
        accuracies.append(accuracy)
        recalls.append(recall)
        class_recalls.append(class_recall)
        precisions.append(precision)
        class_precisions.append(class_precision)
        train_accuracies.append(train_accuracy)

        write_aa_metrics(aa_metrics_folder, epoch + start_epoch + 1, aa_metrics[epoch + start_epoch + 1], train.all_specificities)

        print(
            f"Epoch: {epoch + 1 + start_epoch}/{num_epochs + start_epoch}; "
            f"LR: {lr}; "
            f"Train loss: ", train_loss,
            f"Test loss: ", test_loss,
            f"Accuracy: ", accuracy,
            f"Recall: ", recall,
            f"Precision: ", precision,
        )

        if (epoch + 1) % 10 == 0:
            save_model(epoch + start_epoch, model, optimizer, args.o)

    out_plots = os.path.join(args.o, "plots")

    if not os.path.exists(out_plots):
        os.mkdir(out_plots)
    metric_plots = os.path.join(out_plots, "amino_acid_metrics")
    if not os.path.exists(metric_plots):
        os.mkdir(metric_plots)

    plot_aa_metrics(metric_plots, aa_metrics, train.all_specificities)

    metrics_out = os.path.join(args.o, "metrics.txt")
    write_metrics(metrics_out, epochs, train_losses, test_losses, accuracies, learning_rates, precisions, recalls,
                  class_precisions, class_recalls, train_accuracies)

    plot_metrics(out_plots, epochs, train_losses, test_losses, accuracies, precisions, recalls,
                 class_precisions, class_recalls, train_accuracies)

    save_model(num_epochs, model, optimizer, args.o)

def calculate_metrics(pred, target, threshold=0.5):
    pred = np.array(pred)
    target = np.array(target)
    pred = np.array(pred > threshold, dtype=float)

    return {'micro/precision': precision_score(y_true=target, y_pred=pred, average='micro', zero_division=0),
            'micro/recall': recall_score(y_true=target, y_pred=pred, average='micro', zero_division=0),
            'micro/f1': f1_score(y_true=target, y_pred=pred, average='micro', zero_division=0),
            'macro/precision': precision_score(y_true=target, y_pred=pred, average='macro', zero_division=0),
            'macro/recall': recall_score(y_true=target, y_pred=pred, average='macro', zero_division=0),
            'macro/f1': f1_score(y_true=target, y_pred=pred, average='macro', zero_division=0),
            'samples/precision': precision_score(y_true=target, y_pred=pred, average='samples', zero_division=0),
            'samples/recall': recall_score(y_true=target, y_pred=pred, average='samples', zero_division=0),
            'samples/f1': f1_score(y_true=target, y_pred=pred, average='samples', zero_division=0),
            'accuracy': accuracy_score(y_true=target, y_pred=pred)
            }

def store_aa_metrics(metrics, prediction, specificities, specificity_list, epoch):

    for i, pred in enumerate(prediction):
        spec = specificities[i]
        correct_indices = []

        for j, true_y in enumerate(spec):
            pred_y = pred[j]
            specificity = specificity_list[j]
            metrics[epoch][specificity].process_prediction(pred_y, true_y)

            if true_y > 0.5:
                correct_indices.append(j)

        for k, pred_y in enumerate(pred):
            if pred_y > 0.5 and k not in correct_indices:
                for idx in correct_indices:
                    metrics[epoch][specificity_list[k]].mispredictions.append(specificity_list[idx])

if __name__ == "__main__":
    main()


def loopSize(subject, times):
    value = 1
    for _ in range(times):
        value *= subject
        value = value % 20201227
    return value

def getLoopFromPublic(x):
    for i in range(0, 100000):
        if loopSize(7, i) == x:
            return i
    return -1

def findSize(x):
    # x = 14505727
    size = 0
    while x > 1:
        size += 1
        while x % 7 != 0:
            x += 20201227
        x /= 7
        print(round(size * 1000 / 20201227) / 10)
    print(size)
    return size

# findSize(16616892)
print(loopSize(7, 4711642))


for i in range(int(input())):
    mx = 0
    lst = list(map(int, input().split()))
    del lst[0]
    lst.sort()
    for idx in range(1, len(lst)):
        a = abs(lst[idx] - lst[idx - 1])
        if mx < a:
            mx = a
    print(f"Class {i + 1}")
    print(f"Max {max(lst)}, Min {min(lst)}, Largest gap {mx}")


import json
import spacy
from spacy.matcher import PhraseMatcher
from utils import preprocess
import nltk.data

def create_dataset():
    languages = []
    training = []
    sentences = []
    tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
    with open('languages_sm.json') as f:
        languages = json.load(f)
    with open('training_data.json') as f:
        training = json.load(f)[:1000]
    for doc in training:
        if doc is not None:
            for sentence  in tokenizer.tokenize(doc):
                sentences.append(sentence)

    labeled_data = extract_patterns(languages, sentences)
    print("Number of labelled data {0} ".format(len(labeled_data)))
    with open('prog_language_train_data_lang_sm.json', 'w') as outfile:
        json.dump(labeled_data, outfile)

def extract_patterns(languages, data):
    nlp = spacy.load('en_core_web_sm')
    matcher = PhraseMatcher(nlp.vocab)
    result = []
    data_len = len(data)
    print("Number of available data {0} ".format(data_len))
    patterns = [nlp.make_doc(text) for text in languages]
    matcher.add('PROG_LANG', None, *patterns)

    for i, text in enumerate(data):
        print("{0}/{1}".format(i, data_len))
        if text is not None:
            match_results = []
            doc = nlp(preprocess(text))
            used_indices = []
            matches = matcher(doc)
            lang_available = False
            for match_id, start, end in matches:
                string_id = nlp.vocab.strings[match_id]  # get string representation
                lang_available = True
                span = doc[start: end]
                match_results.append((span.start_char, span.end_char, string_id))
                used_indices.append(start)
                print(span.start_char, span.end_char, span.text)
            for entity in doc.ents:
                if entity.start_char not in used_indices:
                    match_results.append((entity.start_char, entity.end_char , entity.label_))
            if lang_available:
                result.append((text, match_results))
    return result

if __name__ == "__main__":
    create_dataset()


import pandas as pd
from sklearn import tree, train_test_split
import pickle

# データの読み込み
df = pd.read_csv('iris.csv')

# 各列の中央値を計算
col_mean = df.mean()

# 欠損値を中央値で埋める
df2 = df.fillna(col_mean)

# 特徴量と正解データに分割
xcol = ['がく片長さ', 'がく片幅', '花弁長さ', '花弁幅']
x = df2[xcol]
t = df2['種類']

# 訓練データとテストデータに分割
x_train, x_test, t_train, t_test = train_test_split(x, t, test_size = 0.3, random_state = 0)

# 決定木モデルの準備
tree_model = tree.DecisionTreeClassifier(random_state = 0)

tree_model.fit(x_train, t_train)
score = tree_model.score(x_test, t_test)

print('正解率: {}'.format(score))

# モデルの保存
with open('irismodel.pkl', mode = 'wb') as fp:
    pickle.dump(tree_model, fp)


import os
import numpy as np
import torch
import torch.nn as nn
import logging
import utils.data_utils as data_utils
from utils.misc import create_folder
import pandas as pd
from AgeMapper import AgeMapper
from collections import OrderedDict

log = logging.getLogger(__name__)
MSELoss = nn.MSELoss()

def evaluate_data(trained_model_path: str,
                     data_directory: str,
                     data_list: str,
                     prediction_output_path: str,
                     prediction_output_statistics_name: str,
                     modality_flag: str,
                     control: str,
                     device: str,
                     test_ages: str,
                     dataset_sex: str,
                     scaling_values: str,
                     exit_on_error: bool = False,) -> None:

    """ Evaluate Data Function
    This function evaluates the data and generates the output statistics

    Parameters:
    -----------
    trained_model_path : str
        Path to the trained model
    data_directory : str
        Path to the data directory
    data_list : str
        Path to the data list
    prediction_output_path : str
        Path to the prediction output
    prediction_output_statistics_name : str
        Name of the prediction output statistics
    modality_flag : str
        Modality flag
    control : str
        Control
    device : str
        Device
    test_ages : str
        Path to the test ages

    Returns:
    --------
    None

    """

    log.info("Started Evaluation. Check tensorboard for plots (if a LogWriter is provided)")

    with open(data_list) as data_list_file:
        volumes_to_be_used = data_list_file.read().splitlines()

    scaling_values_simple = pd.read_csv(scaling_values, index_col=0)

    scale_factor = scaling_values_simple.loc[modality_flag].scale_factor
    resolution = scaling_values_simple.loc[modality_flag].resolution
    mapping_data_file = scaling_values_simple.loc[modality_flag].data_file

    modality_flag_split = modality_flag.rsplit('_', 1)
    if modality_flag_split[0] == 'rsfmri':
        rsfmri_volume = int(modality_flag_split[1])
    else:
        rsfmri_volume = None

    model = AgeMapper(resolution=resolution,
                      )

    trained_model = torch.load(trained_model_path,map_location=torch.device(device))

    if hasattr(trained_model, 'state_dict'):
        model_sd = trained_model.state_dict()
        if torch.cuda.device_count()>1 or device=='cpu':
            correct_state_dict = {}
            for key in model_sd.keys():
                if key.startswith('module.'):
                    new_key = key.replace('module.', "")
                    correct_state_dict[new_key] = model_sd[key]
                else:
                    correct_state_dict[key] = model_sd[key]
            correct_state_dict = OrderedDict(correct_state_dict)
            del model_sd
        model.load_state_dict(correct_state_dict)
    else:
        model.load_state_dict(trained_model)

    del trained_model

    if torch.cuda.is_available() == True and device!='cpu':
        model.cuda(device)
        cuda_available = torch.cuda.is_available()
    else:
        cuda_available = 'False'

    model.eval()

    # Create the prediction path folder if this is not available

    create_folder(prediction_output_path)

    # Initiate the evaluation

    log.info("Evaluation Started")

    file_paths, volumes_to_be_used = data_utils.load_file_paths(data_list, data_directory, mapping_data_file)

    if control == 'mean':
        prediction_output_statistics_name = "output_statistics_mean_target.csv"
    elif control == 'null':
        prediction_output_statistics_name = "output_statistics_null_target.csv"

    output_statistics = {}
    output_statistics_path = os.path.join(prediction_output_path, prediction_output_statistics_name)

    with torch.no_grad():

        for volume_index, file_path in enumerate(file_paths):
            try:
                print("\r Mapping Volume {:.3f}%: {}/{} test subjects".format((volume_index+1)/len(file_paths) * 100.0, volume_index+1, len(file_paths)), end='')

                # Generate volume & header

                subject = volumes_to_be_used[volume_index]

                output_age = _generate_age(file_path,
                                        model,
                                        device,
                                        cuda_available,
                                        modality_flag,
                                        resolution,
                                        scale_factor,
                                        rsfmri_volume
                                        )

                if control == 'mean':
                    target_age = _load_mean(dataset_sex)
                elif control == 'null':
                    target_age = np.array(0.0)
                elif control == 'both':
                    target_age_mean = _load_mean(dataset_sex)
                    target_age_null = np.array(0.0)
                    target_age = _load_target(
                        volume_index,
                        test_ages
                    )
                else:
                    target_age = _load_target(
                        volume_index,
                        test_ages
                    )
                if control == 'both':
                    age_delta, loss = _statistics_calculator(output_age, target_age)
                    age_delta_mean, loss_mean = _statistics_calculator(output_age, target_age_mean)
                    age_delta_null, loss_null = _statistics_calculator(output_age, target_age_null)
                    output_statistics[subject] = [target_age, output_age, age_delta, loss, age_delta_mean, loss_mean, age_delta_null, loss_null]
                else:
                    age_delta, loss = _statistics_calculator(output_age, target_age)
                    output_statistics[subject] = [target_age, output_age, age_delta, loss]

                log.info("Processed: " + volumes_to_be_used[volume_index] + " " + str(volume_index + 1) + " out of " + str(len(volumes_to_be_used)))

            except FileNotFoundError as exception_expression:
                log.error("Error in reading the provided file!")
                log.exception(exception_expression)
                if exit_on_error:
                    raise(exception_expression)

            except Exception as exception_expression:
                log.error("Error code execution!")
                log.exception(exception_expression)
                if exit_on_error:
                    raise(exception_expression)

        if control == 'both':
            columns=['target_age', 'output_age', 'age_delta', 'loss', 'age_delta_mean', 'loss_mean', 'age_delta_null', 'loss_null']
            output_statistics_df = pd.DataFrame.from_dict(output_statistics, orient='index', columns=columns)
        else:
            output_statistics_df = pd.DataFrame.from_dict(output_statistics, orient='index', columns=['target_age', 'output_age', 'age_delta', 'loss'])
        output_statistics_df.to_csv(output_statistics_path)

    log.info("Output Data Generation Complete")

def _generate_age(file_path: str,
                model: nn.Module,
                device: str,
                cuda_available: bool,
                modality_flag: str,
                resolution: int,
                scale_factor: float,
                rsfmri_volume: int
                ) -> np.ndarray:

    """ Function wrapper for generating the age prediction

    Parameters:
    -----------
    file_path : str
        Path to the file
    model : nn.Module
        Model
    device : str
        Device
    cuda_available : bool
        Whether cuda is available or not
    modality_flag : str
        Modality flag
    resolution : int
        Resolution
    scale_factor : float
        Scale factor
    rsfmri_volume : int
        rsfmri volume

    Returns:
    --------
    output : np.ndarray
        Output

    """

    volume = data_utils.load_and_preprocess_evaluation(file_path=file_path, modality_flag=modality_flag,
                                                        resolution=resolution, scale_factor=scale_factor,
                                                        rsfmri_volume=rsfmri_volume)

    if len(volume.shape) != 5:
        volume = volume[np.newaxis, np.newaxis, :, :, :]

    volume = torch.tensor(volume).type(torch.FloatTensor)

    if cuda_available and (type(device) == int):
        volume = volume.cuda(device)

    output = model(volume)
    output = (output.cpu().numpy()).astype('float32')
    output = np.squeeze(output)

    return output

def _load_mean(dataset_sex: str) -> np.ndarray:
    """ Function wrapper for loading the mean age

    Parameters:
    -----------
    dataset_sex : str
        Sex of the subjects in the dataset under consideration

    Returns:
    --------
    mean_age : np.ndarray
        Mean age

    """

    if dataset_sex == 'male':
        mean_age = 63.370316719492536
    else:
        mean_age = 64.64810970818492

    mean_age = np.array(np.float32(mean_age))

    return mean_age

def _load_target(volume_index: int,
                 test_ages: str) -> np.ndarray:

    """ Function wrapper for loading the target age

    Parameters:
    -----------
    volume_index : int
        Volume index
    test_ages : str
        Path to the test ages

    Returns:
    --------
    target_age : np.ndarray
        Target age

    """

    target_age = np.load(test_ages)[volume_index]

    return target_age

def _statistics_calculator(output_age: np.ndarray,
                           target_age: np.ndarray) -> tuple:

    """ Function wrapper for calculating the statistics

    Parameters:
    -----------
    output_age : np.ndarray
        Output age
    target_age : np.ndarray
        Target age

    Returns:
    --------
    age_delta : np.ndarray
        Age delta
    loss : np.ndarray
        MSE loss

    """

    output_age = np.array(output_age)
    target_age = np.array(target_age)
    age_delta = output_age - target_age
    loss = MSELoss(torch.from_numpy(output_age), torch.from_numpy(target_age)).numpy()

    return age_delta, loss


#!/usr/bin/env python
# -*-coding:utf-8-*-
# @Time:2021/7/12 19:04
# @Author:ZhangFY
# @File:hill_sort.py

import random
import time

'''
    此文件为希尔排序的python实现
    希尔排序算法核心，是由简单的插入排序进阶改进，又称缩小增量排序，核心操作为首先将待排序队列
    依据某种增量值进行分组，每个分组都进行插入排序，直到增量值为1时，最后执行一遍插入排序完成排序
    希尔排序执行效率远高于普通插入排序
'''

def timer_count(f):
    def decor(*args,**kwargs):
        s_time = time.time()
        r = f(*args,**kwargs)
        e_time = time.time()
        print("{}函数耗时{}秒".format(f.__name__,e_time-s_time))
        return r
    return decor

# 插入排序功能函数,插入排序核心为：单个元素可认为是有序的
@timer_count
def insert_sort(l):
    print("排序前：",l)
    for i in range(1,len(l)):
        j = i
        temp = l[i]
        while j > 0 and l[j-1] > temp:    # 寻找插入位置
            l[j] = l[j-1]
            j -= 1
        l[j] = temp
    print("排序后：",l)

def insert_sort_inside(l):
    for i in range(1,len(l)):
        j = i
        temp = l[i]
        while j > 0 and l[j-1] > temp:    # 寻找插入位置
            l[j] = l[j-1]
            j -= 1
        l[j] = temp

# 这里定义增量为len(l)/2,len(l)/2/2,len(l)/2/2/2.....1
# 希尔排序主要逻辑
@timer_count
def hill_way(l):
    if isinstance(l,list):
        print("排序前",l)
        increment = len(l) // 2
        while increment > 0:
            # 获取当此循环分组个数，分组个数决定下方
            for index in range(increment):
                group_l = []
                index_l = []
                group_num = len(l) // increment  # 获取当前分组个数下每个组中元素个数

                for ti in range(group_num):
                    group_l.append(l[index + ti * increment])     # 根据组数，每个组元素个数，构建分组后的列表
                    index_l.append(index + ti * increment)        # 分组后列表对应原数据的索引，便于排序后逐个替换到原数据中

                insert_sort_inside(group_l)                       # 分组后数据排序
                i = 0
                for index_info in index_l:
                    l[index_info] = group_l[i]                    # 分组后数据替换
                    i += 1
            increment = increment // 2
            if increment == 1:
                insert_sort_inside(l)                             # 最后一轮
                break
        print("排序后",l)
    else:
        raise TypeError("函数hill_way参数类型有误")

if __name__ == '__main__':
    l = [random.randint(5,500) for x in range(250000)]
    import copy
    ll = copy.deepcopy(l)
    # 测试两块执行时间对比
    hill_way(l)
    insert_sort(ll)


from flask import Flask, request, abort

from linebot import (
    LineBotApi, WebhookHandler
)
from linebot.exceptions import (
    InvalidSignatureError
)
from linebot.models import *

from io import StringIO
from pyquery import PyQuery as pq

import requests
import random
import json
import math
import time
import datetime
import os
import apiai

#---------------- self define module ----------------
import text_push as text_push
import text_reply as text_reply

#---------------- line settings ----------------
# Channel Access Token
LINE_CHANNEL_ACCESS_TOKEN = os.environ.get('LINE_CHANNEL_ACCESS_TOKEN')
line_bot_api = LineBotApi(LINE_CHANNEL_ACCESS_TOKEN)
# Channel Secret
LINE_CHANNEL_SECRET = os.environ.get('LINE_CHANNEL_SECRET')
handler = WebhookHandler(LINE_CHANNEL_SECRET)
# DIALOGFLOW_CLIENT_ACCESS_TOKEN
DIALOGFLOW_CLIENT_ACCESS_TOKEN = os.environ.get('DIALOGFLOW_CLIENT_ACCESS_TOKEN')
ai = apiai.ApiAI(DIALOGFLOW_CLIENT_ACCESS_TOKEN)

#---------------------------------------------------

app = Flask(__name__)

@app.route("/callback", methods=['POST'])
def callback():
    signature = request.headers['X-Line-Signature']
    body = request.get_data(as_text=True)
    app.logger.info("Request body: " + body)
    try:
        handler.handle(body, signature)
    except InvalidSignatureError:
        abort(400)
    return 'OK'
def news():
    res = requests.get('https://m.ctee.com.tw/livenews')
    res.encoding = 'utf-8'
    doc = pq(res.text)
    doc.make_links_absolute(base_url=res.url)
    count = 0
    content = ''
    for eachItem in doc('.item-content .title').items():
        if count < 5:
            title = eachItem('a').text()
            url = eachItem('a').attr('href')
            count += 1
        else:
            break
        content += '{}\n{}\n\n'.format(title, url)
    return content

def find_weather():
    res = requests.get('https://weather.com/zh-TW/weather/today/l/TWXX0021:1:TW')
    res.encoding = 'utf-8'
    doc = pq(res.text)
    content = ''
    temper = doc('.today_nowcard-temp').text()
    weather = doc('.today_nowcard-phrase').text()
    soma_temper = doc('.deg-feels').text()
    content = '地點: 台灣, 台北\n現在氣溫: {}\n天氣: {}\n體感溫度: {}'.format(temper, weather, soma_temper)
    return content

@handler.add(MessageEvent)
def handle_message(event):
    print(event)
    message_send_time = float(event.timestamp)/1000
    message_get_time = float(time.time())
    msg_type = event.message.type

    if event.message.text == "info":
        output_message = TextSendMessage(text=str(event))
        line_bot_api.reply_message(event.reply_token, output_message)

    if event.message.text.lower() == "speed" :
        output_message = ("【收到訊息時間】\n{} 秒\n【處理訊息時間】\n{} 秒".format(message_get_time-message_send_time,float(time.time())-message_get_time))
        output_message = text_reply.text_reply_message(user_message)
        line_bot_api.reply_message(event.reply_token, output_message)

    if msg_type == "text":
        if event.message.text == "更多選項":
            buttons_template = TemplateSendMessage(
                alt_text='更多選項 Template',
                template=ButtonsTemplate(
                    title='選單',
                    text='請選擇',
                    thumbnail_image_url='https://i.imgur.com/xQF5dZT.jpg',
                    actions=[
                        MessageTemplateAction(
                            label='關於我',
                            text='關於我'
                        ),
                        MessageTemplateAction(
                            label='經歷',
                            text='經歷'
                        ),
                        MessageTemplateAction(
                            label='作品集',
                            text='作品集'
                        ),
                        MessageTemplateAction(
                            label='其他功能',
                            text='其他功能'
                        )
                    ]
                )
            )
            line_bot_api.reply_message(event.reply_token, buttons_template)
            return 0
        elif event.message.text == "關於我":
            buttons_template = TemplateSendMessage(
                alt_text='關於我 Template',
                template=ButtonsTemplate(
                    title='關於我',
                    text='請選擇',
                    thumbnail_image_url='https://i.imgur.com/xQF5dZT.jpg',
                    actions=[
                        MessageTemplateAction(
                            label='個性',
                            text='個性'
                        ),
                        MessageTemplateAction(
                            label='核心技能',
                            text='核心技能'
                        ),
                        MessageTemplateAction(
                            label='競爭優勢',
                            text='競爭優勢'
                        ),
                        MessageTemplateAction(
                            label='職涯目標',
                            text='職涯目標'
                        )
                    ]
                )
            )
            line_bot_api.reply_message(event.reply_token, buttons_template)
            return 0
        elif event.message.text == "經歷":
            buttons_template = TemplateSendMessage(
                alt_text='經歷 Template',
                template=ButtonsTemplate(
                    title='經歷',
                    text='請選擇',
                    thumbnail_image_url='https://i.imgur.com/xQF5dZT.jpg',
                    actions=[
                        MessageTemplateAction(
                            label='求學經歷',
                            text='求學經歷'
                        ),
                        MessageTemplateAction(
                            label='工作經驗',
                            text='工作經驗'
                        ),
                        MessageTemplateAction(
                            label='社團經歷',
                            text='社團經歷'
                        )
                    ]
                )
            )
            line_bot_api.reply_message(event.reply_token, buttons_template)
            return 0
        elif event.message.text == "作品集":
            buttons_template = TemplateSendMessage(
                alt_text='作品集 Template',
                template=ButtonsTemplate(
                    title='作品介紹',
                    text='請選擇',
                    thumbnail_image_url='https://i.imgur.com/xQF5dZT.jpg',
                    actions=[
                        MessageTemplateAction(
                            label='實習職缺爬蟲整合工具',
                            text='實習職缺爬蟲整合工具'
                        ),
                        MessageTemplateAction(
                            label='狗狗主題搜尋引擎網站',
                            text='狗狗主題搜尋引擎網站'
                        ),
                        MessageTemplateAction(
                            label='麵包店產銷系統',
                            text='麵包店產銷系統'
                        ),
                        URIAction(
                            label='Github連結',
                            uri='https://github.com/nicolela5693'
                        )
                    ]
                )
            )
            line_bot_api.reply_message(event.reply_token, buttons_template)
            return 0
        elif event.message.text == "其他功能":
            buttons_template = TemplateSendMessage(
                alt_text='其他功能 Template',
                template=ButtonsTemplate(
                    title='其他功能',
                    text='請選擇',
                    thumbnail_image_url='https://i.imgur.com/xQF5dZT.jpg',
                    actions=[
                        MessageTemplateAction(
                            label='今日天氣',
                            text='今日天氣'
                        ),
                        MessageTemplateAction(
                            label='即時新聞',
                            text='即時新聞'
                        ),
                        URIAction(
                            label='分享Bot',
                            uri='https://line.me/R/nv/recommendOA/@657jmdbc'
                        )
                    ]
                )
            )
            line_bot_api.reply_message(event.reply_token, buttons_template)
            return 0
        elif event.message.text == "聯絡我":
            buttons_template = TemplateSendMessage(
                alt_text='聯絡我 Template',
                template=ButtonsTemplate(
                    title='聯絡資訊',
                    text='請選擇',
                    thumbnail_image_url='https://i.imgur.com/xQF5dZT.jpg',
                    actions=[
                        MessageTemplateAction(
                            label='Email',
                            text='Email'
                        ),
                        URIAction(
                            label='LinkedIn',
                            uri='https://www.linkedin.com/in/yujin-lee-taiwan/'
                        ),
                        URIAction(
                            label='Github',
                            uri='https://github.com/nicolela5693'
                        )
                    ]
                )
            )
            line_bot_api.reply_message(event.reply_token, buttons_template)
            return 0
        elif event.message.text == "即時新聞":
            content = news()
            line_bot_api.reply_message(
                event.reply_token,
                TextSendMessage(text=content))
            return 0
        elif event.message.text == "今日天氣":
            content = find_weather()
            line_bot_api.reply_message(
                event.reply_token,
                TextSendMessage(text=content))
            return 0
        else:
            user_message = event.message.text
            output_message = text_reply.text_reply_message(user_message)
            line_bot_api.reply_message(event.reply_token, output_message)

@handler.add(MessageEvent, message=StickerMessage)
def handle_sticker_message(event):
    print("package_id:", event.message.package_id)
    print("sticker_id:", event.message.sticker_id)
    sticker_ids = [52002734, 52002735, 52002736, 52002738, 52002744, 52002745, 52002752, 52002768]
    index_id = random.randint(0, len(sticker_ids) - 1)
    sticker_id = str(sticker_ids[index_id])
    print(index_id)
    sticker_message = StickerSendMessage(
        package_id = '11537',
        sticker_id=sticker_id
    )
    line_bot_api.reply_message(
        event.reply_token,
        sticker_message)

if __name__ == "__main__":
    port = int(os.environ.get('PORT', 5000))
    app.run(host='0.0.0.0', port=port)


# -*- encoding: utf-8 -*-
"""
两数相加
"""

from __future__ import absolute_import, unicode_literals

# Definition for singly-linked list.
class ListNode(object):
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

"""
输入：l1 = [2,4,3], l2 = [5,6,4]
输出：[7,0,8]
解释：342 + 465 = 807.
示例 2：

输入：l1 = [0], l2 = [0]
输出：[0]
示例 3：

输入：l1 = [9,9,9,9,9,9,9], l2 = [9,9,9,9]
输出：[8,9,9,9,0,0,0,1]
"""
class Solution(object):
    def addTwoNumbers(self, l1, l2):
        """
        :type l1: ListNode
        :type l2: ListNode
        :rtype: ListNode
        """
        def do(cur, node):
            div_num = cur / 10
            yu_num = cur % 10
            node.next = ListNode(yu_num, None)
            node = node.next
            return div_num, node

        node = ListNode()
        dummy = node
        div_num = 0
        while (l1 and l2):
            cur = l1.val + l2.val + div_num
            div_num, node = do(cur, node)
            l1 = l1.next
            l2 = l2.next

        while l2:
            value = l2.val + div_num
            div_num, node = do(value, node)
            l2 = l2.next
        while l1:
            value = l1.val + div_num
            div_num, node = do(value, node)
            l1 = l1.next
        if div_num:
            node.next = ListNode(div_num, None)
        return dummy.next


# 2. 程序设计：（以下这些是一个题目）
# 自定义一个Integer类
# 该类有个input方法，调用该方法一定会得到一个合法的数字(非数字为不合法，越界为不合法)
# 1. 如果是非数字不合法，需要打印不合法消息，然后用户需要重新输入
# 如输入'abc'，不合法消息就为：invalid literal for int() with base 10: 'abc'
# 2. 如果是越界不合法，需要打印不合法消息，然后用户需要重新输入
# 如输入'2147483648'，不合法消息就为：ErrorMsg：2147483648 - 越界
# 提示：
# 1. 该方法需要捕获并处理两次异常(内置异常ValueError，自定义异常SlopOverError)
# 2. 该方法运用到递归方式处理更简单，如果用不到递归也可以
# 3. 该类有个verifySlopOver方法，可以判断传入的数字是否越界(非-2147483648 ~ 2147483647为越界)
#     如果数字越界，会主动抛出自定义SlopOverError异常，并传入数字和异常消息
# 4. 自定义异常SlopOverError类
#     该类需要重写__init__方法
#     有number、massage两个参数，number是数字类型的数，massage是字符串类型的异常消息
#     通过number、massage两个属性格式化异常信息

class SlopOverError(BaseException):
    def __init__(self, number, message):
        self.number = number
        self.message = message

    def __str__(self):
        return 'ErrorMsg：%d - %s' % (self.number, self.message)

class Integer:
    data = None

    def __init__(self):
        self.data = self.get_num()

    def get_num(self):
        error_flag = True
        try:
            input_number = input("please input a integer ")
            number = int(input_number)
            self.verifySlopOver(number)

        except ValueError as err:
            print(err)
        except SlopOverError as err:
            print(err)
        except Exception as err:
            print("err")
        else:
            error_flag = False
        finally:
            if error_flag:
                number = self.get_num()
                return number
            else:
                return number

    @staticmethod
    def verifySlopOver(number):
        if -2147483648 <= number <= 2147483647:
            return True
        else:
            raise SlopOverError(number, '越界')
            return False

if __name__ == '__main__':
    a = Integer()
    print(a.data)


import os

import jax

from config import get_args
from data import get_dataset
from loss import compute_losses
from model import build_and_restore_model
from summary import get_summary_fn
from training import train

def main(args):
    experiment_path = os.path.join(args.experiments_dir, args.experiment_name)
    checkpoint_path = os.path.join(experiment_path, "checkpoint")
    summary_path = os.path.join(experiment_path, "summary")
    os.makedirs(checkpoint_path, exist_ok=True)
    os.makedirs(summary_path, exist_ok=True)

    data_iterator, dimensions = get_dataset(args.dataset, args.datasets_dir,
                                            args.batch_size)

    model, params, opt_state, step = build_and_restore_model(dimensions, args,
                                                             checkpoint_path,
                                                             args.model_seed)

    def loss_fn(params, k, data, t):
        def model_fn(x): return model.apply(
            params, x, method=model.with_mf_grad)
        return compute_losses(model_fn, k, data, t, args.weights)

    summary_fn = get_summary_fn(summary_path, model)

    k_train = jax.random.PRNGKey(1)

    print("Beginning training")
    train(params, opt_state, args.learning_rate, loss_fn, k_train, data_iterator, summary_fn,
          checkpoint_path, args.total_iterations, step, args.summary_period,
          args.checkpoint_period, args.keep_checkpoints)

if __name__ == "__main__":
    main(get_args())


import pymunk

def get_shape(x, y, space):
    # See if we clicked on anything
    shape_list = space.point_query((x, y), 1, pymunk.ShapeFilter())

    # If we did, remember what we clicked on
    if len(shape_list) > 0:
        shape = shape_list[0]
    else:
        shape = None
    return shape

def make_pin_joint_connection(x, y, space, joints, shape_1, shape_2):
    if get_shape(x, y, space) is None:
        return
    if shape_1 is None:
        print("Shape 1 Found")
        shape_1 = get_shape(x, y, space)
    elif shape_2 is None:
        print("Shape 2 Found")
        shape_2 = get_shape(x, y, space)
        joint = pymunk.PinJoint(shape_1.shape.body, shape_2.shape.body)
        space.add(joint)
        joints.append(joint)
        shape_1 = None
        shape_2 = None
        print("Joint Connection Made")
    return shape_1, shape_2

def make_damped_spring_connection(x, y, space, joints, shape_1, shape_2):
    if get_shape(x, y, space) is None:
        return
    if shape_1 is None:
        print("Shape 1 Found")
        shape_1 = get_shape(x, y, space)
    elif shape_2 is None:
        print("Shape 2 Found")
        shape_2 = get_shape(x, y, space)
        joint = pymunk.DampedSpring(shape_1.shape.body, shape_2.shape.body, (0, 0), (0, 0), 45, 300, 30)
        space.add(joint)
        joints.append(joint)
        shape_1 = None
        shape_2 = None
        print("Spring Connection Made")
    return shape_1, shape_2


from django.db import models

class Piece(models.Model):
    name = models.CharField(max_length=30)
    icon = models.CharField(max_length=30)
    color = models.CharField(max_length=30)

    def __str__(self):
        return self.name

class Board(models.Model):
    rows = 8
    columns = ["A", "B", "C", "D", "E", "F", "G", "H"]
    pieces = models.ManyToManyField(Piece)

    @staticmethod
    def create_new():
        board = Board()
        board.save()
        for row in range(1, board.rows + 1):
            for column in board.columns:
                square = BoardSquare(
                    board=board,
                    row=row,
                    column=column
                )
                square.save()
        return board

class BoardSquare(models.Model):
    board = models.ForeignKey(Board, related_name="board", on_delete=models.CASCADE)
    piece = models.ForeignKey(Piece, related_name="piece", on_delete=models.CASCADE,
                                null=True)
    row = models.IntegerField()
    column = models.CharField(max_length=1)

class Player(models.Model):
    name = models.CharField(max_length=30)
    pieces = models.ManyToManyField(Piece)

    def __str__(self):
        return self.name

class Game(models.Model):
    player_one = models.ForeignKey(Player, related_name="player_one", on_delete=models.CASCADE)
    player_two = models.ForeignKey(Player, related_name="player_two", on_delete=models.CASCADE)
    board = models.ForeignKey(Board, related_name="gameboard", on_delete=models.CASCADE)
    current_turn = models.ForeignKey(Player, related_name="current_turn", on_delete=models.CASCADE, default=None)

    @staticmethod
    def create_player_one_pieces():

        pieces = [
            Piece(name='Rook', icon=u'\u265C', color='white'),
            Piece(name='Knight', icon=u'\u265E', color='white'),
            Piece(name='Bishop', icon=u'\u265D', color='white'),
            Piece(name='Queen', icon=u'\u265B', color='white'),
            Piece(name='King', icon=u'\u265A', color='white'),
            Piece(name='Bishop', icon=u'\u265D', color='white'),
            Piece(name='Knight', icon=u'\u265E', color='white'),
            Piece(name='Rook', icon=u'\u265C', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
            Piece(name='Pawn', icon= u'\u265F', color='white'),
        ]
        return pieces

    @staticmethod
    def create_player_two_pieces():
        pieces = [
            Piece(name='Rook', icon=u'\u265C', color='black'),
            Piece(name='Knight', icon=u'\u265E', color='black'),
            Piece(name='Bishop', icon=u'\u265D', color='black'),
            Piece(name='Queen', icon=u'\u265B', color='black'),
            Piece(name='King', icon=u'\u265A', color='black'),
            Piece(name='Bishop', icon=u'\u265D', color='black'),
            Piece(name='Knight', icon=u'\u265E', color='black'),
            Piece(name='Rook', icon=u'\u265C', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
            Piece(name='Pawn', icon= u'\u265F', color='black'),
        ]
        return pieces

    @staticmethod
    def create_new(player_one_name, player_two_name):
        board = Board.create_new()
        player_one_pieces = Game.create_player_one_pieces()
        player1 = Player(
            name=player_one_name,
        )
        player1.save()

        for piece in player_one_pieces:
            piece.save()
            player1.pieces.add(piece)
            board.pieces.add(piece)

        player_two_pieces = Game.create_player_two_pieces()
        player2 = Player(
            name=player_two_name,
        )
        player2.save()
        for piece in player_two_pieces:
            piece.save()
            player2.pieces.add(piece)
            board.pieces.add(piece)

        game = Game(
            player_one = player1,
            player_two = player2,
            board = board,
            current_turn = player1
        )
        game.save()

        first_player_board_pieces = [BoardSquare.objects.filter(board=board.id, row=row, column=col)
                            for row in [1,2] for col in Board.columns]

        # add pieces to board piece
        for row, piece in list(zip(first_player_board_pieces, player_one_pieces)):
            row.update(piece=piece)

        second_player_board_pieces = [BoardSquare.objects.filter(board=board.id, row=row, column=col)
                            for row in [8,7] for col in Board.columns]
        for row, piece in list(zip(second_player_board_pieces, player_two_pieces)):
            row.update(piece=piece)


from functools import partial
import jax.numpy as jnp
from jax.api import jit
from jbdl.rbdl.model import joint_model
from jbdl.rbdl.math import cross_motion_space, cross_force_space
from jbdl.rbdl.utils import xyz2int

@partial(jit, static_argnums=(2, 3, 4, 5))
def forward_dynamics_core(x_tree, inertia, parent, jtype, jaxis, nb, q, qdot, tau, a_grav):
    s = []
    x_up = []
    v = []
    c = []
    p_aa = []
    inertia_aa = inertia.copy()

    for i in range(nb):
        xj, si = joint_model(jtype[i], jaxis[i], q[i])
        s.append(si)
        vj = jnp.multiply(s[i], qdot[i])
        x_up.append(jnp.matmul(xj, x_tree[i]))
        if parent[i] == 0:
            v.append(vj)
            c.append(jnp.zeros((6, 1)))
        else:
            v.append(jnp.add(jnp.matmul(x_up[i], v[parent[i]-1]), vj))
            c.append(jnp.matmul(cross_motion_space(v[i]), vj))
        p_aa.append(jnp.matmul(cross_force_space(v[i]), jnp.matmul(inertia_aa[i], v[i])))

    uu = [jnp.empty((0,))] * nb
    d = [jnp.empty((0,))] * nb
    u = [jnp.empty((0,))] * nb

    for i in range(nb-1, -1, -1):
        uu[i] = jnp.matmul(inertia_aa[i], s[i])
        d[i] = jnp.squeeze(jnp.matmul(s[i].transpose(), uu[i]))
        u[i] = tau[i] - jnp.squeeze(jnp.matmul(s[i].transpose(), p_aa[i]))
        if parent[i] != 0:
            ia = inertia_aa[i] - jnp.matmul(uu[i] / d[i], jnp.transpose(uu[i]))
            pa = p_aa[i] + jnp.matmul(ia, c[i]) + jnp.multiply(uu[i], u[i]) / d[i]
            inertia_aa[parent[i] - 1] = inertia_aa[parent[i] - 1] \
                + jnp.matmul(jnp.matmul(jnp.transpose(x_up[i]), ia), x_up[i])
            p_aa[parent[i] - 1] = p_aa[parent[i] - 1] + jnp.matmul(jnp.transpose(x_up[i]), pa)

    a = []
    qddot = []

    for i in range(nb):
        if parent[i] == 0:
            a.append(jnp.matmul(x_up[i], -a_grav) + c[i])
        else:
            a.append(jnp.matmul(x_up[i], a[parent[i] - 1]) + c[i])

        qddot.append((u[i] - jnp.squeeze(jnp.matmul(jnp.transpose(uu[i]), a[i])))/d[i])
        a[i] = a[i] + jnp.multiply(s[i], qddot[i])

    qddot = jnp.reshape(jnp.stack(qddot), (nb, ))
    return qddot

def forward_dynamics(model, q, qdot, tau):
    q = q.flatten()
    qdot = qdot.flatten()
    tau = tau.flatten()
    a_grav = model["a_grav"]
    nb = model["nb"]
    jtype = tuple(model["jtype"])
    jaxis = xyz2int(model["jaxis"])
    parent = tuple(model["parent"])
    x_tree = model["x_tree"]
    inertia = model["inertia"]

    qddot = forward_dynamics_core(x_tree, inertia, tuple(parent), tuple(jtype), jaxis, nb, q, qdot, tau, a_grav)
    return qddot


"""
Face detection using dlib frontal face detector, which is based on Histogram of Oriented Gradients (HOG) features
and a linear classifier in a sliding window detection approach
"""

# Import required packages:
import cv2
import dlib
from matplotlib import pyplot as plt

def show_img_with_matplotlib(color_img, title, pos):
    """Shows an image using matplotlib capabilities"""

    img_RGB = color_img[:, :, ::-1]

    ax = plt.subplot(1, 2, pos)
    plt.imshow(img_RGB)
    plt.title(title)
    plt.axis('off')

def show_detection(image, faces):
    """Draws a rectangle over each detected face"""

    for face in faces:
        cv2.rectangle(image, (face.left(), face.top()), (face.right(), face.bottom()), (255, 0, 0), 10)
    return image

# Load image and convert to grayscale:
img = cv2.imread("test_face_detection.jpg")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# Load frontal face detector from dlib:
detector = dlib.get_frontal_face_detector()

# Detect faces:
rects_1 = detector(gray, 0)
rects_2 = detector(gray, 1)

# Draw face detections:
img_faces_1 = show_detection(img.copy(), rects_1)
img_faces_2 = show_detection(img.copy(), rects_2)

# Create the dimensions of the figure and set title:
fig = plt.figure(figsize=(10, 4))
plt.suptitle("Face detection using dlib frontal face detector", fontsize=14, fontweight='bold')
fig.patch.set_facecolor('silver')

# Plot the images:
show_img_with_matplotlib(img_faces_1, "detector(gray, 0): " + str(len(rects_1)), 1)
show_img_with_matplotlib(img_faces_2, "detector(gray, 1): " + str(len(rects_2)), 2)

# Show the Figure:
plt.show()


# --- Day 13: Distress Signal ---
from functools import cmp_to_key

def read_puzzle(file):
    return [[eval(x) for x in line.split("\n")] for line in open(file).read().strip().split("\n\n")]

def compare(a, b): # (result, fuzzy) : fuzzy == True -> compare_result is unknown, else: compare_result = result
    aIsList = isinstance(a, list)
    bIsList = isinstance(b, list)
    if not aIsList and not bIsList : return (a < b, a == b)
    elif aIsList != bIsList:
        if not aIsList: a = [a]
        if not bIsList: b = [b]
        return compare(a, b)
    else:
        for az, bz in zip(a, b):
            result, isUndef = compare(az, bz)
            if not isUndef: return result, False
        return len(a) < len(b), len(a) == len(b)

def comp(a, b):
    return compare(a, b)[0]

def solve1(puzzle):
    return sum([(i + 1) * compare(item[0], item[1])[0] for i, item in enumerate(puzzle)])

def solve2(puzzle):
    p = []
    for items in puzzle:
        a, b = items
        p.append(a)
        p.append(b)

    addList = [[2], [6]]
    p.append(addList[0])
    p.append(addList[1])

    p.sort(key = cmp_to_key(comp))

    i1 = p.index(addList[0]) + 1
    i2 = p.index(addList[1]) + 1

    return i1 * i2

puzzle = read_puzzle('d13.txt')
print("Task 1", solve1(puzzle))
print("Task 2", solve2(puzzle))


import queue
import threading

def preload_generator(g, early_quit_flag=None, queue_size=10):
    '''
    一个预加载缓存队列，会自动预载数据
    :param g:
    :return:
    '''
    # 提前退出标志
    if early_quit_flag is None:
        early_quit_flag = [False]

    # 预载入线程
    def _run(g, q: queue.Queue, no_more_flag, early_quit_flag):
        for a in g:
            if early_quit_flag[0]:
                break
            q.put(a, block=True, timeout=None)
        no_more_flag[0] = True

    # 数据载入结束标志
    no_more_flag = [False]
    q = queue.Queue(maxsize=queue_size)
    th = threading.Thread(target=_run, args=(g, q, no_more_flag, early_quit_flag), daemon=True)
    th.start()
    while True:
        try:
            a = q.get(block=True, timeout=2)
            yield a
        except queue.Empty:
            if no_more_flag[0] and q.empty():
                break

if __name__ == '__main__':

    '''
    测试提前退出标志。当然可以不管，因为设定了守护线程，主线程退出时，守护线程也会退出，或者生成器被删除时，守护线程会自动异常退出
    流程：
    构造一个无限数字生成器
    使用预加载生成器修饰该生成器
    设定提前退出标志
    循环首
    当数字等于100时，设定提前退出标志
    循环尾
    程序退出
    '''

    def num_gen():
        i = 0
        while True:
            i += 1
            yield i

    g = num_gen()
    early_exit_flag = [False]
    g = preload_generator(g, early_exit_flag)

    for i in g:
        if i == 100:
            early_exit_flag[0] = True
        print(i)


#!/usr/bin/env python
"""Exploratory Data Analysis
"""

from lists import REFINED_DIC, CORE_DIC
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestRegressor
from sklearn.cross_validation import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.externals import joblib
from sklearn.metrics import mean_squared_error
from scipy.stats import pearsonr
from myreduce import Dists15, Dists15Randomized, Dists15ShuffleLig

import aff_2015
import json
import luigi
import numpy as np
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def distancesProfile():
    """Explore the distances profile
    """
    ifn = "/work/jaydy/working/PDBbind_refined07-core07.distances.json"
    with open(ifn) as ifs:
        dat = json.loads(ifs.read())

    # minimum distances
    max_dists = []
    min_dists = []
    mean_dists = []
    all_dists = []
    ligand_sizes_4_res = []
    for key, val in dat.iteritems():
        df = pd.DataFrame(val)
        min_dists.extend(df.dists.map(min))
        max_dists.extend(df.dists.map(max))
        mean_dists.extend(df.dists.map(np.mean))
        ligand_sizes_4_res.extend(df.dists.map(
            lambda dists: max(dists) - min(dists)))
        all_dists.extend(df.dists.values.tolist())

    all_dists = [d for l in all_dists for d in l]
    all_dists = pd.Series(all_dists)
    max_dists = pd.Series(max_dists)
    min_dists = pd.Series(min_dists)
    mean_dists = pd.Series(mean_dists)
    lig_sizes = pd.Series(ligand_sizes_4_res)

    plt.figure()
    all_dists.hist(bins=30)
    plt.xlabel("Distances [$\mathrm{\AA}$]")
    plt.savefig("/ddnB/work/jaydy/working/pdbbind/all_dists_hist.tiff")

    print("10% quatile of all distances: {}".format(all_dists.quantile(0.1)))

    plt.figure()
    lig_sizes.hist(bins=20)
    plt.xlabel("Ligand sizes [$\mathrm{\AA}$]")
    plt.savefig("/ddnB/work/jaydy/working/pdbbind/lig_sizes_hist.tiff")

    plt.figure()
    min_dists.hist(bins=30)
    plt.xlabel("Distances [$\mathrm{\AA}$]")
    plt.savefig("/ddnB/work/jaydy/working/pdbbind/min_dists.tiff")

    plt.figure()
    max_dists.hist(bins=30)
    plt.xlabel("Distances [$\mathrm{\AA}$]")
    plt.savefig("/ddnB/work/jaydy/working/pdbbind/max_dists.tiff")

    plt.figure()
    mean_dists.hist(bins=30)
    plt.xlabel("Distances [$\mathrm{\AA}$]")
    plt.savefig("/ddnB/work/jaydy/working/pdbbind/mean_dists.tiff")

class Tokens(luigi.Task):
    binning_size = luigi.Parameter(default=5.0)

    def output(self):
        ofns = [
            "/ddnB/work/jaydy/working/pdbbind/refined.{}.csv".format(
                self.binning_size),
            "/ddnB/work/jaydy/working/pdbbind/core.{}.csv".format(
                self.binning_size)
        ]

        return [luigi.LocalTarget(ofn) for ofn in ofns]

    def getTokens(self, profiles):
        df = pd.DataFrame(profiles)

        lig_span = df.dists.map(lambda dists: max(dists) - min(dists)).mean()

        tokens = []
        for profile in profiles:
            # ignore water molecule or residues too far from the ligand
            if profile["residue"] == "HOH":
                pass
            elif min(profile["dists"]) > 13.08:
                tokens.append(profile["residue"])
            else:
                ends = np.arange(13.08, 13.08 + lig_span, self.binning_size)
                type_dists = sorted(
                    zip(profile["atom_types"], profile["dists"]),
                    key=lambda x: x[1])

                token = profile["residue"]
                for end in ends:
                    my_type_dists = [
                        t
                        for t in type_dists
                        if t[1] > end and t[1] < (end + self.binning_size)
                    ]
                    if len(my_type_dists) > 0:
                        token = token + "-" + my_type_dists[0][0]
                tokens.append(token)

        return ' '.join(tokens)

    def run(self):
        ifn = "/work/jaydy/working/PDBbind_refined07-core07.distances.json"
        with open(ifn) as ifs:
            dat = json.loads(ifs.read())

        tokens = [self.getTokens(p) for p in dat.values()]
        myid_tokens = dict(zip(dat.keys(), tokens))

        refined_dat = [(myid, myid_tokens[myid], REFINED_DIC[myid])
                       for myid in dat.keys() if myid in REFINED_DIC]
        core_dat = [(myid, myid_tokens[myid], CORE_DIC[myid])
                    for myid in dat.keys() if myid in CORE_DIC]

        ofns = [output.path for output in self.output()]
        cols = ['myid', 'tokens', 'ki']
        pd.DataFrame(refined_dat, columns=cols).to_csv(ofns[0])
        pd.DataFrame(core_dat, columns=cols).to_csv(ofns[1])

class Tokens15(Tokens):
    def output(self):
        ofns = [
            "/ddnB/work/jaydy/working/pdbbind/refined.15.{}.csv".format(
                self.binning_size),
            "/ddnB/work/jaydy/working/pdbbind/core.15.{}.csv".format(
                self.binning_size)
        ]

        return [luigi.LocalTarget(ofn) for ofn in ofns]

    def requires(self):
        return Dists15()

    def run(self):
        ifn = self.requires().output().path

        with open(ifn) as ifs:
            dat = json.loads(ifs.read())

        tokens = [self.getTokens(p) for p in dat.values()]
        myid_tokens = dict(zip(dat.keys(), tokens))

        refined_dat = [(myid, myid_tokens[myid], aff_2015.refined_dat[myid])
                       for myid in dat.keys() if myid in aff_2015.refined_dat]
        core_dat = [(myid, myid_tokens[myid], aff_2015.core_dat[myid])
                    for myid in dat.keys() if myid in aff_2015.core_dat]

        ofns = [output.path for output in self.output()]
        cols = ['myid', 'tokens', 'ki']
        pd.DataFrame(refined_dat, columns=cols).to_csv(ofns[0])
        pd.DataFrame(core_dat, columns=cols).to_csv(ofns[1])

class Tokens15Randomized(Tokens):
    def output(self):
        ofns = [
            "/ddnB/work/jaydy/working/pdbbind/refined.15.rnd.{}.ki.csv".format(
                self.binning_size),
            "/ddnB/work/jaydy/working/pdbbind/refined.15.rnd.{}.kd.csv".format(
                self.binning_size),
        ]
        return [luigi.LocalTarget(ofn) for ofn in ofns]

    def requires(self):
        return Dists15Randomized()

    def run(self):
        ifn = self.requires().output().path

        with open(ifn) as ifs:
            dat = json.loads(ifs.read())

        tokens = [self.getTokens(p) for p in dat.values()]
        myid_tokens = dict(zip(dat.keys(), tokens))

        kds = set(aff_2015.Kds)
        kis = set(aff_2015.Kis)

        kds_dat = [(myid, myid_tokens[myid], aff_2015.refined_dat[myid])
                   for myid in dat.keys() if myid in kds]
        kis_dat = [(myid, myid_tokens[myid], aff_2015.refined_dat[myid])
                   for myid in dat.keys() if myid in kis]

        ofns = [output.path for output in self.output()]
        cols = ['myid', 'tokens', 'ki']
        pd.DataFrame(kis_dat, columns=cols).to_csv(ofns[0])
        pd.DataFrame(kds_dat, columns=cols).to_csv(ofns[1])

class Tokens15ShuffleLig(Tokens):
    def output(self):
        ofns = [
            "/ddnB/work/jaydy/working/pdbbind/refined.15.shuffled.{}.ki.csv".format(
                self.binning_size),
            "/ddnB/work/jaydy/working/pdbbind/refined.15.shuffled.{}.kd.csv".format(
                self.binning_size),
        ]
        return [luigi.LocalTarget(ofn) for ofn in ofns]

    def requires(self):
        return Dists15ShuffleLig()

class RF(luigi.Task):
    """try with the Random Forest algorithm
    """
    binning_size = luigi.Parameter(default=5.0)

    def requires(self):
        return Tokens(binning_size=self.binning_size)

    def read(self):
        task = self.requires()
        if not task.complete():
            raise Exception("{} not completed".format(task))

        refined_ifn, core_ifn = [_.path for _ in task.output()]

        refined_df = pd.read_csv(refined_ifn, index_col=0)
        core_df = pd.read_csv(core_ifn, index_col=0)

        return refined_df, core_df

    def split(self):
        refined_df, core_df = self.read()
        refined_df = refined_df[~refined_df['myid'].isin(core_df['myid'])]
        return refined_df, core_df

    def run(self):
        refined_df, core_df = self.split()
        tokens = refined_df.tokens.map(lambda x: x.split()).values
        unique_tokens = set([t for l in tokens for t in l])
        print("{} unique tokens".format(len(unique_tokens)))

        print("BinningSize MaxDf MinDf RMSE Corr")
        for max_df in [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
            for min_df in [0.0, 0.1, 0.2]:
                pipe_line = Pipeline([
                    ('tfidf', TfidfVectorizer(max_df=max_df,
                                              min_df=min_df,
                                              lowercase=False,
                                              token_pattern=r'(?u)\b\S+\b',
                                              analyzer='word')
                     ), ('model', RandomForestRegressor(n_estimators=50,
                                                        n_jobs=16))
                ])

                pipe_line.fit(refined_df['tokens'], refined_df['ki'])
                prediction = pipe_line.predict(core_df['tokens'])
                score = mean_squared_error(core_df['ki'], prediction)
                corr = pearsonr(core_df['ki'], prediction)[0]
                print("{} {} {} {} {}".format(self.binning_size, max_df,
                                              min_df, score, corr))

    def output(self):
        pass

def readDfireScores():
    """read the scores of Dfire on the PDBBind 2014 refined dataset
    """
    ifn = "../dat/pdbbind2014-dfire"
    df = pd.read_csv(ifn, index_col=0)
    return df

class RF15(RF):
    def requires(self):
        return Tokens15(binning_size=self.binning_size)

class RF15AgainstDfire(RF15):
    def output(self):
        ki_ofn = "/work/jaydy/working/pdbbind/PDBBind_refined_15/rf.ki.pkl"
        kd_ofn = "/work/jaydy/working/pdbbind/PDBBind_refined_15/rf.kd.pkl"
        return [luigi.LocalTarget(ki_ofn), luigi.LocalTarget(kd_ofn)]

    def run(self):
        refined_df, core_df = self.split()
        dfire_df = readDfireScores()
        merged = pd.merge(refined_df,
                          dfire_df,
                          left_on='myid',
                          right_on='pdbid')

        def train(df, target='ki'):
            y = df['ki']
            columns = df.columns.tolist()
            columns.remove('ki')
            X = df[columns]

            X_train, X_test, y_train, y_test = train_test_split(
                X, y, test_size=0.5)

            pipe_line = Pipeline([
                ('tfidf', TfidfVectorizer(max_df=1.0,
                                          min_df=0.0,
                                          lowercase=False,
                                          token_pattern=r'(?u)\b\S+\b',
                                          analyzer='word')
                 ), ('model', RandomForestRegressor(n_estimators=50,
                                                    n_jobs=1))
            ])

            pipe_line.fit(X_train['tokens'], y_train)
            ofn = "/work/jaydy/working/pdbbind/PDBBind_refined_15/rf.{}.pkl".format(
                target)
            joblib.dump(pipe_line, ofn)

            prediction = pipe_line.predict(X_test['tokens'])
            corr = pearsonr(y_test, prediction)

            dfire_corr = pearsonr(y_test, X_test['dfire'])
            uncorr_dfire_corr = pearsonr(y_test, X_test['uncor_dfire'])

            print("rf correlation: {}".format(corr))
            print("dfire correlation: {}".format(dfire_corr))
            print("uncorrelated-dfire correlation: {}".format(
                uncorr_dfire_corr))

        print("Ki")
        ki_df = merged[merged['myid'].isin(aff_2015.Kis)]
        train(ki_df, target='ki')

        print("Kd")
        kd_df = merged[merged['myid'].isin(aff_2015.Kds)]
        train(kd_df, target='kd')

class Test15Randomized(luigi.Task):
    def requires(self):
        return [Tokens15Randomized(binning_size=7.0),
                RF15AgainstDfire(binning_size=7.0)]

    def run(self):
        tokens_task, model_task = self.requires()

        ki_tokens_task, kd_tokens_task = tokens_task.output()
        ki_model_task, kd_model_task = model_task.output()

        ki_df = pd.read_csv(ki_tokens_task.path, index_col=0)
        ki_model = joblib.load(ki_model_task.path)
        ki_prediction = ki_model.predict(ki_df['tokens'])
        ki_corr = pearsonr(ki_prediction, ki_df['ki'])
        print("Ki randomized test:")
        print(ki_corr)

        kd_df = pd.read_csv(kd_tokens_task.path, index_col=0)
        kd_model = joblib.load(kd_model_task.path)
        kd_prediction = kd_model.predict(kd_df['tokens'])
        kd_corr = pearsonr(kd_prediction, kd_df['ki'])
        print("Kd randomized test:")
        print(kd_corr)

def main():
    luigi.build(
        [
            # RF(binning_size=8.0),
            # RF(binning_size=7.0),
            # RF(binning_size=6.0),
            # RF(binning_size=5.0),

            # RF15(binning_size=8.0),
            # RF15(binning_size=7.0),
            # RF15(binning_size=6.0),
            # RF15(binning_size=5.0),
            # RF15AgainstDfire(binning_size=7.0),
            # Tokens15Randomized(binning_size=7.0),
            # Tokens15ShuffleLig(binning_size=7.0),
            # Test15Randomized(),
        ],
        local_scheduler=True)

if __name__ == '__main__':
    main()


import networks
import torch
import torch.nn as nn

class CDCGAN(nn.Module):
    def __init__(self, opts):
        super(CDCGAN, self).__init__()
        # parameters
        lr = 0.0002
        self.nz = opts.nz
        self.class_num = opts.class_num
        self.G = networks.generator(opts)
        self.D = networks.discriminator(opts)

        self.gen_opt = torch.optim.Adam(self.G.parameters(), lr=lr, betas=(0.5, 0.999), weight_decay=0.0001)
        self.dis_opt = torch.optim.Adam(self.D.parameters(), lr=lr, betas=(0.5, 0.999), weight_decay=0.0001)

        self.BCE_loss = torch.nn.BCELoss()

    def initialize(self):
        self.G.weight_init()
        self.D.weight_init()

    def setgpu(self, gpu):
        self.gpu = gpu
        self.D.cuda(self.gpu)
        self.G.cuda(self.gpu)

    def get_z_random(self, batchSize, nz):
        z = torch.cuda.FloatTensor(batchSize, nz)
        z.copy_(torch.randn(batchSize, nz))
        return z

    def onehot_encoding(self, label):
        onehot = torch.zeros(self.class_num, self.class_num)
        index = torch.zeros([self.class_num, 1], dtype= torch.int64)
        for i in range(self.class_num):
            index[i] = i

        onehot = onehot.scatter_(1, index, 1).view(self.class_num, self.class_num, 1, 1)
        label_one_hot = onehot[label]
        return label_one_hot.cuda(self.gpu).detach()

    def forward(self):
        self.label_one_hot = self.onehot_encoding(self.label)
        self.z_random = self.get_z_random(self.real_image.size(0), self.nz)
        self.z_random2 = self.get_z_random(self.real_image.size(0), self.nz)
        z_conc = torch.cat((self.z_random, self.z_random2), 0)
        label_conc = torch.cat((self.label_one_hot, self.label_one_hot),0)
        self.fake_image=self.G.forward(z_conc, label_conc)
        self.fake_image1, self.fake_image2 = torch.split(self.fake_image, self.z_random.size(0), dim=0)
        self.image_display = torch.cat((self.real_image.detach().cpu(), self.fake_image1.cpu(), self.fake_image2.cpu()), dim=2)

    def update_D(self, image, label):
        self.real_image = image
        self.label = label
        self.forward()

        # update discriminator
        self.dis_opt.zero_grad()
        self.loss_D = self.backward_D(self.D, self.real_image, self.fake_image1, self.label_one_hot)+ \
                       self.backward_D(self.D, self.real_image, self.fake_image2, self.label_one_hot)
        self.loss_D.backward()
        self.dis_opt.step()

    def update_G(self):
        self.gen_opt.zero_grad()
        self.loss_G_GAN = self.backward_G(self.D, self.fake_image1, self.label_one_hot)+ \
                        self.backward_G(self.D, self.fake_image2, self.label_one_hot)

        # mode seeking loss
        lz = torch.mean(torch.abs(self.fake_image2 - self.fake_image1)) / torch.mean(
            torch.abs(self.z_random2 - self.z_random))
        eps = 1 * 1e-5
        self.loss_lz = 1 / (lz + eps)

        self.loss_G = self.loss_G_GAN + self.loss_lz
        self.loss_G.backward()
        self.gen_opt.step()

    def backward_D(self, netD, real, fake, label):
        pred_fake = netD.forward(fake.detach(), label)
        pred_real = netD.forward(real, label)
        all0 = torch.zeros_like(pred_fake).cuda(self.gpu)
        all1 = torch.ones_like(pred_real).cuda(self.gpu)
        ad_fake_loss = nn.functional.binary_cross_entropy(pred_fake, all0)
        ad_true_loss = nn.functional.binary_cross_entropy(pred_real, all1)
        loss_D = ad_true_loss + ad_fake_loss
        return loss_D

    def backward_G(self, netD, fake, label):
        pred_fake = netD.forward(fake, label)
        all_ones = torch.ones_like(pred_fake).cuda(self.gpu)
        loss_G = nn.functional.binary_cross_entropy(pred_fake, all_ones)
        return loss_G

    def update_lr(self):
        self.dis_sch.step()
        self.gen_sch.step()

    def resume(self, model_dir, train=True):
        checkpoint = torch.load(model_dir)
        # weight
        if train:
            self.D.load_state_dict(checkpoint['dis'])
        self.G.load_state_dict(checkpoint['gen'])
        # optimizer
        if train:
            self.dis_opt.load_state_dict(checkpoint['dis_opt'])
            self.gen_opt.load_state_dict(checkpoint['gen_opt'])
        return checkpoint['ep'], checkpoint['total_it']

    def save(self, filename, ep, total_it):
        state = {
                'dis': self.D.state_dict(),
                'gen': self.G.state_dict(),
                'dis_opt': self.dis_opt.state_dict(),
                'gen_opt': self.gen_opt.state_dict(),
                'ep': ep,
                'total_it': total_it
                    }
        torch.save(state, filename)
        return

    def assemble_outputs(self):
        image_real = self.normalize_image(self.real_image).detach()
        image_fake1 = self.normalize_image(self.fake_image1).detach()
        image_fake2 = self.normalize_image(self.fake_image2).detach()
        return torch.cat((image_real,image_fake1,image_fake2),2)

    def normalize_image(self, x):
        return x[:,0:3,:,:]

    def test_forward(self, label):
        label_one_hot = self.onehot_encoding(label)
        z_random = self.get_z_random(label.size(0), self.nz)
        outputs = self.G.forward(z_random, label_one_hot)
        return  outputs


#!/usr/bin/python3
import glob
import os
import sys
import subprocess
import random

OINK_PATH = os.getenv("OINK_PATH", default="../oink/build/oink")
RNG_PATH = os.getenv("RNG_PATH", default="../oink/build/rngame")
TEST_DIR = os.getenv("TEST_DIR", default="./inputs/tests")
EXEC_PATH = os.getenv("EXEC_PATH", default="./target/release/lmc")

def generate_game(file, size):
    sh(f"{RNG_PATH} {size} {size + random.randint(0, 1)} 1 5 > {file}")

def sh(cmd):
    ps = subprocess.Popen(cmd,
                          shell=True,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.DEVNULL)
    return {"text": ps.communicate()[0], "status": ps.returncode}

def test_generic(file, algorithm):
    lmc_regions = sh(
        f"{EXEC_PATH} parity --algorithm {algorithm} --regions --target game.sol {file}"
    )["text"]
    oink_regions = sh(
        f"{OINK_PATH} -p --no {file} | grep -o -E 'won by.*'")["text"]

    if lmc_regions != oink_regions:
        raise AssertionError(
            f"winning regions differ:\n\toink: {oink_regions}\n\tlmc:  {lmc_regions}"
        )

    oink_verify = sh(f"{OINK_PATH} -v {file} --sol game.sol")
    sh("rm game.sol")

    if oink_verify["status"] != 0:
        raise AssertionError(f"oink could not verify solution")

def test_fpi(file):
    test_generic(file, "fpi")

def test_zielonka(file):
    test_generic(file, "zielonka")

def test_tangle(file):
    test_generic(file, "tangle")

def test_spm(file):
    test_generic(file, "spm")

def test_file(file, tangle=True, fpi=True, spm=True, zielonka=True):
    fpi_result = "OK "
    try:
        if fpi:
            test_fpi(file)
        else:
            fpi_result = "---"
    except AssertionError as error:
        print(f"{file} {error}")
        fpi_result = "ERR"

    zielonka_result = "OK "
    try:
        if zielonka:
            test_zielonka(file)
        else:
            zielonka_result = "---"
    except AssertionError as error:
        print(f"{file} {error}")
        zielonka_result = "ERR"

    tangle_result = "OK "
    try:
        if tangle:
            test_tangle(file)
        else:
            tangle_result = "---"
    except AssertionError as error:
        print(f"{file} {error}")
        tangle_result = "ERR"

    spm_result = "OK "
    try:
        if spm:
            test_spm(file)
        else:
            spm_result = "---"
    except AssertionError as error:
        print(f"{file} {error}")
        spm_result = "ERR"

    print("file {}: fpi {}  zlk {}  tgl {}  spm {}".format(
        file, fpi_result, zielonka_result, tangle_result, spm_result))

if __name__ == "__main__":
    print(f"compiling executable")
    sh("cargo build --release")

    if not os.path.exists(EXEC_PATH):
        print(f"ERR could not find executable {EXEC_PATH}")
        print(
            "either you are not in the project directory or the EXEC_PATH environment variable is not set"
        )
        sys.exit(1)

    if not os.path.exists(OINK_PATH):
        print(f"ERR could not find oink executable {OINK_PATH}")
        print("set the environment variable OINK_PATH")
        sys.exit(1)

    if not os.path.isdir(TEST_DIR):
        print(f"ERR could not find test directory {TEST_DIR}")
        print("set the environment variable TEST_DIR")
        sys.exit(1)

    if not os.path.exists(RNG_PATH):
        print(f"ERR could not find rngame executable {RNG_PATH}")
        print(
            "set the environment variable RNG_PATH or recompile oink with -DOINK_BUILD_EXTRA_TOOLS"
        )
        sys.exit(1)

    for file in sorted(glob.glob(f"{TEST_DIR}/*")):
        test_file(file)

    number_rng_games = 100
    rng_game_size = 150
    print(f"testing {number_rng_games} random large parity games")

    for i in range(0, number_rng_games):
        generate_game(f"tmp_{i}", rng_game_size)
        test_file(f"tmp_{i}", spm=False)
        sh(f"rm tmp_{i}")