report ROC-AUC value
@@ -14,7 +14,7 @@ except ImportError:
import argparse
import chainer
-from chainer import functions as F
+from chainer import functions as F, reporter, cuda
from chainer import iterators as I
from chainer import links as L
from chainer import optimizers as O
@@ -102,9 +102,43 @@ def main():
raise ValueError('Invalid iterator type {}'.format(iterator_type))
val_iter = I.SerialIterator(val, args.batchsize,
repeat=False, shuffle=False)
+
+ if 0:
+ accfun = F.binary_accuracy
+ else:
+ from sklearn import metrics
+
+ def get_1d_numpy_array(v):
+ if isinstance(v, chainer.Variable):
+ v = v.data
+ return cuda.to_cpu(v).ravel()
+
+ def accfun(y, t):
+ # -- calc & report ROC-AUC ---
+ # note that this is dirty hack implementation.
+ # roc auc is calculated per minibatch, and mean is taken to show
+ # PrintReport. This calculation is not same as total batch roc auc
+ # calculation.
+ t_data = get_1d_numpy_array(t)
+ y_data = get_1d_numpy_array(y)
+
+ y_data = y_data[t_data != -1]
+ t_data = t_data[t_data != -1]
+ try:
+ roc_auc = metrics.roc_auc_score(t_data, y_data)
+ reporter.report({'roc_auc': roc_auc}, classifier)
+ except ValueError as e:
+ # When `t_data` only contains one label (ex. only 0), roc auc
+ # cannot be calculated and ValueError is raised.
+ # This implementation just ignores this minibatch for roc auc
+ # calculation.
+ pass
+ # --- calc ROC-AUC end ---
+ return F.binary_accuracy(y, t)
+
classifier = L.Classifier(predictor_,
lossfun=F.sigmoid_cross_entropy,
- accfun=F.binary_accuracy)
+ accfun=accfun)
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
classifier.to_gpu()
@@ -120,10 +154,15 @@ def main():
device=args.gpu, converter=concat_mols))
trainer.extend(E.snapshot(), trigger=(args.epoch, 'epoch'))
trainer.extend(E.LogReport())
- trainer.extend(E.PrintReport(['epoch', 'main/loss', 'main/accuracy',
- 'validation/main/loss',
- 'validation/main/accuracy',
- 'elapsed_time']))
+ if 0:
+ trainer.extend(E.PrintReport([
+ 'epoch', 'main/loss', 'main/accuracy', 'validation/main/loss',
+ 'validation/main/accuracy', 'elapsed_time']))
+ else:
+ trainer.extend(E.PrintReport([
+ 'epoch', 'main/loss', 'main/accuracy', 'main/roc_auc',
+ 'validation/main/loss', 'validation/main/accuracy',
+ 'validation/main/roc_auc', 'elapsed_time']))
trainer.extend(E.ProgressBar(update_interval=10))
trainer.run()
