Updated early stopping / gradient clipping for NaNs in loss
@@ -49,7 +49,7 @@ class Trainer(object):
validation_split=0.25,
early_stopping=True,
early_stopping_patience=None,
- clip_gradient=100.0,
+ clip_gradient=10.0,
verbose="some",
):
logger.debug("Initialising training data")
@@ -303,6 +303,9 @@ class Trainer(object):
if early_stopping_patience is not None and i_epoch - best_epoch > early_stopping_patience >= 0:
raise EarlyStoppingException
+ if loss is None or not np.isfinite(loss):
+ raise EarlyStoppingException
+
return best_loss, best_model, best_epoch
@staticmethod
@@ -331,9 +334,9 @@ class Trainer(object):
logging_fn(val_report)
def wrap_up_early_stopping(self, best_model, currrent_loss, best_loss, best_epoch):
- if currrent_loss is None or best_loss is None:
+ if best_loss is None or not np.isfinite(best_loss):
logger.warning("Loss is None, cannot wrap up early stopping")
- elif best_loss < currrent_loss:
+ elif currrent_loss is None or not np.isfinite(currrent_loss) or best_loss < currrent_loss:
logger.info(
"Early stopping after epoch %s, with loss %8.5f compared to final loss %8.5f",
best_epoch + 1,
