Add timing per-epoch and time remaining estimate.
@@ -364,7 +364,9 @@ class Trainer:
logger.info("Beginning training.")
+ training_start_time = time.time()
for epoch in range(epoch_counter, self._num_epochs):
+ epoch_start_time = time.time()
train_metrics = self._train_epoch(epoch)
if self._validation_dataset is not None:
@@ -394,6 +396,16 @@ class Trainer:
self._metrics_to_console(train_metrics, val_metrics)
self._update_learning_rate(epoch, val_metric=this_epoch_val_metric)
+ epoch_elapsed_time = time.time() - epoch_start_time
+ logger.info("Epoch duration: " + time.strftime("%H:%M:%S", time.gmtime(epoch_elapsed_time)))
+
+ if epoch < self._num_epochs - 1:
+ training_elapsed_time = time.time() - training_start_time
+ estimated_time_remaining = training_elapsed_time * \
+ ((self._num_epochs - epoch_counter) / float(epoch - epoch_counter + 1) - 1)
+ formatted_time = time.strftime("%H:%M:%S", time.gmtime(estimated_time_remaining))
+ logger.info("Estimated training time remaining: " + formatted_time)
+
def _forward(self, batch: dict, for_training: bool) -> dict:
tensor_batch = arrays_to_variables(batch, self._cuda_device, for_training=for_training)
return self._model.forward(**tensor_batch)
@@ -431,7 +443,7 @@ class Trainer:
"training_state_epoch_{}.th".format(epoch)))
if is_best:
logger.info("Best validation performance so far. "
- "Copying weights to %s/best.th'.", self._serialization_dir)
+ "Copying weights to '%s/best.th'.", self._serialization_dir)
shutil.copyfile(model_path, os.path.join(self._serialization_dir, "best.th"))
def _restore_checkpoint(self) -> Tuple[int, List[float]]:
