Add logging every 30s for evaluate
""" Helper functions to evaluate a model on a dataset """
import os
+import time
import logging as log
import json
@@ -13,6 +14,8 @@ from . import preprocess
from typing import List, Sequence, Iterable, Tuple, Dict
+LOG_INTERVAL = 30
+
def _coerce_list(preds) -> List:
if isinstance(preds, torch.Tensor):
return preds.data.tolist()
@@ -40,13 +43,14 @@ def evaluate(model, tasks: Sequence[tasks_module.Task], batch_size: int,
all_preds = {}
n_examples_overall = 0
for task in tasks:
- log.info("Evaluating on: %s", task.name)
+ log.info("Evaluating on: %s, split: %s", task.name, split)
+ last_log = time.time()
n_examples = 0
task_preds = [] # accumulate DataFrames
assert split in ["train", "val", "test"]
dataset = getattr(task, "%s_data" % split)
generator = iterator(dataset, num_epochs=1, shuffle=False, cuda_device=cuda_device)
- for batch in generator:
+ for batch_idx, batch in enumerate(generator):
out = model.forward(task, batch, predict=True)
# We don't want mnli-diagnostic to affect the micro and macro average.
@@ -65,9 +69,15 @@ def evaluate(model, tasks: Sequence[tasks_module.Task], batch_size: int,
for field in FIELDS_TO_EXPORT:
if field in batch:
cols[field] = _coerce_list(batch[field])
+
# Transpose data using Pandas
df = pd.DataFrame(cols)
task_preds.append(df)
+
+ if time.time() - last_log > LOG_INTERVAL:
+ log.info("\tTask %s: batch %d", task.name, batch_idx)
+ last_log = time.time()
+
# task_preds will be a DataFrame with columns
# ['preds'] + FIELDS_TO_EXPORT
# for GLUE tasks, preds entries should be single scalars.
@@ -91,6 +101,7 @@ def evaluate(model, tasks: Sequence[tasks_module.Task], batch_size: int,
log.info("Task '%s': sorting predictions by 'idx'", task.name)
task_preds.sort_values(by=['idx'], inplace=True)
all_preds[task.name] = task_preds
+ log.info("Finished evaluating on: %s", task.name)
all_metrics["micro_avg"] /= n_examples_overall
all_metrics["macro_avg"] /= len(tasks)
