Show progressbar during training
* [lmnet] Show progress bar with ETA instead of printing step count
Also remove a few print() calls within the training loop
* remove extra space
* progbar now reaches to the end
previously it only goes up to max_steps - 1
@@ -19,6 +19,7 @@ import math
import click
import tensorflow as tf
from tensorflow.core.util.event_pb2 import SessionLog
+from tensorflow.keras.utils import Progbar
from lmnet.utils import executor, module_loader, config as config_util
from lmnet import environment
@@ -32,7 +33,6 @@ def _save_checkpoint(saver, sess, global_step, step):
os.path.join(environment.CHECKPOINTS_DIR, checkpoint_file),
global_step=global_step,
)
- print("Save ckpt. step: {}.".format(step + 1))
def setup_dataset(config, subset, rank):
@@ -211,11 +211,10 @@ def start_training(config):
max_steps = int(train_dataset.num_per_epoch / config.BATCH_SIZE * config.MAX_EPOCHS)
else:
max_steps = config.MAX_STEPS
- print("max_steps: {}".format(max_steps))
+ progbar = Progbar(max_steps)
+ progbar.update(last_step)
for step in range(last_step, max_steps):
- print("step", step)
-
if config.IS_DISTRIBUTION:
# scatter dataset
if step % step_per_epoch == 0:
@@ -322,10 +321,8 @@ def start_training(config):
# init metrics values
sess.run(reset_metrics_op)
test_step_size = int(math.ceil(validation_dataset.num_per_epoch / config.BATCH_SIZE))
- print("test_step_size", test_step_size)
for test_step in range(test_step_size):
- print("test_step", test_step)
images, labels = validation_dataset.feed()
feed_dict = {
@@ -351,8 +348,9 @@ def start_training(config):
if rank == 0:
val_writer.add_summary(metrics_summary, step + 1)
+ progbar.update(step + 1)
# training loop end.
- print("reach max step")
+ print("Done")
def run(network, dataset, config_file, experiment_id, recreate):
