update progbar only if the rank == 0
@@ -213,6 +213,7 @@ def start_training(config):
max_steps = config.MAX_STEPS
progbar = Progbar(max_steps)
+ if rank == 0:
progbar.update(last_step)
for step in range(last_step, max_steps):
if config.IS_DISTRIBUTION:
@@ -348,6 +349,7 @@ def start_training(config):
if rank == 0:
val_writer.add_summary(metrics_summary, step + 1)
+ if rank == 0:
progbar.update(step + 1)
# training loop end.
print("Done")
