update checkpoint mkdir behavior (issue
Summary:
Create checkpoint directory in ```save_checkpoint()```instead of ```load_checkpoint()```.
Pull Request resolved:
@@ -23,6 +23,10 @@ logger = logging.getLogger(__name__)
def save_checkpoint(args, trainer, epoch_itr, val_loss):
from fairseq import distributed_utils, meters
+ # only one worker should attempt to create the required dir
+ if args.distributed_rank == 0:
+ os.makedirs(args.save_dir, exist_ok=True)
+
prev_best = getattr(save_checkpoint, "best", val_loss)
if val_loss is not None:
best_function = max if args.maximize_best_checkpoint_metric else min
@@ -118,9 +122,6 @@ def load_checkpoint(args, trainer, **passthrough_args):
*passthrough_args* will be passed through to
``trainer.get_train_iterator``.
"""
- # only one worker should attempt to create the required dir
- if args.distributed_rank == 0:
- os.makedirs(args.save_dir, exist_ok=True)
suffix = getattr(args, "checkpoint_suffix", "")
if args.restore_file == "checkpoint_last.pt":
