Have translation task always use valid/test in the the first shard to avoid copying valid/test data across all shards
Summary: Have translation task always use valid/test in the the first  shard to avoid copying valid/test data across all shards
@@ -159,7 +159,9 @@ class TranslationTask(FairseqTask):
"""Add task-specific arguments to the parser."""
# fmt: off
parser.add_argument('data', help='colon separated path to data directories list, \
- will be iterated upon during epochs in round-robin manner')
+ will be iterated upon during epochs in round-robin manner; \
+ however, valid and test data are always in the first directory to \
+ avoid the need for repeating them in all directories')
parser.add_argument('-s', '--source-lang', default=None, metavar='SRC',
help='source language')
parser.add_argument('-t', '--target-lang', default=None, metavar='TARGET',
@@ -246,6 +248,9 @@ class TranslationTask(FairseqTask):
"""
paths = utils.split_paths(self.args.data)
assert len(paths) > 0
+ if split != getattr(self.args, "train_subset", None):
+ # if not training data set, use the first shard for valid and test
+ paths = paths[:1]
data_path = paths[(epoch - 1) % len(paths)]
# infer langcode
