get_batch_iterator: allow max_positions=None
Summary:
Pull Request resolved:
This function breaks when leaving the argument `max_positions` with the default value `None`, which is presumably not the intended behavior.
@@ -132,6 +132,7 @@ class FairseqTask(object):
indices = dataset.ordered_indices()
# filter examples that are too large
+ if max_positions is not None:
indices = data_utils.filter_by_size(
indices, dataset.size, max_positions, raise_exception=(not ignore_invalid_inputs),
)
