Fix breakage after upstream change in sacrebleu
Summary: Pull Request resolved:
@@ -368,5 +368,7 @@ class TranslationTask(FairseqTask):
if self.args.eval_bleu_print_samples:
logger.info('example hypothesis: ' + hyps[0])
logger.info('example reference: ' + refs[0])
- tokenize = sacrebleu.DEFAULT_TOKENIZER if not self.args.eval_tokenized_bleu else 'none'
- return sacrebleu.corpus_bleu(hyps, [refs], tokenize=tokenize)
+ if self.args.eval_tokenized_bleu:
+ return sacrebleu.corpus_bleu(hyps, [refs], tokenize='none')
+ else:
+ return sacrebleu.corpus_bleu(hyps, [refs])
