Fix bug in wiki103 s2s preproc
@@ -1960,7 +1960,7 @@ class Wiki103Seq2SeqTask(MTTask):
continue
sent = _atomic_tokenize(toks, UNK_TOK_ATOMIC, nonatomic_toks,
self.max_seq_len)
- yield (sent, [])
+ yield sent, []
def get_num_examples(self, split_text):
''' Return number of examples in the result of get_split_text.
@@ -1983,7 +1983,7 @@ class Wiki103Seq2SeqTask(MTTask):
return Instance(d)
prev_sent = None
- for sent in split:
+ for sent, _ in split:
if prev_sent is None:
prev_sent = sent
continue
