Fix tokenization (fixes
Summary:
Fixes
Pull Request resolved:
@@ -36,8 +36,8 @@ class RobertaHubInterface(nn.Module):
def encode(self, sentence: str, *addl_sentences) -> torch.LongTensor:
bpe_sentence = '<s> ' + self.bpe.encode(sentence) + ' </s>'
for s in addl_sentences:
- bpe_sentence += ' </s> ' + self.bpe.encode(s)
- tokens = self.task.source_dictionary.encode_line(bpe_sentence, append_eos=True)
+ bpe_sentence += ' </s> ' + self.bpe.encode(s) + ' </s>'
+ tokens = self.task.source_dictionary.encode_line(bpe_sentence, append_eos=False)
return tokens.long()
def extract_features(self, tokens: torch.LongTensor, return_all_hiddens=False) -> torch.Tensor:
