Fix dummy tokenizer
-from typing import List
+from typing import List, Union
import torch
import copy
@@ -55,6 +55,9 @@ class DummyTokenizer:
output["token_type_ids"] = torch.tensor(output["token_type_ids"]) # type: ignore
return output
+ def convert_ids_to_tokens(self, ids: Union[List[int], torch.Tensor]) -> List[str]:
+ return [self.int2str[int(id_)] for id_ in ids]
+
def _pad(self, batch):
batch = copy.deepcopy(batch)
longest = max(len(ids) for ids in batch["input_ids"])
@@ -87,6 +90,7 @@ class DummyTokenizer:
ids = []
for word in words:
if word not in self.str2int:
+ self.int2str[len(self.str2int)] = word
self.str2int[word] = len(self.str2int)
ids.append(self.str2int[word])
return ids
