fix bug for masking
Summary:
Pull Request resolved:
previously we sample masked tokens with replace=True (default). Because of this, we would mask same tokens multiple times, which will make us mask less tokens finally
@@ -152,7 +152,7 @@ class MaskedLMDataset(FairseqDataset):
masked_sent = np.copy(sentence)
sent_length = len(sentence)
mask_num = math.ceil(sent_length * self.masking_ratio)
- mask = np.random.choice(sent_length, mask_num)
+ mask = np.random.choice(sent_length, mask_num, replace=False)
target = np.copy(sentence)
for i in range(sent_length):
