Add total data size logging to sampled_multi_dataset
Summary: Add total data size logging to help debugging
@@ -370,8 +370,12 @@ class SampledMultiDataset(FairseqDataset):
self.cumulated_sizes = plasma_utils.PlasmaArray(cumulated_sizes)
self.virtual_size_per_dataset = plasma_utils.PlasmaArray(virtual_size_per_dataset)
- logger.info(f'[{self.split}] Raw sizes: {str(dict(zip(self.keys, [len(d) for d in self.datasets])))}')
- logger.info(f'[{self.split}] Resampled sizes: {str(dict(zip(self.keys, self.virtual_size_per_dataset.array)))}')
+ raw_sizes = [len(d) for d in self.datasets]
+ sampled_sizes = self.virtual_size_per_dataset.array
+ logger.info(f'[{self.split}] Raw sizes: {str(dict(zip(self.keys, raw_sizes)))}; '
+ f'raw total size: {sum(raw_sizes)}')
+ logger.info(f'[{self.split}] Resampled sizes: {str(dict(zip(self.keys, sampled_sizes)))}; '
+ f'resampled total size: {sum(sampled_sizes)}')
if self.sample_ratios is not None:
logger.info(f'[{self.split}] Upsampling ratios: {str(dict(zip(self.keys, self.sample_ratios.array)))}')
else:
