draft collocate
@@ -512,7 +512,22 @@ class ExtSumProcessor:
elif oracle_mode == "combination":
oracle_ids = combination_selection(source, target, selections)
- def _preprocess(src, tgt=None, oracle_ids=None):
+ return {
+ "oracle_ids": oracle_ids,
+ "src_txt": src_txt,
+ "tgt_txt": tgt_txt,
+ }
+ else:
+ return {
+ "src_txt": src_txt,
+ }
+ def collate(self, data, block_size, device, train_mode=True):
+ data = [x for x in data if not len(x["src"]) == 0] # remove empty_files
+ if len(data) == 0:
+ return None
+
+
+ def collate_single(self, src, block_size, tgt=None, oracle_ids=None):
if len(src) == 0:
return None
@@ -566,19 +581,6 @@ class ExtSumProcessor:
src_txt = [original_src_txt[i] for i in idxs]
return src_subtoken_idxs, labels, segments_ids, cls_ids, src_txt, tgt_txt
- b_data = _preprocess(source, target, oracle_ids)
-
- if b_data is None:
- return None
- indexed_tokens, labels, segments_ids, cls_ids, src_txt, tgt_txt = b_data
- return {
- "src": indexed_tokens,
- "labels": labels,
- "segs": segments_ids,
- "clss": cls_ids,
- "src_txt": src_txt,
- "tgt_txt": tgt_txt,
- }
class ExtractiveSummarizer(Transformer):
