fix a bug in matching loader
@@ -171,7 +171,7 @@ class MatchingLoader(DataSetLoader):
new_field_name=Const.INPUT_LENS(1), is_input=auto_set_input)
if auto_pad_length is not None:
- cut_text = min(auto_pad_length, cut_text if cut_text is not None else 0)
+ cut_text = min(auto_pad_length, cut_text if cut_text is not None else auto_pad_length)
if cut_text is not None:
for data_name, data_set in data_info.datasets.items():
@@ -207,10 +207,10 @@ class MatchingLoader(DataSetLoader):
is_input=auto_set_input, is_target=auto_set_target)
if auto_pad_length is not None:
- for data_name, data_set in data_info.datasets.items():
if seq_len_type == 'seq_len':
- raise RuntimeError(f'sequence will be padded with the length {auto_pad_length},'
- f'the seq_len_type cannot be `{seq_len_type}`!')
+ raise RuntimeError(f'the sequence will be padded with the length {auto_pad_length}, '
+ f'so the seq_len_type cannot be `{seq_len_type}`!')
+ for data_name, data_set in data_info.datasets.items():
for fields in data_set.get_field_names():
if Const.INPUT in fields:
data_set.apply(lambda x: x[fields] + [words_vocab.to_index(words_vocab.padding)] *
@@ -299,6 +299,7 @@ class RTELoader(MatchingLoader, CSVLoader):
ds = CSVLoader._load(self, path)
for k, v in self.fields.items():
+ if v in ds.get_field_names():
ds.rename_field(k, v)
for fields in ds.get_all_fields():
if Const.INPUT in fields:
@@ -338,6 +339,7 @@ class QNLILoader(MatchingLoader, CSVLoader):
ds = CSVLoader._load(self, path)
for k, v in self.fields.items():
+ if v in ds.get_field_names():
ds.rename_field(k, v)
for fields in ds.get_all_fields():
if Const.INPUT in fields:
