bugfix of the bugfix
@@ -674,16 +674,17 @@ def minibatch_by_words(examples, size, count_words=len, tolerance=0.2, discard_o
for example in examples:
n_words = count_words(example.doc)
- # add the example to the current batch if it still fits
- if (current_size + n_words) < (target_size + tol_size):
- batch.append(example)
- current_size += n_words
- else:
# if the current example exceeds the batch size, it is returned separately
# but only if discard_oversize=False.
- if current_size > target_size:
+ if n_words > target_size:
if not discard_oversize:
yield [example]
+
+ # add the example to the current batch if it still fits
+ elif (current_size + n_words) < (target_size + tol_size):
+ batch.append(example)
+ current_size += n_words
+
# yield the previous batch and start a new one
else:
yield batch
