Update train_textcat example
@@ -58,12 +58,14 @@ def main(
)
else:
(train_texts, train_cats), (eval_texts, eval_cats) = load_data(limit=n_texts)
+ print(f"Using {len(train_texts)} training docs, {len(eval_texts)} evaluation)")
+ split_training_by_sentence = False
+ if split_training_by_sentence:
# If we're using a model that averages over sentence predictions (we are),
# there are some advantages to just labelling each sentence as an example.
# It means we can mix the sentences into different batches, so we can make
# more frequent updates. It also changes the loss somewhat, in a way that's
# not obviously better -- but it does seem to work well.
- print(f"Using {len(train_texts)} training docs, {len(eval_texts)} evaluation)")
train_texts, train_cats = make_sentence_examples(nlp, train_texts, train_cats)
print(f"Extracted {len(train_texts)} training sents")
total_words = sum(len(text.split()) for text in train_texts)
@@ -142,8 +144,8 @@ def load_data(*, limit=0, dev_size=2000):
random.shuffle(train_data)
dev_data = train_data[:dev_size]
train_data = train_data[dev_size:]
- train_texts, train_labels = _prepare_partition(train_data)
- dev_texts, dev_labels = _prepare_partition(dev_data)
+ train_texts, train_labels = _prepare_partition(train_data, preprocess=False)
+ dev_texts, dev_labels = _prepare_partition(dev_data, preprocess=False)
return (train_texts, train_labels), (dev_texts, dev_labels)
@@ -159,8 +161,11 @@ def load_data_for_final_test(*, limit=0):
return (train_texts, train_labels), (test_texts, test_labels)
-def _prepare_partition(text_label_tuples):
+def _prepare_partition(text_label_tuples, *, preprocess=False):
texts, labels = zip(*text_label_tuples)
+ if preprocess:
+ # Preprocessing can mask errors in our handling of noisy text, so
+ # we don't want to do it by default
texts = [preprocess_text(text) for text in texts]
cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
return texts, cats
