Update train_textcat script
#!/usr/bin/env python
import plac
+import re
import random
from pathlib import Path
import thinc.extra.datasets
@@ -14,6 +15,7 @@ from spacy.util import minibatch, compounding
n_iter=("Number of training iterations", "option", "n", int),
)
def main(model, output_dir=None, n_iter=20, n_texts=100):
+ random.seed(0)
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
@@ -36,7 +38,7 @@ def main(model, output_dir=None, n_iter=20, n_texts=100):
# load the IMDB dataset
print("Loading IMDB data...")
- (train_texts, train_cats), (dev_texts, dev_cats) = load_data()
+ (train_texts, train_cats), (dev_texts, dev_cats) = load_data(limit=n_texts)
train_texts = train_texts[:n_texts]
train_cats = train_cats[:n_texts]
print(
@@ -83,13 +85,15 @@ def main(model, output_dir=None, n_iter=20, n_texts=100):
print(test_text, doc2.cats)
+white_re = re.compile(r"\s\s+")
def load_data(limit=0, split=0.8):
"""Load data from the IMDB dataset."""
# Partition off part of the train data for evaluation
- train_data, _ = thinc.extra.datasets.imdb()
+ train_data, _ = thinc.extra.datasets.imdb(limit=limit)
random.shuffle(train_data)
train_data = train_data[-limit:]
texts, labels = zip(*train_data)
+ texts = [white_re.sub(" ", text) for text in texts]
cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
split = int(len(train_data) * split)
return (texts[:split], cats[:split]), (texts[split:], cats[split:])
