Improve preprocessing hack
@@ -7,6 +7,7 @@ import thinc.extra.datasets
import spacy
from spacy.util import minibatch, compounding
import tqdm
+import unicodedata
@plac.annotations(
@@ -89,6 +90,15 @@ def main(model, output_dir=None, n_iter=20, n_texts=100):
white_re = re.compile(r"\s\s+")
+
+
+def preprocess_text(text):
+ text = white_re.sub(" ", text).strip()
+ return "".join(
+ c for c in unicodedata.normalize("NFD", text) if unicodedata.category(c) != "Mn"
+ )
+
+
def load_data(limit=0, split=0.8):
"""Load data from the IMDB dataset."""
# Partition off part of the train data for evaluation
@@ -96,7 +106,7 @@ def load_data(limit=0, split=0.8):
random.shuffle(train_data)
train_data = train_data[-limit:]
texts, labels = zip(*train_data)
- texts = [white_re.sub(" ", text).strip() for text in texts]
+ texts = [preprocess_text(text) for text in texts]
cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
split = int(len(train_data) * split)
return (texts[:split], cats[:split]), (texts[split:], cats[split:])
