Add fallback for forced alignment (WIP)
@@ -4,10 +4,12 @@ import pytorch_transformers as pytt
from thinc.neural.ops import get_array_module
from thinc.extra.wrappers import torch2xp
import torch
+import re
from . import _tokenizers
+alpha_re = re.compile(r"[^A-Za-z]+")
SPECIAL_TOKENS = (
"[CLS]",
"[BOS]",
@@ -221,6 +223,10 @@ def align_word_pieces(spacy_tokens, wp_tokens, specials=SPECIAL_TOKENS):
elif not spacy_tokens:
return []
# Check alignment
+ if "".join(spacy_tokens).lower() != "".join(wp_tokens).lower():
+ # Force alignment
+ spacy_tokens = [alpha_re.sub("", t) for t in spacy_tokens]
+ wp_tokens = [alpha_re.sub("", t) for t in wp_tokens]
spacy_string = "".join(spacy_tokens).lower()
wp_string = "".join(wp_tokens).lower()
if spacy_string != wp_string:
