Fix initial special cases for Tokenizer.explain
Add the missing initial check for special cases to `Tokenizer.explain`
to align with `Tokenizer._tokenize_affixes`.
@@ -643,6 +643,10 @@ cdef class Tokenizer:
for substring in text.split():
suffixes = []
while substring:
+ if substring in special_cases:
+ tokens.extend(("SPECIAL-" + str(i + 1), self.vocab.strings[e[ORTH]]) for i, e in enumerate(special_cases[substring]))
+ substring = ''
+ continue
while prefix_search(substring) or suffix_search(substring):
if token_match(substring):
tokens.append(("TOKEN_MATCH", substring))
