Load token_match regex with .match, not .search
@@ -407,7 +407,7 @@ cdef class Tokenizer:
if data.get('infix_finditer'):
self.infix_finditer = re.compile(data['infix_finditer']).finditer
if data.get('token_match'):
- self.token_match = re.compile(data['token_match']).search
+ self.token_match = re.compile(data['token_match']).match
for string, substrings in data.get('rules', {}).items():
self.add_special_case(string, substrings)
return self
