Handle deprecated tokenizer prefix data
@@ -94,8 +94,13 @@ def read_regex(path):
def compile_prefix_regex(entries):
+ if '(' in entries:
+ # Handle deprecated data
expression = '|'.join(['^' + re.escape(piece) for piece in entries if piece.strip()])
return re.compile(expression)
+ else:
+ expression = '|'.join(['^' + piece for piece in entries if piece.strip()])
+ return re.compile(expression)
def compile_suffix_regex(entries):
