Document regex utilities [ci skip]
@@ -315,6 +315,11 @@ def read_regex(path):
def compile_prefix_regex(entries):
+ """Compile a list of prefix rules into a regex object.
+
+ entries (tuple): The prefix rules, e.g. spacy.lang.punctuation.TOKENIZER_PREFIXES.
+ RETURNS (regex object): The regex object. to be used for Tokenizer.prefix_search.
+ """
if "(" in entries:
# Handle deprecated data
expression = "|".join(
@@ -327,11 +332,21 @@ def compile_prefix_regex(entries):
def compile_suffix_regex(entries):
+ """Compile a list of suffix rules into a regex object.
+
+ entries (tuple): The suffix rules, e.g. spacy.lang.punctuation.TOKENIZER_SUFFIXES.
+ RETURNS (regex object): The regex object. to be used for Tokenizer.suffix_search.
+ """
expression = "|".join([piece + "$" for piece in entries if piece.strip()])
return re.compile(expression)
def compile_infix_regex(entries):
+ """Compile a list of infix rules into a regex object.
+
+ entries (tuple): The infix rules, e.g. spacy.lang.punctuation.TOKENIZER_INFIXES.
+ RETURNS (regex object): The regex object. to be used for Tokenizer.infix_finditer.
+ """
expression = "|".join([piece for piece in entries if piece.strip()])
return re.compile(expression)
