Add day of month tokenizer exceptions for Danish.
@@ -14,6 +14,11 @@ def test_da_tokenizer_handles_ambiguous_abbr(da_tokenizer, text):
tokens = da_tokenizer(text)
assert len(tokens) == 2
+@pytest.mark.parametrize('text', ["1.", "10.", "31."])
+def test_da_tokenizer_handles_dates(da_tokenizer, text):
+ tokens = da_tokenizer(text)
+ assert len(tokens) == 1
+
def test_da_tokenizer_handles_exc_in_text(da_tokenizer):
text = "Det er bl.a. ikke meningen"
tokens = da_tokenizer(text)
