Handle empty and whitespace-only docs for Japanese
Handle empty and whitespace-only docs in the custom alignment method
used by the Japanese tokenizer.
@@ -93,3 +93,12 @@ def test_ja_tokenizer_split_modes(ja_tokenizer, text, len_a, len_b, len_c):
assert len(nlp_a(text)) == len_a
assert len(nlp_b(text)) == len_b
assert len(nlp_c(text)) == len_c
+
+
+def test_ja_tokenizer_emptyish_texts(ja_tokenizer):
+ doc = ja_tokenizer("")
+ assert len(doc) == 0
+ doc = ja_tokenizer(" ")
+ assert len(doc) == 1
+ doc = ja_tokenizer("\n\n\n \t\t \n\n\n")
+ assert len(doc) == 1
