Update regression test for to match current expected behaviour
@@ -4,15 +4,14 @@ from __future__ import unicode_literals
import pytest
-@pytest.mark.xfail
@pytest.mark.parametrize('text,tokens', [
- ('"deserve,"--and', ['"', "deserve", ",", '"', "--", "and"]),
- ("exception;--exclusive", ["exception", ";", "--", "exclusive"]),
- ("day.--Is", ["day", ".", "--", "Is"]),
- ("refinement:--just", ["refinement", ":", "--", "just"]),
- ("memories?--To", ["memories", "?", "--", "To"]),
- ("Useful.=--Therefore", ["Useful", ".", "=", "--", "Therefore"]),
- ("=Hope.=--Pandora", ["=", "Hope", ".", "=", "--", "Pandora"])])
+ ('"deserve,"--and', ['"', "deserve", ',"--', "and"]),
+ ("exception;--exclusive", ["exception", ";--", "exclusive"]),
+ ("day.--Is", ["day", ".--", "Is"]),
+ ("refinement:--just", ["refinement", ":--", "just"]),
+ ("memories?--To", ["memories", "?--", "To"]),
+ ("Useful.=--Therefore", ["Useful", ".=--", "Therefore"]),
+ ("=Hope.=--Pandora", ["=", "Hope", ".=--", "Pandora"])])
def test_issue801(en_tokenizer, text, tokens):
"""Test that special characters + hyphens are split correctly."""
doc = en_tokenizer(text)
