Add get_sents util
from typing import Union, List, Sequence, Callable, Any, Optional
import pytorch_transformers as pytt
import numpy
+from spacy.tokens import Doc, Span
from . import _tokenizers
@@ -208,6 +209,13 @@ def is_special_token(text: str) -> bool:
return text in SPECIAL_TOKENS
+def get_sents(doc: Union[Span, Doc]) -> List[Span]:
+ if doc.is_sentenced:
+ return list(doc.sents)
+ else:
+ return [doc[:]]
+
+
def warmup_linear_rates(initial_rate, warmup_steps, total_steps):
"""Generate a series, starting from an initial rate, and then with a warmup
period, and then a linear decline. Used for learning rates.
