moved common plaintext and offset code to common/parse_tex
@@ -3,7 +3,7 @@ from typing import Iterator, List
import pysbd
-from common.parse_tex import DEFAULT_CONTEXT_SIZE, EntityExtractor, PlaintextExtractor
+from common.parse_tex import DEFAULT_CONTEXT_SIZE, EntityExtractor, reserved_char_check, plaintext_and_offset
from .types import Sentence
@@ -46,38 +46,9 @@ class SentenceExtractor(EntityExtractor):
"""
def parse(self, tex_path: str, tex: str) -> Iterator[Sentence]:
- for reserved_char in PYSBD_RESERVED_CHARACTERS:
- if reserved_char in tex:
- logging.warning(
- 'Reserved character from pysbd "%s" found in tex string, this might break the sentence extractor.',
- reserved_char,
- )
-
- # Extract plaintext segments from TeX
- plaintext_extractor = PlaintextExtractor()
- plaintext_segments = plaintext_extractor.parse(tex_path, tex)
-
- # Build a map from character offsets in the plaintext to TeX offsets. This will let us
- # map from the character offsets of the sentences returned from the sentence boundary
- # detector back to positions in the original TeX.
- plaintext_to_tex_offset_map = {}
- plaintext = ""
- last_segment = None
- for segment in plaintext_segments:
- for i in range(len(segment.text)):
- tex_offset = (
- (segment.tex_start + i)
- if not segment.transformed
- else segment.tex_start
- )
- plaintext_to_tex_offset_map[len(plaintext) + i] = tex_offset
-
- # While building the map, also create a contiguous plaintext string
- plaintext += segment.text
- last_segment = segment
+ reserved_char_check(tex)
+ plaintext, plaintext_to_tex_offset_map = plaintext_and_offset(tex_path, tex)
- if last_segment is not None:
- plaintext_to_tex_offset_map[len(plaintext)] = last_segment.tex_end
# Segment the plaintext. Return offsets for each setence relative to the TeX input
segmenter = pysbd.Segmenter(language="en", clean=False)
# Record the current length of the plain text so account for the extractor bug
