refactor MTextParser() into internal generator
@@ -1239,6 +1239,7 @@ class MTextParser:
ctx: initial MText context
"""
+
__slots__ = ("ctx", "scanner", "_ctx_stack")
def __init__(self, content: str, ctx: MTextContext = None):
@@ -1248,15 +1249,8 @@ class MTextParser:
self.scanner = TextScanner(caret_decode(content))
self._ctx_stack = []
- def __next__(self) -> MTextToken:
- type_, data = self.next_token()
- if type_:
- return MTextToken(type_, self.ctx, data)
- else:
- raise StopIteration
-
def __iter__(self):
- return self
+ return self.parse()
def push_ctx(self) -> None:
self._ctx_stack.append(self.ctx)
@@ -1265,19 +1259,21 @@ class MTextParser:
if self._ctx_stack:
self.ctx = self._ctx_stack.pop()
- def next_token(self):
- def word_or_token(token, count: int = 1):
+ def parse(self) -> Iterable[MTextToken]:
+ # localize method calls
+ scanner = self.scanner
+ consume = scanner.fast_consume
+ peek = scanner.fast_peek
+
+ def word_or_token(word, token, count: int = 1):
if word:
return TokenType.WORD, word
else:
consume(count)
return token, None
+ def next_token():
word = ""
- # localize method calls
- scanner = self.scanner
- consume = scanner.fast_consume
- peek = scanner.fast_peek
while scanner.has_data:
escape = False
letter = peek()
@@ -1315,9 +1311,9 @@ class MTextParser:
# process control chars, caret decoding is already done!
if letter < " ":
if letter == "\t":
- return word_or_token(TokenType.TABULATOR)
+ return word_or_token(word, TokenType.TABULATOR)
elif letter == "\n": # LF
- return word_or_token(TokenType.NEW_PARAGRAPH)
+ return word_or_token(word, TokenType.NEW_PARAGRAPH)
else: # replace other control chars by a space
letter = " "
elif letter == "%" and peek(1) == "%":
@@ -1328,7 +1324,7 @@ class MTextParser:
letter = special_char
if letter == " ":
- return word_or_token(TokenType.SPACE)
+ return word_or_token(word, TokenType.SPACE)
if not escape:
if letter == "{":
@@ -1356,6 +1352,13 @@ class MTextParser:
else:
return TokenType.NONE, None
+ while True:
+ type_, data = next_token()
+ if type_:
+ yield MTextToken(type_, self.ctx, data)
+ else:
+ break
+
def parse_stacking(self) -> Tuple:
"""Returns a tuple of strings: (numerator, denominator, type).
The numerator and denominator is always a single and can contain spaces,
