Bugfix: Catch token CSV encoding error earlier
@@ -115,6 +115,7 @@ class ExtractSymbols(ArxivBatchCommand[ArxivId, SymbolData]):
with open(os.path.join(tokens_dir, "tokens.csv"), "a") as tokens_file:
writer = csv.writer(tokens_file, quoting=csv.QUOTE_ALL)
for token in result.characters:
+ try:
writer.writerow(
[
result.path,
@@ -126,6 +127,12 @@ class ExtractSymbols(ArxivBatchCommand[ArxivId, SymbolData]):
token.text,
]
)
+ except Exception: # pylint: disable=broad-except
+ logging.warning(
+ "Couldn't write row for symbol for arXiv %s: can't be converted to utf-8",
+ item,
+ )
+ return
if result.symbols is not None and len(result.symbols) > 0:
with open(
@@ -145,7 +152,6 @@ class ExtractSymbols(ArxivBatchCommand[ArxivId, SymbolData]):
)
for symbol_index, symbol in enumerate(result.symbols):
- try:
symbols_writer.writerow(
[
result.path,
@@ -155,12 +161,6 @@ class ExtractSymbols(ArxivBatchCommand[ArxivId, SymbolData]):
symbol.mathml,
]
)
- except Exception: # pylint: disable=broad-except
- logging.warning(
- "Couldn't write row for symbol for arXiv %s: can't be converted to utf-8",
- item,
- )
- else:
for character in symbol.characters:
symbol_tokens_writer.writerow(
[
