TeX: Catch encoding errors when outputting extracted symbols
@@ -145,6 +145,7 @@ class ExtractSymbols(ArxivBatchCommand[ArxivId, SymbolData]):
)
for symbol_index, symbol in enumerate(result.symbols):
+ try:
symbols_writer.writerow(
[
result.path,
@@ -154,6 +155,12 @@ class ExtractSymbols(ArxivBatchCommand[ArxivId, SymbolData]):
symbol.mathml,
]
)
+ except Exception: # pylint: disable=broad-except
+ logging.warning(
+ "Couldn't write row for symbol for arXiv %s: can't be converted to utf-8",
+ item,
+ )
+ else:
for character in symbol.characters:
symbol_tokens_writer.writerow(
[
