TeX: Attempt to load files in both 'utf-8' and 'latin-1'
@@ -10,6 +10,8 @@ from explanations.scrape_tex import TexSoupParseError, extract_bibitems
from explanations.types import Bibitem, FileContents
from scripts.command import Command
+ENCODINGS = ["utf-8", "latin-1"]
+
class ExtractBibitems(Command[FileContents, Bibitem]):
@staticmethod
@@ -25,12 +27,26 @@ class ExtractBibitems(Command[FileContents, Bibitem]):
sources_dir = sources(arxiv_id)
clean_directory(directories.bibitems(arxiv_id))
for path in find_files(sources_dir, [".tex", ".bbl"]):
- with open(path, "r", encoding="utf-8") as file_:
+ contents = None
+ for encoding in ENCODINGS:
+ with open(path, "r", encoding=encoding) as file_:
try:
contents = file_.read()
+ break
+ except Exception: # pylint: disable=broad-except
+ logging.debug(
+ "Could not decode file %s using encoding %s",
+ path,
+ encoding,
+ )
+
+ if contents is None:
+ logging.error(
+ "Could not find an appropriate encoding for file %s", path
+ )
+ continue
+
yield FileContents(arxiv_id, path, contents)
- except UnicodeDecodeError as e:
- logging.error("Could not read file %s: %s", path, e)
def process(self, item: FileContents) -> Iterator[Bibitem]:
try:
