Pipeline: when running with '--extraction-only' flag, skip all compilation and rastering commands
@@ -15,6 +15,7 @@ from common.commands.base import (
load_arxiv_ids_using_args,
read_arxiv_ids_from_file,
)
+from common.commands.compile_tex import CompileTexSources
from common.commands.database import DatabaseUploadCommand
from common.commands.fetch_arxiv_sources import (
DEFAULT_S3_ARXIV_SOURCES_BUCKET,
@@ -22,10 +23,12 @@ from common.commands.fetch_arxiv_sources import (
)
from common.commands.fetch_new_arxiv_ids import FetchNewArxivIds
from common.commands.locate_entities import LocateEntitiesCommand
+from common.commands.raster_pages import RasterPages
from common.commands.store_pipeline_log import StorePipelineLog
from common.commands.store_results import DEFAULT_S3_LOGS_BUCKET, StoreResults
from common.make_digest import make_paper_digest
from common.types import PipelineDigest
+
from scripts.job_config import fetch_config, load_job_from_s3
from scripts.pipelines import entity_pipelines
from scripts.process import (
@@ -398,8 +401,11 @@ if __name__ == "__main__":
if skip_command:
continue
if args.extraction_only:
- if issubclass(CommandClass, LocateEntitiesCommand) or issubclass(
- CommandClass, DatabaseUploadCommand
+ if (
+ issubclass(CommandClass, LocateEntitiesCommand)
+ or issubclass(CommandClass, DatabaseUploadCommand)
+ or CommandClass is CompileTexSources
+ or CommandClass is RasterPages
):
continue
# Optionally skip over database upload commands.
