Pipeline: add option for uploading data with specific version number
@@ -59,6 +59,7 @@ def run_commands_for_arxiv_ids(
command_args.log_names = [log_filename]
command_args.one_entity_at_a_time = process_one_entity_at_a_time
command_args.schema = pipeline_args.database_schema
+ command_args.data_version = pipeline_args.data_version
if CommandCls == FetchArxivSources:
command_args.s3_bucket = pipeline_args.s3_arxiv_sources_bucket
if CommandCls in [StorePipelineLog, StoreResults]:
@@ -268,6 +269,14 @@ if __name__ == "__main__":
+ "(see https://www.postgresql.org/docs/9.2/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS)."
),
)
+ parser.add_argument(
+ "--data-version",
+ type=int,
+ help=(
+ "Version number to assign to the uploaded data. Defaults to creating a new version "
+ + "number for each paper for each run of the pipeline."
+ ),
+ )
args = parser.parse_args()
# Set up logging
