feat: add upload rule for reference sets
@@ -46,7 +46,7 @@ rule clean_export_regions:
rule extract_meta:
input:
alignment = rules.build_align.output.alignment,
- metadata = rules.adjust_metadata_regions.output.metadata
+ metadata = _get_metadata_by_wildcards
output:
metadata = "results/{build_name}/extracted_metadata.tsv"
run:
@@ -54,7 +54,7 @@ rule extract_meta:
import pandas as pd
seq_names = [s.id for s in SeqIO.parse(input.alignment, 'fasta')]
- all_meta = pd.read_csv(input.metadata, sep='\t', index_col=0)
+ all_meta = pd.read_csv(input.metadata, sep='\t', index_col=0, dtype=str)
extracted_meta = all_meta.loc[seq_names]
extracted_meta.to_csv(output.metadata, sep='\t')
@@ -183,6 +183,23 @@ rule deploy_to_staging:
fi
"""
+rule upload_reference_sets:
+ input:
+ alignments = expand("results/{build_name}/aligned.fasta", build_name=config["builds"]),
+ metadata = expand("results/{build_name}/extracted_metadata.tsv", build_name=config["builds"])
+ params:
+ s3_bucket = config["S3_DST_BUCKET"],
+ compression = config["S3_DST_COMPRESSION"]
+ run:
+ for fname in alignments:
+ cmd = f"./scripts/upload-to-s3 {fname} s3://{params.s3_bucket}/{os.path.dirname(fname).split('/')[-1]}_alignment.fasta.{params.compression} | tee -a {log}"
+ print("upload command:", cmd)
+ shell(cmd)
+ for fname in metadata:
+ cmd = f"./scripts/upload-to-s3 {fname} s3://{params.s3_bucket}/{os.path.dirname(fname).split('/')[-1]}_metadata.tsv.{params.compression} | tee -a {log}"
+ print("upload command:", cmd)
+ shell(cmd)
+
rule upload:
message: "Uploading intermediate files for specified origins to {params.s3_bucket}"
