Port updates from ncov-ingest scripts
Improved memory usage during hashing and better error handling during S3
upload.
@@ -4,4 +4,12 @@ Portable sha256sum utility.
"""
from hashlib import sha256
from sys import stdin
-print(sha256(stdin.buffer.read()).hexdigest())
+
+chunk_size = 5 * 1024**2 # 5 MiB
+
+h = sha256()
+
+for chunk in iter(lambda: stdin.buffer.read(chunk_size), b""):
+ h.update(chunk)
+
+print(h.hexdigest())
