Use parfive in gammapy download
@@ -5,9 +5,9 @@ import json
import logging
import sys
import yaml
-import multiprocessing
from urllib.request import urlretrieve, urlopen
from pathlib import Path
+from parfive import Downloader
from .. import version
log = logging.getLogger(__name__)
@@ -19,23 +19,6 @@ DEV_SCRIPTS_YAML_URL = BASE_URL_DEV + "examples/scripts.yaml"
DEV_DATA_JSON_URL = BASE_URL_DEV + "dev/datasets/gammapy-data-index.json"
-def get_file(ftuple):
- url, filepath, md5server = ftuple
-
- retrieve = True
- if md5server and Path(filepath).exists():
- md5local = hashlib.md5(Path(filepath).read_bytes()).hexdigest()
- if md5local == md5server:
- retrieve = False
-
- if retrieve:
- try:
- urlretrieve(url, filepath)
- except Exception as ex:
- log.error(filepath + " could not be copied.")
- log.error(ex)
-
-
def parse_datafiles(datasearch, datasetslist):
for dataset in datasetslist:
if (datasearch == dataset["name"] or datasearch == "") and dataset.get(
@@ -82,7 +65,7 @@ class ComputePlan:
try:
log.info("Downloading {}".format(url_file_env))
Path(filepath_env).parent.mkdir(parents=True, exist_ok=True)
- get_file((url_file_env, filepath_env, ""))
+ urlretrieve(url_file_env, filepath_env)
except Exception as ex:
log.error(ex)
exit()
@@ -223,7 +206,7 @@ class ParallelDownload:
if self.listfiles:
log.info("Content will be downloaded in {}".format(self.outfolder))
- pool = multiprocessing.Pool(5)
+ dl = Downloader()
for rec in self.listfiles:
url = self.listfiles[rec]["url"]
path = self.outfolder / self.listfiles[rec]["path"]
@@ -232,11 +215,19 @@ class ParallelDownload:
md5 = self.listfiles[rec]["hashmd5"]
ifolder = Path(path).parent
ifolder.mkdir(parents=True, exist_ok=True)
- ftuple = (url, str(path), md5)
- pool.apply_async(get_file, args=(ftuple,), callback=self.progressbar)
- pool.close()
- pool.join()
- pool.close()
+ retrieve = True
+ if md5 and path.exists():
+ md5local = hashlib.md5(path.read_bytes()).hexdigest()
+ if md5local != md5:
+ retrieve = False
+ if retrieve:
+ dl.enqueue_file(url, path=str(path))
+
+ try:
+ dl.download()
+ except Exception as ex:
+ log.error("Failed to download files.")
+ log.error(ex)
def show_info_datasets(self):
print("")
@@ -259,16 +250,3 @@ class ParallelDownload:
print("*** You might want to declare GAMMAPY_DATA env variable")
print("export GAMMAPY_DATA={}".format(GAMMAPY_DATA))
print("")
-
- def progressbar(self, args):
- self.bar += 1
- barLength, status = 50, ""
- progress = self.bar / len(self.listfiles)
- if progress >= 1.0:
- progress, status = 1, "\r\n"
- block = int(round(barLength * progress))
- text = "\rDownloading files [{}] {:.0f}% {}".format(
- "=" * block + "." * (barLength - block), round(progress * 100, 0), status
- )
- sys.stdout.write(text)
- sys.stdout.flush()
