goat progress bar
from tinygrad.tensor import Tensor
import pickle
+from tqdm import tqdm
import numpy as np
from tinygrad.helpers import prod
@@ -14,10 +15,10 @@ def fetch(url):
with open(fp, "rb") as f:
dat = f.read()
else:
- print("fetching %s" % url)
- r = requests.get(url)
+ r = requests.get(url, stream=True)
assert r.status_code == 200
- dat = r.content
+ progress_bar = tqdm(total=int(r.headers.get('content-length', 0)), unit='B', unit_scale=True, desc=url)
+ dat = b''.join(x for x in r.iter_content(chunk_size=16384) if progress_bar.update(len(x)) or True)
with open(fp+".tmp", "wb") as f:
f.write(dat)
os.rename(fp+".tmp", fp)
