parallelizing run notebooks
@@ -5,6 +5,7 @@ import glob
import os
import re
import sys
+import multiprocessing
import nbformat
from nbconvert.preprocessors import CellExecutionError, ExecutePreprocessor
@@ -14,12 +15,17 @@ if len(sys.argv) >= 2:
else:
pattern = "*.ipynb"
+filenames = [fn for fn in glob.glob(pattern) if not fn.endswith("_exec.ipynb")]
+
+num_files = len(filenames)
+cpu_count = multiprocessing.cpu_count()
+num_jobs = min(1, cpu_count // 4)
+print("Running on {0} CPUs".format(cpu_count))
+print("Running {0} jobs".format(num_jobs))
-errors = []
-for filename in glob.glob(pattern):
- if filename.endswith("_exec.ipynb"):
- continue
+def process_notebook(filename):
+ errors = []
with open(filename) as f:
notebook = nbformat.read(f, as_version=4)
@@ -40,6 +46,13 @@ for filename in glob.glob(pattern):
) as f:
nbformat.write(notebook, f)
+ return "\n\n".join(errors)
+
+
+with multiprocessing.Pool(num_jobs) as pool:
+ errors = list(pool.imap_unordered(process_notebook))
+
+errors = [e for e in errors if len(e.strip())]
txt = "\n\n".join(errors)
ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
txt = ansi_escape.sub("", txt)
