parallelizing right notebooks
@@ -5,7 +5,6 @@ import glob
import os
import re
import sys
-import multiprocessing
import nbformat
from nbconvert.preprocessors import CellExecutionError, ExecutePreprocessor
@@ -15,18 +14,13 @@ if len(sys.argv) >= 2:
else:
pattern = "*.ipynb"
-filenames = [fn for fn in glob.glob(pattern) if not fn.endswith("_exec.ipynb")]
-num_files = len(filenames)
-cpu_count = multiprocessing.cpu_count()
-num_jobs = min(1, cpu_count // 4)
-print("Running on {0} CPUs".format(cpu_count))
-print("Running {0} jobs".format(num_jobs))
-
-
-def process_notebook(filename):
errors = []
+for filename in glob.glob(pattern):
+ if filename.endswith("_exec.ipynb"):
+ continue
+
with open(filename) as f:
notebook = nbformat.read(f, as_version=4)
@@ -46,13 +40,6 @@ def process_notebook(filename):
) as f:
nbformat.write(notebook, f)
- return "\n\n".join(errors)
-
-
-with multiprocessing.Pool(num_jobs) as pool:
- errors = list(pool.imap_unordered(process_notebook))
-
-errors = [e for e in errors if len(e.strip())]
txt = "\n\n".join(errors)
ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
txt = ansi_escape.sub("", txt)
