use multiple threads
@@ -70,8 +70,34 @@ def test_format_pep8():
# If you run the version described in the linked issue, pylint
# calls sys.exit once it is done, so it kills the test.
+ # Running all files in one pylint command is important for 2 reasons:
+ # 1) Correctness: pylint can detect issues that require access to multiple
+ # files, such as cyclic imports
+ # 2) Speed: pylint imports modules for deep analysis, so if you run
+ # multiple subprocesses each needs to re-import tensorflow.
+ # On Ian's laptop, pylint takes about 10s per file to run on the repo,
+ # and there are about 90 files as of the writing of this comment.
+ # Running pylint on all files simultaneously takes about 70s, so it
+ # is a little better than a 10X speedup.
+
+ # Running multiple jobs in parallel helps but far less than linearly.
+ # On Ian's 4-core laptop, running 4 jobs drops the runtime from 70s
+ # to 45s.
+ # Some of the work is I/O, so it actually makes some sense to run
+ # more jobs than cores. On Ian's 4-core laptop, running 8 jobs drops
+ # the runtime to 40s.
+ # There's a further complication though: I think each job needs to
+ # redo imports, so the total amount of work to do increases with
+ # the number of jobs. On Ian's laptop, using 64 jobs causes the
+ # runtime to increase to 220s. There is not an obvious simple
+ # formula like "use one job per CPU core" or "use way more jobs
+ # than cores to saturate I/O". For now I'm hoping that 8 will be
+ # a reasonable default: it gets good performance on my laptop,
+ # and on machines with fewer than 4 cores there should still be
+ # a benefit to not being blocked on I/O.
+
try:
- shell_call(['pylint', '--rcfile', rcpath] + files_to_check)
+ shell_call(['pylint', '--rcfile', rcpath, '--jobs', '8'] + files_to_check)
except subprocess.CalledProcessError as e:
raise ValueError(e.output.decode("utf-8"))
