Limit number of workers, in order to avoid 'too many open files' on machines with many processing units
@@ -1581,8 +1581,9 @@ def main(args=None):
cpu_count = os.cpu_count() or 1
except AttributeError:
pass
- # Use up to cpu_count*2 workers because some tasks are I/O bound.
- parallel = concurrent.futures.ProcessPoolExecutor(max_workers=cpu_count * 2)
+ # Use up to cpu_count*2 workers because some tasks are I/O bound,
+ # but limit the number of worker to avoid too many open files.
+ parallel = concurrent.futures.ProcessPoolExecutor(max_workers=min(cpu_count * 2, 256))
name = options.output_name
outputPath = options.outputPath
