report leaking resources while tests are running
@@ -12,6 +12,7 @@ import os
import sys
import logging
import psutil
+import threading
import moler.config.loggers
from moler.helpers import instance_id
@@ -20,7 +21,16 @@ current_process = psutil.Process()
(max_open_files_limit_soft, max_open_files_limit_hard) = current_process.rlimit(psutil.RLIMIT_NOFILE)
-# plugins to let us see (in moler logs) where we are in testing
+def system_resources_usage():
+ curr_fds_open = current_process.num_fds()
+ curr_threads_nb = threading.active_count()
+ return curr_fds_open, curr_threads_nb
+
+
+def system_resources_usage_msg(curr_fds_open, curr_threads_nb):
+ msg = "RESOURCES USAGE: {}/{} fds OPEN, {} threads active.".format(curr_fds_open, max_open_files_limit_soft,
+ curr_threads_nb)
+ return msg
def check_system_resources_limit(logger):
@@ -33,11 +43,19 @@ def check_system_resources_limit(logger):
sys.stderr.write(msg + "\n")
raise Exception(msg)
+# plugins to let us see (in moler logs) where we are in testing
+
+
+resources_stats = {'START': (0, 0), 'SETUP': (0, 0), 'CALL': (0, 0), 'TEARDOWN': (0, 0)}
+
def pytest_runtest_protocol(item, nextitem):
+ resources_stats['START'] = system_resources_usage()
+ resources_stats['SETUP'] = resources_stats['CALL'] = resources_stats['TEARDOWN'] = (0, 0)
logger = logging.getLogger("moler")
logger.propagate = False
- logger.log(level=moler.config.loggers.TEST_CASE, msg=item.nodeid)
+ resources_usage_msg = system_resources_usage_msg(*resources_stats['START'])
+ logger.log(level=moler.config.loggers.TEST_CASE, msg="{} - {}".format(item.nodeid, resources_usage_msg))
sys.stderr.write(item.nodeid + "\n")
check_system_resources_limit(logger)
@@ -45,9 +63,22 @@ def pytest_runtest_protocol(item, nextitem):
def pytest_runtest_logreport(report):
logger = logging.getLogger("moler")
logger.propagate = False
- msg = "TC {} [{}]".format(str(report.when).upper(), str(report.outcome).upper())
+ when = str(report.when).upper()
+ resources_stats[when] = system_resources_usage()
+ resources_usage_msg = system_resources_usage_msg(*resources_stats[when])
+ msg = "TC {} [{}] - {}".format(when, str(report.outcome).upper(), resources_usage_msg)
logger.log(level=moler.config.loggers.TEST_CASE, msg=msg)
sys.stderr.write(msg + "\n")
+ if when == 'TEARDOWN':
+ logger.log(level=moler.config.loggers.TEST_CASE, msg=str(resources_stats))
+ initial_fds = resources_stats['START'][0]
+ final_fds = resources_stats['TEARDOWN'][0]
+ initial_threads = resources_stats['START'][1]
+ final_threads = resources_stats['TEARDOWN'][1]
+ if (final_fds > initial_fds) or (final_threads > initial_threads):
+ err_msg = " !!! LEAKING RESOURCES: {} -> {} FDs, {} -> {} threads".format(initial_fds, final_fds,
+ initial_threads, final_threads)
+ logger.log(level=logging.ERROR, msg=err_msg)
check_system_resources_limit(logger)
