Added Exception Logging to TB Watchers Threads
@@ -198,7 +198,7 @@ class TBDirWatcher(object):
self._generator = self.directory_watcher.DirectoryWatcher(
logdir, self._loader(save, namespace), self._is_our_tfevents_file
)
- self._thread = threading.Thread(target=self._thread_body)
+ self._thread = threading.Thread(target=self._thread_except_body)
self._first_event_timestamp = None
self._shutdown = threading.Event()
self._queue = queue
@@ -270,6 +270,13 @@ class TBDirWatcher(object):
if not self._shutdown.is_set() and not shutdown_call:
time.sleep(ERROR_DELAY)
+ def _thread_except_body(self) -> None:
+ try:
+ self._thread_body()
+ except Exception as e:
+ logger.exception("generic exception in TBDirWatcher thread")
+ raise e
+
def _thread_body(self) -> None:
"""Check for new events every second"""
shutdown_time: "Optional[float]" = None
@@ -334,7 +341,7 @@ class TBEventConsumer(object):
) -> None:
self._tbwatcher = tbwatcher
self._queue = queue
- self._thread = threading.Thread(target=self._thread_body)
+ self._thread = threading.Thread(target=self._thread_except_body)
self._shutdown = threading.Event()
self.tb_history = TBHistory()
self._delay = delay
@@ -366,6 +373,13 @@ class TBEventConsumer(object):
self._save_row(item,)
self._thread.join()
+ def _thread_except_body(self) -> None:
+ try:
+ self._thread_body()
+ except Exception as e:
+ logger.exception("generic exception in TBEventConsumer thread")
+ raise e
+
def _thread_body(self) -> None:
while True:
try:
