[rpc] Include local service logs on init failure.
Issue
@@ -12,7 +12,7 @@ import sys
from datetime import datetime
from pathlib import Path
from time import sleep, time
-from typing import List, NamedTuple, Optional, TypeVar, Union
+from typing import Iterable, List, NamedTuple, Optional, TypeVar, Union
import grpc
@@ -25,6 +25,7 @@ from compiler_gym.service.proto import (
)
from compiler_gym.util.debug_util import get_debug_level
from compiler_gym.util.runfiles_path import runfiles_path, transient_cache_path
+from compiler_gym.util.truncate import truncate_lines
GRPC_CHANNEL_OPTIONS = [
# Raise the default inbound message filter from 4MB.
@@ -215,6 +216,13 @@ class Connection(object):
f"RPC call returned status code {e.code()} and error `{e.details()}`"
) from None
+ def loglines(self) -> Iterable[str]:
+ """Fetch any available log lines from the service backend.
+
+ :return: An iterator over lines of logs.
+ """
+ yield from ()
+
def make_working_dir():
"""Make a working directory for a service. The calling code is responsible for
@@ -329,14 +337,35 @@ class ManagedConnection(Connection):
else:
self.process.kill()
self.process.communicate(timeout=process_exit_max_seconds)
+
+ # Include the last few lines of logs generated by the compiler
+ # service, if any.
+ logs = truncate_lines(
+ self.loglines(), max_line_len=100, max_lines=25, tail=True
+ )
+ logs_message = f" Service logs:\n{logs}" if logs else ""
+
shutil.rmtree(self.working_dir)
raise TimeoutError(
"Failed to connect to RPC service after "
- f"{process_exit_max_seconds:.1f} seconds"
+ f"{process_exit_max_seconds:.1f} seconds.{logs_message}"
)
super().__init__(channel, url, logger)
+ def loglines(self) -> Iterable[str]:
+ """Fetch any available log lines from the service backend.
+
+ :return: An iterator over lines of logs.
+ """
+ # Compiler services write log files in the logs directory. Iterate over
+ # them and return their contents.
+ for path in sorted((self.working_dir / "logs").iterdir()):
+ if not path.is_file():
+ continue
+ with open(path) as f:
+ yield from f.readlines()
+
def close(self):
"""Terminate a local subprocess and close the connection."""
self.process.kill()
