Skip to content

The agent that grows with you

fix(logging): never take logging's module lock under a profile router lock

Teknium9edfc5db1048
diff --git a/hermes_logging.py b/hermes_logging.py
index 047eabd6e64..4a02d9023ae 100644
--- a/hermes_logging.py
+++ b/hermes_logging.py
@@ -689,14 +689,35 @@ class _ProfileRoutingFileHandler(logging.Handler):
def _handler_for_home(self, home: Path) -> _ManagedRotatingFileHandler:
with self._profile_handlers_lock:
- if home not in self._profile_handlers:
- handler = _new_file_handler(
- home / "logs" / self._filename, level=self.level, max_bytes=self._max_bytes,
- backup_count=self._backup_count, formatter=self.formatter,
- )
- handler._skip_write_when_dir_missing = home != self._default_home
- self._profile_handlers[home] = handler
- return self._profile_handlers[home]
+ handler = self._profile_handlers.get(home)
+ if handler is not None:
+ return handler
+ # Built with no router lock held: ``Handler.__init__`` takes ``logging._lock``, and
+ # ``logging.shutdown`` (uvicorn's ``dictConfig``) holds that lock while it acquires each
+ # handler's lock and closes it (``close`` takes ``_profile_handlers_lock``). Holding
+ # either router lock here deadlocked the QueueListener against ``dictConfig`` (T23).
+ built = _new_file_handler(
+ home / "logs" / self._filename, level=self.level, max_bytes=self._max_bytes,
+ backup_count=self._backup_count, formatter=self.formatter,
+ )
+ built._skip_write_when_dir_missing = home != self._default_home
+ with self._profile_handlers_lock:
+ handler = self._profile_handlers.setdefault(home, built)
+ if handler is not built: # another thread opened it first
+ _quietly(built.close)
+ return handler
+
+ def handle(self, record: logging.LogRecord):
+ """``Handler.handle`` without this router's own I/O lock: each per-home handler takes its
+ own lock to write, and routing state has ``_profile_handlers_lock``. Holding ``self.lock``
+ across ``emit`` (which can open a handler, taking ``logging._lock``) inverted the order
+ ``logging.shutdown`` uses (module lock, then handler lock) (T23)."""
+ rv = self.filter(record)
+ if isinstance(rv, logging.LogRecord):
+ record = rv
+ if rv:
+ self.emit(record)
+ return rv
def emit(self, record: logging.LogRecord) -> None:
try:
diff --git a/tests/test_hermes_logging.py b/tests/test_hermes_logging.py
index 0ac03e7d93d..db9a01a944e 100644
--- a/tests/test_hermes_logging.py
+++ b/tests/test_hermes_logging.py
@@ -1033,3 +1033,42 @@ class TestRolloverPreservesLogOwnership:
monkeypatch.setattr(os, "chown", _refuse)
self._rolled_handler(log_path)
assert log_path.exists()
+
+
+def test_profile_router_never_holds_its_lock_while_taking_the_logging_module_lock(hermes_home, monkeypatch):
+ """T23: uvicorn's ``dictConfig`` holds ``logging._lock`` and, in ``logging.shutdown``, waits on
+ every handler's lock. The QueueListener thread routing a record to a not-yet-opened profile
+ home built that home's file handler (``Handler.__init__`` takes ``logging._lock``) while
+ ``Handler.handle`` held the router's lock: a lock-order deadlock that hung
+ ``test_runtime_listener_supervision`` on CI. The router must take the module lock with no
+ lock of its own held."""
+ profile = hermes_home / "profiles" / "t23"
+ (profile / "logs").mkdir(parents=True)
+ hermes_home.joinpath("logs").mkdir(exist_ok=True)
+ existing = RotatingFileHandler(str(hermes_home / "logs" / "agent.log"), encoding="utf-8")
+ router = hermes_logging._ProfileRoutingFileHandler(existing, [profile])
+ existing.close()
+ building, real_new = threading.Event(), hermes_logging._new_file_handler
+
+ def new_file_handler(*args, **kwargs):
+ building.set()
+ return real_new(*args, **kwargs)
+
+ monkeypatch.setattr(hermes_logging, "_new_file_handler", new_file_handler)
+ record = logging.LogRecord("t23", logging.ERROR, __file__, 1, "routed", None, None)
+ record.hermes_home = str(profile)
+ listener = threading.Thread(target=router.handle, args=(record,), daemon=True)
+ with logging._lock: # dictConfig's configure() holds this across _clearExistingHandlers()
+ listener.start()
+ assert building.wait(5)
+ # logging.shutdown -> h.acquire(): must not wait on a listener blocked on our module lock.
+ acquired = router.lock.acquire(timeout=2)
+ if acquired:
+ router.lock.release()
+ listener.join(5)
+ try:
+ assert acquired, "router held its lock while waiting on logging._lock (dictConfig deadlock)"
+ assert not listener.is_alive()
+ assert "routed" in (profile / "logs" / "agent.log").read_text(encoding="utf-8")
+ finally:
+ router.close()