The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(logging): never take logging's module lock under a profile router lock
Teknium9edfc5db1048
diff --git a/hermes_logging.py b/hermes_logging.pyindex 047eabd6e64..4a02d9023ae 100644--- a/hermes_logging.py+++ b/hermes_logging.py@@ -689,14 +689,35 @@ class _ProfileRoutingFileHandler(logging.Handler):def _handler_for_home(self, home: Path) -> _ManagedRotatingFileHandler:with self._profile_handlers_lock:- if home not in self._profile_handlers:- handler = _new_file_handler(- home / "logs" / self._filename, level=self.level, max_bytes=self._max_bytes,- backup_count=self._backup_count, formatter=self.formatter,- )- handler._skip_write_when_dir_missing = home != self._default_home- self._profile_handlers[home] = handler- return self._profile_handlers[home]+ handler = self._profile_handlers.get(home)+ if handler is not None:+ return handler+ # Built with no router lock held: ``Handler.__init__`` takes ``logging._lock``, and+ # ``logging.shutdown`` (uvicorn's ``dictConfig``) holds that lock while it acquires each+ # handler's lock and closes it (``close`` takes ``_profile_handlers_lock``). Holding+ # either router lock here deadlocked the QueueListener against ``dictConfig`` (T23).+ built = _new_file_handler(+ home / "logs" / self._filename, level=self.level, max_bytes=self._max_bytes,+ backup_count=self._backup_count, formatter=self.formatter,+ )+ built._skip_write_when_dir_missing = home != self._default_home+ with self._profile_handlers_lock:+ handler = self._profile_handlers.setdefault(home, built)+ if handler is not built: # another thread opened it first+ _quietly(built.close)+ return handler++ def handle(self, record: logging.LogRecord):+ """``Handler.handle`` without this router's own I/O lock: each per-home handler takes its+ own lock to write, and routing state has ``_profile_handlers_lock``. Holding ``self.lock``+ across ``emit`` (which can open a handler, taking ``logging._lock``) inverted the order+ ``logging.shutdown`` uses (module lock, then handler lock) (T23)."""+ rv = self.filter(record)+ if isinstance(rv, logging.LogRecord):+ record = rv+ if rv:+ self.emit(record)+ return rvdef emit(self, record: logging.LogRecord) -> None:try:diff --git a/tests/test_hermes_logging.py b/tests/test_hermes_logging.pyindex 0ac03e7d93d..db9a01a944e 100644--- a/tests/test_hermes_logging.py+++ b/tests/test_hermes_logging.py@@ -1033,3 +1033,42 @@ class TestRolloverPreservesLogOwnership:monkeypatch.setattr(os, "chown", _refuse)self._rolled_handler(log_path)assert log_path.exists()+++def test_profile_router_never_holds_its_lock_while_taking_the_logging_module_lock(hermes_home, monkeypatch):+ """T23: uvicorn's ``dictConfig`` holds ``logging._lock`` and, in ``logging.shutdown``, waits on+ every handler's lock. The QueueListener thread routing a record to a not-yet-opened profile+ home built that home's file handler (``Handler.__init__`` takes ``logging._lock``) while+ ``Handler.handle`` held the router's lock: a lock-order deadlock that hung+ ``test_runtime_listener_supervision`` on CI. The router must take the module lock with no+ lock of its own held."""+ profile = hermes_home / "profiles" / "t23"+ (profile / "logs").mkdir(parents=True)+ hermes_home.joinpath("logs").mkdir(exist_ok=True)+ existing = RotatingFileHandler(str(hermes_home / "logs" / "agent.log"), encoding="utf-8")+ router = hermes_logging._ProfileRoutingFileHandler(existing, [profile])+ existing.close()+ building, real_new = threading.Event(), hermes_logging._new_file_handler++ def new_file_handler(*args, **kwargs):+ building.set()+ return real_new(*args, **kwargs)++ monkeypatch.setattr(hermes_logging, "_new_file_handler", new_file_handler)+ record = logging.LogRecord("t23", logging.ERROR, __file__, 1, "routed", None, None)+ record.hermes_home = str(profile)+ listener = threading.Thread(target=router.handle, args=(record,), daemon=True)+ with logging._lock: # dictConfig's configure() holds this across _clearExistingHandlers()+ listener.start()+ assert building.wait(5)+ # logging.shutdown -> h.acquire(): must not wait on a listener blocked on our module lock.+ acquired = router.lock.acquire(timeout=2)+ if acquired:+ router.lock.release()+ listener.join(5)+ try:+ assert acquired, "router held its lock while waiting on logging._lock (dictConfig deadlock)"+ assert not listener.is_alive()+ assert "routed" in (profile / "logs" / "agent.log").read_text(encoding="utf-8")+ finally:+ router.close()