The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(logging): the profile routing handler never builds a child handler under its own lock
Teknium477c43292622
diff --git a/hermes_logging.py b/hermes_logging.pyindex 047eabd6e64..8ed772b2d4c 100644--- a/hermes_logging.py+++ b/hermes_logging.py@@ -689,14 +689,39 @@ class _ProfileRoutingFileHandler(logging.Handler):def _handler_for_home(self, home: Path) -> _ManagedRotatingFileHandler:with self._profile_handlers_lock:- if home not in self._profile_handlers:- handler = _new_file_handler(- home / "logs" / self._filename, level=self.level, max_bytes=self._max_bytes,- backup_count=self._backup_count, formatter=self.formatter,- )- handler._skip_write_when_dir_missing = home != self._default_home- self._profile_handlers[home] = handler- return self._profile_handlers[home]+ existing = self._profile_handlers.get(home)+ if existing is not None:+ return existing+ # Built with NO lock held: Handler.__init__ takes logging's module lock, and+ # logging.shutdown (dictConfig's _clearExistingHandlers) holds that lock while it takes+ # this router's lock and then, via close(), _profile_handlers_lock. Building under+ # either inverts that order and deadlocks the listener against dictConfig.+ handler = _new_file_handler(+ home / "logs" / self._filename, level=self.level, max_bytes=self._max_bytes,+ backup_count=self._backup_count, formatter=self.formatter,+ )+ handler._skip_write_when_dir_missing = home != self._default_home+ with self._profile_handlers_lock:+ released = home != self._default_home and home not in self._profile_homes+ winner = None if released else self._profile_handlers.setdefault(home, handler)+ if winner is not handler:+ _quietly(handler.close) # another thread routed it first, or release_profile ran+ return self._handler_for_home(self._default_home) if winner is None else winner++ def handle(self, record: logging.LogRecord) -> logging.LogRecord | bool:+ """Filter and route *record* without taking this handler's own lock.++ ``logging.Handler.handle`` emits under ``self.lock``, and routing may build a child+ handler (which takes logging's module lock): the inverse of ``logging.shutdown``'s+ order. The router's lock guarded nothing of its own: each child serializes its I/O+ under its own lock, and the routing map is guarded by ``_profile_handlers_lock``.+ """+ rv = self.filter(record)+ if isinstance(rv, logging.LogRecord):+ record = rv+ if rv:+ self.emit(record)+ return rvdef emit(self, record: logging.LogRecord) -> None:try:diff --git a/tests/test_hermes_logging.py b/tests/test_hermes_logging.pyindex 0ac03e7d93d..2093ffc38ba 100644--- a/tests/test_hermes_logging.py+++ b/tests/test_hermes_logging.py@@ -1033,3 +1033,62 @@ class TestRolloverPreservesLogOwnership:monkeypatch.setattr(os, "chown", _refuse)self._rolled_handler(log_path)assert log_path.exists()+++def test_router_never_builds_a_child_handler_under_a_lock_logging_shutdown_takes(hermes_home, monkeypatch):+ """CI run 39131422357 hung: the listener thread held the router's handler lock while building a+ child handler (which takes ``logging._lock``), while dictConfig held ``logging._lock`` inside+ ``logging.shutdown`` and waited on the router's lock. Both sides must finish."""+ profile_home = (hermes_home / "profiles" / "lockorder").resolve()+ (profile_home / "logs").mkdir(parents=True)+ (hermes_home / "logs").mkdir(exist_ok=True)+ existing = RotatingFileHandler(str(hermes_home / "logs" / "agent.log"), encoding="utf-8")+ router = hermes_logging._ProfileRoutingFileHandler(existing, [profile_home])+ existing.close()++ building, go = threading.Event(), threading.Event()+ real_new_file_handler = hermes_logging._new_file_handler++ def paused_new_file_handler(*args, **kwargs):+ building.set()+ go.wait(5)+ return real_new_file_handler(*args, **kwargs) # Handler.__init__ takes logging._lock++ monkeypatch.setattr(hermes_logging, "_new_file_handler", paused_new_file_handler)+ record = logging.LogRecord("lockorder", logging.WARNING, __file__, 1, "routed", None, None)+ record.hermes_home = str(profile_home)+ holding_module_lock, emitter_released = threading.Event(), threading.Event()+ shutdown_got_router_lock = []++ def shutdown_like_dictconfig():+ # dictConfig.configure holds logging._lock -> _clearExistingHandlers -> logging.shutdown,+ # whose per-handler step is h.acquire(). Bounded here so a regression fails instead of+ # wedging the process (a real shutdown would block forever while holding logging._lock).+ with logging._lock:+ holding_module_lock.set()+ emitter_released.wait(5)+ got = router.lock.acquire(timeout=5)+ shutdown_got_router_lock.append(got)+ if got:+ try:+ router.flush()+ router.close()+ finally:+ router.release()++ emitter = threading.Thread(target=router.handle, args=(record,), daemon=True)+ emitter.start()+ assert building.wait(5)+ clearer = threading.Thread(target=shutdown_like_dictconfig, daemon=True)+ clearer.start()+ assert holding_module_lock.wait(5)+ go.set() # the emitter now heads for logging._lock, which the clearer holds+ emitter_released.set()+ clearer.join(10)+ emitter.join(10)+ for child in list(router._profile_handlers.values()):+ child.close()+ assert not clearer.is_alive() and not emitter.is_alive()+ assert shutdown_got_router_lock == [True], (+ "lock-order inversion: logging.shutdown (under logging._lock) could not take the router's "+ "lock while the router built a child handler (which needs logging._lock)")