The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a loop-liveness death writes its thread stacks to the dump file
Teknium6e62baae0976
diff --git a/gateway/shutdown_watchdog.py b/gateway/shutdown_watchdog.pyindex 5854b42dc31..4b12abc4112 100644--- a/gateway/shutdown_watchdog.py+++ b/gateway/shutdown_watchdog.py@@ -128,10 +128,12 @@ def start_loop_liveness_watchdog("Gateway event loop missed %d consecutive liveness probes; dumping all thread ""stacks and exiting with code %d so the service supervisor can restart it.",strikes, exit_code)- try:- faulthandler.dump_traceback(all_threads=True)- except Exception:- logger.debug("Loop liveness faulthandler dump failed", exc_info=True)+ # File + stderr: a console-less launcher (Windows .vbs login item) discards stderr, so a+ # liveness death used to leave no stack anywhere (#120356, #91097).+ _write_watchdog_dump(+ get_shutdown_watchdog_dump_path(), delay_s=probe_timeout * strikes,+ snapshot={"strikes": strikes, "probe_interval_s": probe_interval},+ event="loop_liveness_watchdog_fired")if stop_event.is_set():return_mark_exited_quietly(exit_code, "loop_liveness_watchdog")@@ -236,15 +238,13 @@ def resolve_shutdown_watchdog_delay(return _coerce_float(drain_timeout, 0.0) + grace-def _write_watchdog_dump(dump_path: Path, *, delay_s: float,- snapshot: Optional[dict[str, Any]]) -> None:- """Best-effort faulthandler + metadata dump before hard-exit."""- try:- dump_path.parent.mkdir(parents=True, exist_ok=True)- except OSError:- return- header = {"event": "shutdown_watchdog_fired", "pid": os.getpid(), "delay_s": delay_s,+def _write_watchdog_dump(dump_path: Path, *, delay_s: float, snapshot: Optional[dict[str, Any]],+ event: str = "shutdown_watchdog_fired") -> None:+ """Best-effort faulthandler + metadata dump before hard-exit (file, then stderr)."""+ header = {"event": event, "pid": os.getpid(), "delay_s": delay_s,"fired_at": datetime.now(UTC).isoformat(), "snapshot": snapshot or {}}+ with contextlib.suppress(OSError): # unwritable dir: skip only the file copy, stderr still runs+ dump_path.parent.mkdir(parents=True, exist_ok=True)with contextlib.suppress(Exception), open(dump_path, "a", encoding="utf-8") as fh:fh.write(json.dumps(header, default=str) + "\n--- faulthandler dump (all threads) ---\n")fh.flush()@@ -255,7 +255,7 @@ def _write_watchdog_dump(dump_path: Path, *, delay_s: float,fh.write("--- end dump ---\n")fh.flush()with contextlib.suppress(Exception): # stderr too: journald/launchd get it if disk is wedged- sys.stderr.write(f"Gateway shutdown watchdog fired after {delay_s:.0f}s "+ sys.stderr.write(f"Gateway {event.replace('_', ' ')} after {delay_s:.0f}s "f"(pid={os.getpid()}); dumping all thread stacks.\n")sys.stderr.flush()faulthandler.dump_traceback(all_threads=True)diff --git a/tests/gateway/test_loop_liveness_watchdog.py b/tests/gateway/test_loop_liveness_watchdog.pyindex 90c91a57a33..3ab8008f02a 100644--- a/tests/gateway/test_loop_liveness_watchdog.py+++ b/tests/gateway/test_loop_liveness_watchdog.py@@ -41,9 +41,32 @@ def test_loop_liveness_watchdog_stop_during_dump_disarms_hard_exit():assert not handle.is_alive()critical.assert_called_once()- dump.assert_called_once_with(all_threads=True)+ dump.assert_called() # dump-file copy first, then stderr (#120356)assert exit_codes == []++def test_loop_liveness_death_leaves_its_stacks_in_the_dump_file(tmp_path, monkeypatch):+ # A console-less launcher discards stderr; the file is the only copy that survives (#120356).+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))+ exit_codes, ref = [], {}++ def exit_and_stop(code: int) -> None: # a patched exit returns, so stop the loop like a real one+ exit_codes.append(code)+ ref["handle"].stop()++ with patch("gateway.shutdown_watchdog._hard_exit", side_effect=exit_and_stop), \+ patch("gateway.shutdown_watchdog._mark_exited_quietly"):+ ref["handle"] = handle = start_loop_liveness_watchdog(+ MagicMock(spec=asyncio.AbstractEventLoop), probe_interval=0.01, probe_timeout=0.01,+ max_strikes=1)+ handle.join(timeout=5.0)+ from gateway.shutdown_watchdog import get_shutdown_watchdog_dump_path+ dump = get_shutdown_watchdog_dump_path(tmp_path).read_text(encoding="utf-8-sig")+ assert '"event": "loop_liveness_watchdog_fired"' in dump+ assert "most recent call first" in dump+ assert exit_codes == [75]++def test_loop_liveness_watchdog_stop_during_final_miss_disarms_hard_exit():loop = MagicMock(spec=asyncio.AbstractEventLoop)probe_scheduled = threading.Event()