Skip to content

The agent that grows with you

fix(gateway): a loop-liveness death writes its thread stacks to the dump file

Teknium6e62baae0976
diff --git a/gateway/shutdown_watchdog.py b/gateway/shutdown_watchdog.py
index 5854b42dc31..4b12abc4112 100644
--- a/gateway/shutdown_watchdog.py
+++ b/gateway/shutdown_watchdog.py
@@ -128,10 +128,12 @@ def start_loop_liveness_watchdog(
"Gateway event loop missed %d consecutive liveness probes; dumping all thread "
"stacks and exiting with code %d so the service supervisor can restart it.",
strikes, exit_code)
- try:
- faulthandler.dump_traceback(all_threads=True)
- except Exception:
- logger.debug("Loop liveness faulthandler dump failed", exc_info=True)
+ # File + stderr: a console-less launcher (Windows .vbs login item) discards stderr, so a
+ # liveness death used to leave no stack anywhere (#120356, #91097).
+ _write_watchdog_dump(
+ get_shutdown_watchdog_dump_path(), delay_s=probe_timeout * strikes,
+ snapshot={"strikes": strikes, "probe_interval_s": probe_interval},
+ event="loop_liveness_watchdog_fired")
if stop_event.is_set():
return
_mark_exited_quietly(exit_code, "loop_liveness_watchdog")
@@ -236,15 +238,13 @@ def resolve_shutdown_watchdog_delay(
return _coerce_float(drain_timeout, 0.0) + grace
-def _write_watchdog_dump(dump_path: Path, *, delay_s: float,
- snapshot: Optional[dict[str, Any]]) -> None:
- """Best-effort faulthandler + metadata dump before hard-exit."""
- try:
- dump_path.parent.mkdir(parents=True, exist_ok=True)
- except OSError:
- return
- header = {"event": "shutdown_watchdog_fired", "pid": os.getpid(), "delay_s": delay_s,
+def _write_watchdog_dump(dump_path: Path, *, delay_s: float, snapshot: Optional[dict[str, Any]],
+ event: str = "shutdown_watchdog_fired") -> None:
+ """Best-effort faulthandler + metadata dump before hard-exit (file, then stderr)."""
+ header = {"event": event, "pid": os.getpid(), "delay_s": delay_s,
"fired_at": datetime.now(UTC).isoformat(), "snapshot": snapshot or {}}
+ with contextlib.suppress(OSError): # unwritable dir: skip only the file copy, stderr still runs
+ dump_path.parent.mkdir(parents=True, exist_ok=True)
with contextlib.suppress(Exception), open(dump_path, "a", encoding="utf-8") as fh:
fh.write(json.dumps(header, default=str) + "\n--- faulthandler dump (all threads) ---\n")
fh.flush()
@@ -255,7 +255,7 @@ def _write_watchdog_dump(dump_path: Path, *, delay_s: float,
fh.write("--- end dump ---\n")
fh.flush()
with contextlib.suppress(Exception): # stderr too: journald/launchd get it if disk is wedged
- sys.stderr.write(f"Gateway shutdown watchdog fired after {delay_s:.0f}s "
+ sys.stderr.write(f"Gateway {event.replace('_', ' ')} after {delay_s:.0f}s "
f"(pid={os.getpid()}); dumping all thread stacks.\n")
sys.stderr.flush()
faulthandler.dump_traceback(all_threads=True)
diff --git a/tests/gateway/test_loop_liveness_watchdog.py b/tests/gateway/test_loop_liveness_watchdog.py
index 90c91a57a33..3ab8008f02a 100644
--- a/tests/gateway/test_loop_liveness_watchdog.py
+++ b/tests/gateway/test_loop_liveness_watchdog.py
@@ -41,9 +41,32 @@ def test_loop_liveness_watchdog_stop_during_dump_disarms_hard_exit():
assert not handle.is_alive()
critical.assert_called_once()
- dump.assert_called_once_with(all_threads=True)
+ dump.assert_called() # dump-file copy first, then stderr (#120356)
assert exit_codes == []
+
+def test_loop_liveness_death_leaves_its_stacks_in_the_dump_file(tmp_path, monkeypatch):
+ # A console-less launcher discards stderr; the file is the only copy that survives (#120356).
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+ exit_codes, ref = [], {}
+
+ def exit_and_stop(code: int) -> None: # a patched exit returns, so stop the loop like a real one
+ exit_codes.append(code)
+ ref["handle"].stop()
+
+ with patch("gateway.shutdown_watchdog._hard_exit", side_effect=exit_and_stop), \
+ patch("gateway.shutdown_watchdog._mark_exited_quietly"):
+ ref["handle"] = handle = start_loop_liveness_watchdog(
+ MagicMock(spec=asyncio.AbstractEventLoop), probe_interval=0.01, probe_timeout=0.01,
+ max_strikes=1)
+ handle.join(timeout=5.0)
+ from gateway.shutdown_watchdog import get_shutdown_watchdog_dump_path
+ dump = get_shutdown_watchdog_dump_path(tmp_path).read_text(encoding="utf-8-sig")
+ assert '"event": "loop_liveness_watchdog_fired"' in dump
+ assert "most recent call first" in dump
+ assert exit_codes == [75]
+
+
def test_loop_liveness_watchdog_stop_during_final_miss_disarms_hard_exit():
loop = MagicMock(spec=asyncio.AbstractEventLoop)
probe_scheduled = threading.Event()