Skip to content

The agent that grows with you

fix(gateway): one owner for the planned-restart marker, stamped after the old gateway is gone

Leegenux1205a010f769
diff --git a/gateway/restart_notice.py b/gateway/restart_notice.py
index 83574d1f2bb..c00f7ea9cb4 100644
--- a/gateway/restart_notice.py
+++ b/gateway/restart_notice.py
@@ -23,13 +23,16 @@ logger = logging.getLogger(__name__)
PLANNED_RESTART_MARKER_FILENAME = ".restart_pending.json"
-def planned_restart_marker_path() -> Path:
- return get_process_hermes_home() / PLANNED_RESTART_MARKER_FILENAME
+def planned_restart_marker_path(home: Path | None = None) -> Path:
+ return (home or get_process_hermes_home()) / PLANNED_RESTART_MARKER_FILENAME
-def write_planned_restart_marker() -> None:
+def write_planned_restart_marker(*, via_service: bool = False, detached: bool = False) -> None:
"""Record that the running gateway is going down FOR A RESTART; never blocks the restart.
+ One writer for both restart kinds: the gateway's own in-band shutdown (``via_service`` /
+ ``detached`` from its shutdown context) and the CLI, which has no in-band path of its own.
+
Best-effort like ``gateway.status.write_planned_stop_marker``: an unwritable home costs the
notice, not the restart. Wider than ``OSError`` on purpose — a marker write that raises for any
other reason must still not leave the caller's ``stop()`` un-run.
@@ -37,7 +40,11 @@ def write_planned_restart_marker() -> None:
try:
atomic_json_write(
planned_restart_marker_path(),
- {"requested_at": time.time(), "via_service": False, "detached": False},
+ {
+ "requested_at": time.time(),
+ "via_service": bool(via_service),
+ "detached": bool(detached),
+ },
indent=None,
)
except Exception:
diff --git a/gateway/run.py b/gateway/run.py
index 3b79fcb8d75..9a26dd649a7 100644
--- a/gateway/run.py
+++ b/gateway/run.py
@@ -1559,11 +1559,11 @@ def _restart_notification_pending() -> bool:
def _planned_restart_notification_path() -> Path:
- return _hermes_home / ".restart_pending.json"
+ from gateway.restart_notice import planned_restart_marker_path
+ return planned_restart_marker_path(_hermes_home)
def _planned_restart_notification_pending() -> bool:
- """Return True when a non-chat planned restart should notify home channels."""
return _planned_restart_notification_path().exists()
diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py
index 2ba066193f4..5349c173f87 100644
--- a/gateway/run_shutdown.py
+++ b/gateway/run_shutdown.py
@@ -2109,8 +2109,7 @@ class GatewayShutdownMixin(GatewaySessionEndMixin):
async def _stop_persist_exit_state(self, ctx: GatewayShutdownMixin._StopContext) -> None:
"""Clean-shutdown marker, restart markers, terminal runtime status; process bootstrap
releases PID/lock ownership after the writer drain."""
- from gateway.run import _hermes_home, _planned_restart_notification_path, _shutdown_gateway_health_export
- from utils import atomic_json_write
+ from gateway.run import _hermes_home, _shutdown_gateway_health_export
if getattr(self, "_startup_cleanup_only", False): # start never ran: keep startup_failed + crash markers
return _shutdown_gateway_health_export(self)
# Clean-shutdown marker skips crash-turn recovery next boot; a timed-out drain left
@@ -2125,15 +2124,13 @@ class GatewayShutdownMixin(GatewaySessionEndMixin):
if ctx.active_agents:
self._increment_restart_failure_counts(set(ctx.active_agents.keys()))
if self._restart_requested and self._restart_command_source is None:
+ # One writer for the marker (gateway/restart_notice.py): the CLI restart stamps the same
+ # file, and a rename on either side must not strand the other's write.
+ from gateway.restart_notice import write_planned_restart_marker
with _log_suppressed(logging.DEBUG, "Failed to write planned restart notification marker: %s"):
- atomic_json_write(
- _planned_restart_notification_path(),
- {
- "requested_at": time.time(),
- "via_service": bool(self._restart_via_service),
- "detached": bool(self._restart_detached),
- },
- indent=None,
+ write_planned_restart_marker(
+ via_service=bool(self._restart_via_service),
+ detached=bool(self._restart_detached),
)
if self._restart_requested and self._restart_via_service:
# Exit 75 + ``RestartForceExitStatus=75``: systemd replaces us without a racing helper.
diff --git a/hermes_cli/gateway_windows.py b/hermes_cli/gateway_windows.py
index f49d90e3cf2..fcbd9470f9f 100644
--- a/hermes_cli/gateway_windows.py
+++ b/hermes_cli/gateway_windows.py
@@ -1863,11 +1863,15 @@ def _wait_for_gateway_absent(timeout_s: float = 30.0, interval_s: float = 0.5) -
def _mark_planned_restart() -> None:
- """Stamp the planned-restart marker before ``stop()``.
+ """Stamp the planned-restart marker once the old gateway is gone, before ``start()``.
Windows has no SIGUSR1, so ``restart()`` is stop + start: the gateway never runs the in-band
restart path that stamps this marker itself, and the next boot would read a plain stop and skip
the "Gateway online" home-channel notice.
+
+ Not before ``stop()``: the old process stays alive until its planned-stop watcher notices, and
+ for that window a flapping platform can reconnect, replay the notice and unlink the marker the
+ new process then never finds.
"""
from gateway.restart_notice import write_planned_restart_marker
write_planned_restart_marker()
@@ -1879,7 +1883,6 @@ def restart() -> None:
replaces it when it exits (a silent outage). Fails loudly on either side."""
_assert_windows()
- _mark_planned_restart()
stop()
if not _wait_for_gateway_absent(timeout_s=30.0):
@@ -1894,6 +1897,7 @@ def restart() -> None:
from hermes_cli.gateway import _wait_for_api_server_port_free # avoid circular init
_wait_for_api_server_port_free()
+ _mark_planned_restart()
start()
if not _wait_for_gateway_ready(timeout_s=15.0):
diff --git a/tests/gateway/test_planned_restart_marker_cli.py b/tests/gateway/test_planned_restart_marker_cli.py
index af671d70b65..0ea22ecf4c2 100644
--- a/tests/gateway/test_planned_restart_marker_cli.py
+++ b/tests/gateway/test_planned_restart_marker_cli.py
@@ -28,77 +28,7 @@ def test_writer_stamps_the_marker_the_boot_pass_reads(hermes_home):
assert gateway_run._planned_restart_notification_pending() is False
- write_planned_restart_marker()
+ write_planned_restart_marker(via_service=True, detached=False)
assert (hermes_home / MARKER).exists()
assert gateway_run._planned_restart_notification_pending() is True
-
-
-def test_windows_restart_stamps_the_marker_before_stopping(hermes_home, monkeypatch):
- """Ordering is the whole fix: ``stop()`` must already see the marker on disk."""
- from hermes_cli import gateway as gw_facade
- from hermes_cli import gateway_windows as gw
-
- seen: dict[str, bool] = {}
- monkeypatch.setattr(gw, "_assert_windows", lambda: None)
- monkeypatch.setattr(gw, "_wait_for_gateway_absent", lambda timeout_s=30.0: True)
- monkeypatch.setattr(gw, "_wait_for_gateway_ready", lambda timeout_s=15.0: True)
- monkeypatch.setattr(gw_facade, "_wait_for_api_server_port_free", lambda: None)
- monkeypatch.setattr(
- gw, "stop", lambda: seen.setdefault("marker_at_stop", (hermes_home / MARKER).exists())
- )
- monkeypatch.setattr(gw, "start", lambda: None)
-
- gw.restart()
-
- assert seen["marker_at_stop"] is True
-
-
-def test_failed_restart_still_leaves_the_marker(hermes_home, monkeypatch):
- """A restart that never comes back must not swallow the notice.
-
- The marker is what tells the *next* boot that a return is owed, so it has to survive a
- ``start()`` that raises — otherwise a crash-looping gateway goes silent exactly when the user
- most needs to know it is down.
- """
- from hermes_cli import gateway as gw_facade
- from hermes_cli import gateway_windows as gw
-
- def not_ready(timeout_s=15.0):
- raise RuntimeError("Gateway restart did not produce a running gateway process.")
-
- monkeypatch.setattr(gw, "_assert_windows", lambda: None)
- monkeypatch.setattr(gw, "stop", lambda: None)
- monkeypatch.setattr(gw, "_wait_for_gateway_absent", lambda timeout_s=30.0: True)
- monkeypatch.setattr(gw_facade, "_wait_for_api_server_port_free", lambda: None)
- monkeypatch.setattr(gw, "start", lambda: None)
- monkeypatch.setattr(gw, "_wait_for_gateway_ready", not_ready)
-
- with pytest.raises(RuntimeError):
- gw.restart()
-
- assert (hermes_home / MARKER).exists()
-
-
-def test_an_unwritable_marker_does_not_block_the_restart(hermes_home, monkeypatch):
- """Losing the notice is acceptable; losing the restart is not."""
- from gateway import restart_notice
- from hermes_cli import gateway as gw_facade
- from hermes_cli import gateway_windows as gw
-
- stopped: list[str] = []
-
- def boom(*_a, **_kw):
- raise RuntimeError("disk full")
-
- monkeypatch.setattr(restart_notice, "atomic_json_write", boom)
- monkeypatch.setattr(gw, "_assert_windows", lambda: None)
- monkeypatch.setattr(gw, "stop", lambda: stopped.append("stop"))
- monkeypatch.setattr(gw, "_wait_for_gateway_absent", lambda timeout_s=30.0: True)
- monkeypatch.setattr(gw_facade, "_wait_for_api_server_port_free", lambda: None)
- monkeypatch.setattr(gw, "start", lambda: stopped.append("start"))
- monkeypatch.setattr(gw, "_wait_for_gateway_ready", lambda timeout_s=15.0: True)
-
- gw.restart()
-
- assert stopped == ["stop", "start"]
diff --git a/tests/hermes_cli/test_gateway_windows_restart_marker.py b/tests/hermes_cli/test_gateway_windows_restart_marker.py
new file mode 100644
index 00000000000..eae2faf675c
--- /dev/null
+++ b/tests/hermes_cli/test_gateway_windows_restart_marker.py
@@ -0,0 +1,55 @@
+"""The Windows CLI restart stamps the planned-restart marker — after the old gateway is gone.
+
+``hermes gateway restart`` on Windows is ``stop()`` + ``start()`` (no SIGUSR1), so nothing else would
+ever write ``.restart_pending.json`` and the next boot would skip the "Gateway online" home-channel
+notice. The marker has to land *after* the old process is gone — written before ``stop()`` it is
+visible to a still-draining gateway, which can replay the notice and unlink it — and *before*
+``start()``, so a restart that never comes back still leaves it for the next boot.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from gateway import run as gateway_run
+
+MARKER = ".restart_pending.json"
+
+
+def hermes_home(tmp_path, monkeypatch):
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+ monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
+ return tmp_path
+
+
+def test_restart_stamps_the_marker_after_the_old_gateway_is_gone(hermes_home, monkeypatch):
+ from hermes_cli import gateway as gw_facade
+ from hermes_cli import gateway_windows as gw
+
+ marker = hermes_home / MARKER
+ seen: dict[str, bool] = {}
+ monkeypatch.setattr(gw, "_assert_windows", lambda: None)
+ monkeypatch.setattr(gw, "_wait_for_gateway_absent", lambda timeout_s=30.0: True)
+ monkeypatch.setattr(gw_facade, "_wait_for_api_server_port_free", lambda: None)
+ monkeypatch.setattr(gw, "stop", lambda: seen.setdefault("marker_at_stop", marker.exists()))
+ monkeypatch.setattr(gw, "start", lambda: seen.setdefault("marker_at_start", marker.exists()))
+ monkeypatch.setattr(gw, "_wait_for_gateway_ready", lambda timeout_s=15.0: True)
+
+ gw.restart()
+
+ assert seen["marker_at_stop"] is False, "the still-draining gateway must not see the marker"
+ assert seen["marker_at_start"] is True, "the next boot pass must find it"
+
+ # A restart that never comes back must keep the marker: the notice is still owed.
+ monkeypatch.setattr(gw, "stop", lambda: None)
+ marker.unlink()
+
+ def not_ready(timeout_s=15.0):
+ raise RuntimeError("Gateway restart did not produce a running gateway process.")
+
+ monkeypatch.setattr(gw, "_wait_for_gateway_ready", not_ready)
+ with pytest.raises(RuntimeError):
+ gw.restart()
+
+ assert marker.exists()