Skip to content

The agent that grows with you

fix(gateway): park a unit on unprovable --replace ownership instead of restart-looping

salch-cred7609103302e4
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py
index b415b10799f..818af836e79 100644
--- a/gateway/run_bootstrap.py
+++ b/gateway/run_bootstrap.py
@@ -29,7 +29,9 @@ logger = logging.getLogger("gateway.run")
async def _start_gateway_replace_existing_instance(existing_pid: int, replace: bool) -> bool:
"""Handle a live gateway PID under this HERMES_HOME: replace it (``--replace``) or refuse.
- Returns False when startup must abort (refused, permission denied, target still alive)."""
+ Returns False when startup must abort (refused, permission denied, target still alive).
+ Raises ``SystemExit(GATEWAY_FATAL_CONFIG_EXIT_CODE)`` when ownership is unprovable: that verdict
+ is decided by the persisted identity record alone, so a retry can never change it."""
from gateway.run import (_clear_takeover_marker_quiet, _replace_target_belongs_to_other_profile, _wait_for_pid_exit, get_hermes_home, suppress)
from gateway.status import get_process_start_time, remove_pid_file, terminate_pid
if not replace:
@@ -49,13 +51,22 @@ async def _start_gateway_replace_existing_instance(existing_pid: int, replace: b
# Never signal a process not provably ours (a poisoned PID record → cross-profile restart loop).
if _replace_target_belongs_to_other_profile(existing_pid):
+ from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
+ from gateway.run import _write_runtime_status_quiet
from gateway.status import _get_process_hermes_home
logger.error(
"Refusing --replace: PID %d cannot be proven to belong "
"to this profile's gateway (HERMES_HOME %s). Remove the "
"stale PID record or stop the owning profile explicitly.",
existing_pid, _get_process_hermes_home())
- return False
+ # The verdict is decided by the persisted identity record alone, so no supervisor retry can
+ # change it: a bare exit 1 left a Restart=always unit looping on a PID record that will never
+ # prove ours (#135392). Stamp the fatal state and exit EX_CONFIG so systemd's
+ # RestartPreventExitStatus=78 parks the unit and s6 maps 78 → 125 (permanent failure), and
+ # /api/status reports startup_failed instead of a stale heartbeat.
+ _write_runtime_status_quiet(
+ gateway_state="startup_failed", exit_reason="gateway_replace_ownership_unproven")
+ raise SystemExit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
existing_start_time = get_process_start_time(existing_pid)
logger.info("Replacing existing gateway instance (PID %d) with --replace.", existing_pid)
# Takeover marker: target exits 0 on our SIGTERM (exit 1 → systemd Restart=on-failure flap loop).
diff --git a/tests/gateway/test_replace_ownership_guard.py b/tests/gateway/test_replace_ownership_guard.py
index 351fcec6977..e4f1d24bf3d 100644
--- a/tests/gateway/test_replace_ownership_guard.py
+++ b/tests/gateway/test_replace_ownership_guard.py
@@ -15,9 +15,13 @@ Pinned surfaces:
``tim`` (the substring heuristic's false-allow), while an exact different
profile flag contradicts and refuses;
* signal boundary — ``start_gateway(replace=True)`` on unprovable ownership
- returns refusal without calling ``terminate_pid`` or writing a takeover
- marker; the legitimate bound same-home target still reaches the replace
- flow.
+ refuses without calling ``terminate_pid`` or writing a takeover marker; the
+ legitimate bound same-home target still reaches the replace flow;
+* fatal refusal (#135392) — the unprovable-ownership refusal is decided by the
+ persisted identity record alone, so it stamps ``gateway_state: startup_failed``
+ (with an ``exit_reason`` code) and exits ``GATEWAY_FATAL_CONFIG_EXIT_CODE``
+ (78) instead of a bare exit 1, so a Restart=always unit parks rather than
+ restart-looping on the same record.
"""
from __future__ import annotations
@@ -301,7 +305,7 @@ class TestSignalBoundary:
def _run_replace(self, agent_patches):
from gateway import run as gateway_run
- calls = {"terminate": 0, "marker": 0}
+ calls = {"terminate": 0, "marker": 0, "status": []}
def _fake_terminate(pid, force=False):
calls["terminate"] += 1
@@ -309,9 +313,13 @@ class TestSignalBoundary:
def _fake_marker(pid):
calls["marker"] += 1
+ def _fake_status(**fields):
+ calls["status"].append(fields)
+
base = [
patch("gateway.status.get_running_pid", return_value=424242),
patch.object(gateway_run, "_replace_target_belongs_to_other_profile"),
+ patch.object(gateway_run, "_write_runtime_status_quiet", side_effect=_fake_status),
patch("gateway.status.terminate_pid", side_effect=_fake_terminate),
patch("gateway.status.write_takeover_marker", side_effect=_fake_marker),
]
@@ -324,14 +332,16 @@ class TestSignalBoundary:
agent_patches(stack)
try:
result = asyncio_run(gateway_run.start_gateway(replace=True))
+ except SystemExit as exc:
+ result = exc.code
except Exception:
result = "raised"
return result, calls
def test_unprovable_ownership_never_signals(self, profile_env):
- """Unprovable ownership → start_gateway returns False WITHOUT calling
- terminate_pid or writing a takeover marker."""
- from unittest.mock import MagicMock
+ """Unprovable ownership → start_gateway exits fatally (EX_CONFIG) WITHOUT
+ calling terminate_pid or writing a takeover marker."""
+ from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
def configure(stack):
guard = stack.enter_context(
@@ -344,7 +354,10 @@ class TestSignalBoundary:
result, calls = self._run_replace(lambda s: configure(s))
- assert result is False
+ assert result == GATEWAY_FATAL_CONFIG_EXIT_CODE, (
+ "an unprovable-ownership refusal is permanent: exit 78 to park the unit, "
+ "never a bare exit 1 that restarts into the same refusal"
+ )
assert calls["terminate"] == 0, (
"--replace must not signal a target whose ownership is unproven"
)
@@ -352,6 +365,24 @@ class TestSignalBoundary:
"no takeover marker may be written for a refused target"
)
+ def test_unprovable_ownership_stamps_startup_failed(self, profile_env):
+ """The fatal refusal publishes the retained fatal state + an exit reason,
+ so /api/status reports startup_failed instead of a stale heartbeat."""
+ def configure(stack):
+ stack.enter_context(
+ patch(
+ "gateway.run._replace_target_belongs_to_other_profile",
+ return_value=True,
+ )
+ )
+
+ _result, calls = self._run_replace(configure)
+
+ assert calls["status"], "the ownership refusal must stamp runtime status"
+ stamp = calls["status"][-1]
+ assert stamp.get("gateway_state") == "startup_failed"
+ assert stamp.get("exit_reason") == "gateway_replace_ownership_unproven"
+
def test_provable_same_home_reaches_replace_flow(self, profile_env):
"""Counterpart: bound same-home target still enters the replace flow
(terminate attempted) — the fail-closed gate must not disable legit
diff --git a/tests/gateway/test_runner_startup_failures.py b/tests/gateway/test_runner_startup_failures.py
index 0743da8e710..1ff50de1ac4 100644
--- a/tests/gateway/test_runner_startup_failures.py
+++ b/tests/gateway/test_runner_startup_failures.py
@@ -351,6 +351,21 @@ async def test_start_gateway_replace_clears_marker_on_permission_denied(
raise PermissionError("simulated EPERM")
monkeypatch.setattr("gateway.status.get_running_pid", lambda: 42)
+ # Ownership guard (#89315): a bound same-home record for target pid 42, so the replace flow
+ # actually reaches the SIGTERM (and its EPERM) instead of refusing on unprovable ownership.
+ monkeypatch.setattr(
+ "gateway.status._read_pid_record",
+ lambda path=None: {
+ "pid": 42,
+ "kind": "hermes-gateway",
+ "argv": ["python", "-m", "hermes_cli.main", "gateway", "run"],
+ "start_time": 0,
+ "hermes_home": str(tmp_path),
+ },
+ )
+ monkeypatch.setattr(
+ "gateway.status._get_process_start_time", lambda pid: 0 if pid == 42 else None
+ )
monkeypatch.setattr("gateway.status.write_takeover_marker", write_marker)
monkeypatch.setattr("gateway.status.terminate_pid", raise_permission)
monkeypatch.setattr("gateway.run.os.getpid", lambda: 100)