Skip to content

The agent that grows with you

fix(update): the final report credits a unit verification proves restarted; foreign gateways are named once

Tekniumec054e876f81
diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py
index d96b5bdfd69..351e2adf3bc 100644
--- a/hermes_cli/update_cmd_fleet.py
+++ b/hermes_cli/update_cmd_fleet.py
@@ -716,23 +716,23 @@ def _systemd_unit_owned_by_update(scope_cmd: list, svc_name: str) -> bool:
listed, not because it ran the updated code. Foreign or unreadable ownership prints a notice
and leaves the unit alone; it is not a failed restart.
"""
- from hermes_cli.update_fleet_scope import describe_skipped_runtime, systemd_unit_hermes_home, home_in_update_scope
+ from hermes_cli.update_fleet_scope import announce_skipped_runtime, systemd_unit_hermes_home, home_in_update_scope
home = systemd_unit_hermes_home(scope_cmd, svc_name)
if home is not None and home_in_update_scope(home):
return True
- print(describe_skipped_runtime("systemd unit", svc_name, home))
+ announce_skipped_runtime("systemd unit", svc_name, home)
return False
def _scoped_manual_gateway_pids(pids, *, keep=(), quiet: bool = False) -> list[int]:
"""*pids* whose live home this update owns (plus *keep*, PIDs already mapped to this
install's profile PID files); every other gateway process is named and left running."""
- from hermes_cli.update_fleet_scope import describe_skipped_runtime, partition_gateway_pids_by_scope
+ from hermes_cli.update_fleet_scope import announce_skipped_runtime, partition_gateway_pids_by_scope
keep = set(keep)
owned, foreign = partition_gateway_pids_by_scope([pid for pid in pids if pid not in keep])
if not quiet:
for pid, home in foreign:
- print(describe_skipped_runtime("gateway process", f"PID {pid}", home))
+ announce_skipped_runtime("gateway process", f"PID {pid}", home)
return [pid for pid in pids if pid in keep or pid in owned]
@@ -1468,6 +1468,9 @@ class _GatewayRestartOutcome:
#: (``_drain_or_signal_gateway_for_update`` branch 1): they restart only after this process
#: exits, so the fleet matrix renders them as pending instead of STALE (#119597).
self_restart_pending_pids: set = field(default_factory=set)
+ #: Units the restart phase itself could not restart. Verification warns about the ones its fleet
+ #: probe does not prove current: a timed-out ``systemctl`` call can still bring its unit up.
+ unrestarted_units: list = field(default_factory=list)
def fleet_probe_signals(self) -> tuple:
"""``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that
@@ -1749,7 +1752,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool):
if out.failed_or_stale_units:
# Owed restart (C3): verification records the follow-up; the /update marker stays committed.
out.incomplete = True
- _warn_incomplete_gateway_fleet_restart(out.failed_or_stale_units)
+ out.unrestarted_units = list(out.failed_or_stale_units)
out.record_receipt()
_force_kill_stuck_gateways(out.killed_pids)
diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py
index ddad307e690..f281188078c 100644
--- a/hermes_cli/update_cmd_fleet_verify.py
+++ b/hermes_cli/update_cmd_fleet_verify.py
@@ -129,6 +129,61 @@ def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict:
return live
+def _gateway_unit_runs_one_of(unit: str, pids: set) -> bool:
+ """Every scope that has ``hermes-gateway*`` *unit* loaded runs it as one of *pids*.
+
+ The restart bookkeeping keeps bare unit names, so it cannot say which scope failed: a loaded twin
+ in the other scope that is not proven keeps the failure. Anything unreadable proves nothing.
+ """
+ from hermes_cli import update_cmd_fleet as fleet
+ if not pids or not (unit == "hermes-gateway" or unit.startswith("hermes-gateway-")):
+ return False
+ loaded = False
+ for _scope, scope_cmd in fleet._SYSTEMD_SCOPES:
+ try:
+ stdout = fleet._systemctl(scope_cmd + ["show", "-p", "LoadState,MainPID", unit], timeout=5).stdout
+ props = dict(line.split("=", 1) for line in (stdout or "").splitlines() if "=" in line)
+ if props.get("LoadState") != "loaded":
+ continue
+ loaded = True
+ if int(props.get("MainPID") or 0) not in pids:
+ return False
+ except (OSError, subprocess.TimeoutExpired, ValueError):
+ return False
+ return loaded
+
+
+def _settle_units_proven_current(restart, fleet_rows: list) -> bool:
+ """Credit each failed gateway unit whose own MainPID the fleet probe shows serving this checkout.
+
+ A restart call can time out while systemd still finishes the job (a unit whose stop takes
+ minutes), so the restart phase's "failed" can be a gateway that came up on the new code moments
+ later. Proof is the unit's own process: a ``current`` row whose pid is the unit's MainPID and was
+ not running when the restart phase began. Another profile's current row never vouches for it.
+ Returns True when a unit was credited; the phase's own incompleteness is then recomputed.
+ """
+ pre = restart.pre_restart_gateway_pids
+ if pre is None or not restart.failed_or_stale_units:
+ return False
+ fresh = {row.get("pid") for row in fleet_rows if row.get("state") == "current"} - set(pre)
+ proven = [unit for unit in dict.fromkeys(restart.failed_or_stale_units) if _gateway_unit_runs_one_of(unit, fresh)]
+ if not proven:
+ return False
+ restart.failed_or_stale_units[:] = [u for u in restart.failed_or_stale_units if u not in proven]
+ restart.unrestarted_units[:] = [u for u in restart.unrestarted_units if u not in proven]
+ restart.restarted_services.extend(u for u in proven if u not in restart.restarted_services)
+ for unit in proven:
+ print(f" ✓ {unit} finished restarting onto the new code after the restart phase stopped waiting")
+ restart.incomplete = bool(restart.failed_or_stale_units or restart.phase_errors)
+ restart.record_receipt()
+ if not restart.incomplete:
+ # The restart phase withheld this stamp; every other profile's CLI reads it.
+ from hermes_cli import update_cmd_fleet as fleet
+ from hermes_cli.update_host_obligation import mark_host_restart_completed
+ mark_host_restart_completed(fleet._restart_identity_sha())
+ return True
+
+
def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None:
"""Name the pre-update gateways on the obligation once their restart is owed.
@@ -315,6 +370,9 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re
_pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed,
)
_fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected)
+ if _settle_units_proven_current(restart, _fleet_snapshot):
+ _gateway_owed = _restart_phase_owes_gateway(restart)
+ restart.incomplete = restart.incomplete or bool(_dashboards_down)
if print_fleet_version_matrix(_fleet_snapshot):
restart.incomplete = _gateway_owed = True
# A proven-stale survivor must not keep running (its ticker yields every tick and
@@ -335,6 +393,8 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re
" gateway runtimes were expected — verification incomplete."
)
restart.incomplete = _gateway_owed = True
+ # Printed after the probe, not by the restart phase: only units it did not prove current.
+ fleet._warn_incomplete_gateway_fleet_restart(getattr(restart, "unrestarted_units", None) or [])
# Every runtime the PLAN saw must appear in restart bookkeeping; an
# unaccounted one is a silent miss and escalates like a STALE/DOWN row.
diff --git a/hermes_cli/update_fleet_scope.py b/hermes_cli/update_fleet_scope.py
index 10c5ad12608..f24d17157c7 100644
--- a/hermes_cli/update_fleet_scope.py
+++ b/hermes_cli/update_fleet_scope.py
@@ -166,3 +166,20 @@ def describe_skipped_runtime(kind: str, name: str, home: str | None) -> str:
if home is None:
return f" ↷ {name}: {kind} whose Hermes home could not be read — left alone (not restarted)"
return f" ↷ {name}: {kind} of another Hermes home ({home}) — left alone (not restarted)"
+
+
+def announce_skipped_runtime(kind: str, name: str, home: str | None) -> None:
+ """Print :func:`describe_skipped_runtime` once per update run, and name the runtime on its receipt.
+
+ The pre-swap pause and the post-swap restart phase each scope the same host-wide gateway list,
+ and the restart phase runs in a fresh completion process, so only the run's receipt (handed to
+ that process) knows what the pause already announced. No open receipt: every call prints.
+ """
+ from hermes_cli import update_receipt
+ entry = [kind, name, home]
+ current = update_receipt._current.get()
+ announced = list((current.data if current is not None else {}).get("runtimes_left_alone") or [])
+ if entry in announced:
+ return
+ print(describe_skipped_runtime(kind, name, home))
+ update_receipt.record_fact("runtimes_left_alone", [*announced, entry])
diff --git a/tests/hermes_cli/test_update_failed_unit_proven_current.py b/tests/hermes_cli/test_update_failed_unit_proven_current.py
new file mode 100644
index 00000000000..88bbf3f862f
--- /dev/null
+++ b/tests/hermes_cli/test_update_failed_unit_proven_current.py
@@ -0,0 +1,91 @@
+"""A unit the restart phase recorded failed, but whose own gateway verification proves current.
+
+A ``systemctl`` restart call can time out while systemd still finishes the job (a unit whose stop
+takes ~3 minutes). The restart phase then records the unit failed, and the run used to print
+"Update incomplete — some units were not restarted" and owe a ``gateway_restart`` follow-up even
+after the fleet probe showed the unit's fresh gateway serving the checkout. Only the unit's OWN
+process (its MainPID, not running when the restart began) may settle it; a current gateway of
+another profile, or the pre-restart process, must not.
+"""
+
+from __future__ import annotations
+
+import json
+import types
+
+import pytest
+
+from hermes_cli.update_inventory import RuntimeRecord
+
+pytestmark = pytest.mark.platforms("linux")
+
+OLD_PID, NEW_PID, OTHER_PID = 500, 626640, 777
+
+
+ ("main_pid", "fleet_rows", "settled"),
+ [
+ (NEW_PID, [{"profile": "default", "pid": NEW_PID}], True),
+ # The unit is dead; an unrelated profile's current gateway is no evidence for it.
+ (0, [{"profile": "coder", "pid": OTHER_PID}], False),
+ # The unit still runs the process that predates the restart phase.
+ (OLD_PID, [{"profile": "default", "pid": OLD_PID}], False),
+ ],
+ ids=["own-fresh-mainpid", "other-profile-row", "pre-restart-pid"],
+)
+def test_failed_unit_settles_only_on_its_own_fresh_current_gateway(
+ monkeypatch, capsys, main_pid, fleet_rows, settled):
+ from hermes_cli import gateway, update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as verify
+ from hermes_cli import update_receipt
+
+ # Restart phase: the unit's restart call timed out, so it lands in the failed list.
+ monkeypatch.setattr(gateway, "find_gateway_pids", lambda **kw: [OLD_PID])
+ monkeypatch.setattr(fleet, "_scoped_manual_gateway_pids", lambda pids, **kw: list(pids))
+ monkeypatch.setattr(fleet, "_gateway_drain_budget", lambda: 1.0)
+ monkeypatch.setattr(fleet, "_restart_systemd_gateway_units",
+ lambda restarted, failed, *args: failed.append("hermes-gateway"))
+ monkeypatch.setattr(fleet, "_restart_manual_gateways", lambda out, budget: None)
+ monkeypatch.setattr(fleet, "_force_kill_stuck_gateways", lambda killed: None)
+ monkeypatch.setattr(fleet, "_restart_identity_sha", lambda: "new")
+
+ # systemd: the unit exists in user scope only, with *main_pid* as its MainPID.
+ def systemctl(cmd, timeout=None):
+ user = "--user" in cmd
+ stdout = f"LoadState=loaded\nMainPID={main_pid}\n" if user else "LoadState=not-found\nMainPID=0\n"
+ return types.SimpleNamespace(stdout=stdout, stderr="", returncode=0)
+
+ monkeypatch.setattr(fleet, "_systemctl", systemctl)
+
+ # Verification: every row the probe returns is stamped with the checkout's code.
+ rows = [{**row, "code_sha": "new", "code_version": "1", "state": "current"} for row in fleet_rows]
+ monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: [dict(r) for r in rows])
+ monkeypatch.setattr(verify, "_time", types.SimpleNamespace(monotonic=lambda: 0.0, sleep=lambda s: None))
+ monkeypatch.setattr(verify, "_print_legacy_units_warning", lambda: None)
+ monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None)
+ monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: [])
+ monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None)
+ cleared = []
+ monkeypatch.setattr(fleet, "_clear_fleet_restart_pending_marker", lambda: cleared.append(True))
+
+ plan = types.SimpleNamespace(runtimes=[
+ RuntimeRecord(kind="gateway", profile="default", pid=OLD_PID, restart_via="systemd")])
+ update_receipt.begin_update_receipt()
+ restart = fleet._restart_gateway_fleet_after_update(plan, gateway_mode=False)
+ verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None,
+ update_complete=True)
+
+ out = capsys.readouterr().out
+ receipt = json.loads((update_cmd.get_hermes_home() / "logs/update_receipts/latest.json").read_text())
+ followups = [f["step"] for f in receipt.get("followups", [])]
+ if settled:
+ assert "Update incomplete" not in out
+ assert followups == []
+ assert receipt["gateway_restart"]["incomplete"] is False
+ assert receipt["gateway_restart"]["failed_units"] == []
+ assert cleared == [True]
+ else:
+ assert "Update incomplete — some units were not restarted" in out
+ assert " - hermes-gateway" in out
+ assert followups == ["gateway_restart"]
+ assert receipt["gateway_restart"]["incomplete"] is True
+ assert cleared == []
diff --git a/tests/hermes_cli/test_update_left_alone_notice_once.py b/tests/hermes_cli/test_update_left_alone_notice_once.py
new file mode 100644
index 00000000000..e967e87ed0e
--- /dev/null
+++ b/tests/hermes_cli/test_update_left_alone_notice_once.py
@@ -0,0 +1,37 @@
+"""A gateway of another Hermes home is announced once per update run, not once per discovery pass.
+
+The pre-swap pause and the post-swap restart phase both scope the host-wide gateway list, and the
+restart phase runs in a fresh completion process, so every foreign gateway used to be listed twice
+("↷ PID N: gateway process of another Hermes home (...) — left alone"). The run's receipt is the
+only state both processes share (the completion request carries it as JSON).
+"""
+
+from __future__ import annotations
+
+import contextvars
+import json
+
+
+def test_left_alone_notice_prints_once_across_the_completion_handoff(monkeypatch, capsys):
+ from hermes_cli import update_cmd_fleet as fleet, update_fleet_scope, update_receipt
+ from hermes_cli.update_completion import _resume_receipt
+
+ homes = {101: "/srv/other-a/.hermes", 102: "/srv/other-b/.hermes", 103: "/srv/other-c/.hermes"}
+ monkeypatch.setattr(update_fleet_scope, "partition_gateway_pids_by_scope",
+ lambda pids, scope=None: ([], [(pid, homes[pid]) for pid in pids]))
+
+ update_receipt.begin_update_receipt()
+ assert fleet._scoped_manual_gateway_pids([101, 102]) == [] # pause discovery, pre-swap process
+ handed_over = json.loads(json.dumps(update_receipt._current.get().data))
+
+ def completion_process():
+ # A fresh interpreter: no receipt until it hydrates the one the request carries.
+ assert update_receipt._current.get() is None
+ _resume_receipt(handed_over)
+ # Restart phase: the same two foreign gateways, plus one that appeared since the pause.
+ return fleet._scoped_manual_gateway_pids([101, 102, 103])
+
+ assert contextvars.Context().run(completion_process) == []
+ out = capsys.readouterr().out
+ for pid, home in homes.items():
+ assert out.count(f"↷ PID {pid}: gateway process of another Hermes home ({home})") == 1, out