The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(update): the final report credits a unit verification proves restarted; foreign gateways are named once
Tekniumec054e876f81
diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.pyindex d96b5bdfd69..351e2adf3bc 100644--- a/hermes_cli/update_cmd_fleet.py+++ b/hermes_cli/update_cmd_fleet.py@@ -716,23 +716,23 @@ def _systemd_unit_owned_by_update(scope_cmd: list, svc_name: str) -> bool:listed, not because it ran the updated code. Foreign or unreadable ownership prints a noticeand leaves the unit alone; it is not a failed restart."""- from hermes_cli.update_fleet_scope import describe_skipped_runtime, systemd_unit_hermes_home, home_in_update_scope+ from hermes_cli.update_fleet_scope import announce_skipped_runtime, systemd_unit_hermes_home, home_in_update_scopehome = systemd_unit_hermes_home(scope_cmd, svc_name)if home is not None and home_in_update_scope(home):return True- print(describe_skipped_runtime("systemd unit", svc_name, home))+ announce_skipped_runtime("systemd unit", svc_name, home)return Falsedef _scoped_manual_gateway_pids(pids, *, keep=(), quiet: bool = False) -> list[int]:"""*pids* whose live home this update owns (plus *keep*, PIDs already mapped to thisinstall's profile PID files); every other gateway process is named and left running."""- from hermes_cli.update_fleet_scope import describe_skipped_runtime, partition_gateway_pids_by_scope+ from hermes_cli.update_fleet_scope import announce_skipped_runtime, partition_gateway_pids_by_scopekeep = set(keep)owned, foreign = partition_gateway_pids_by_scope([pid for pid in pids if pid not in keep])if not quiet:for pid, home in foreign:- print(describe_skipped_runtime("gateway process", f"PID {pid}", home))+ announce_skipped_runtime("gateway process", f"PID {pid}", home)return [pid for pid in pids if pid in keep or pid in owned]@@ -1468,6 +1468,9 @@ class _GatewayRestartOutcome:#: (``_drain_or_signal_gateway_for_update`` branch 1): they restart only after this process#: exits, so the fleet matrix renders them as pending instead of STALE (#119597).self_restart_pending_pids: set = field(default_factory=set)+ #: Units the restart phase itself could not restart. Verification warns about the ones its fleet+ #: probe does not prove current: a timed-out ``systemctl`` call can still bring its unit up.+ unrestarted_units: list = field(default_factory=list)def fleet_probe_signals(self) -> tuple:"""``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that@@ -1749,7 +1752,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool):if out.failed_or_stale_units:# Owed restart (C3): verification records the follow-up; the /update marker stays committed.out.incomplete = True- _warn_incomplete_gateway_fleet_restart(out.failed_or_stale_units)+ out.unrestarted_units = list(out.failed_or_stale_units)out.record_receipt()_force_kill_stuck_gateways(out.killed_pids)diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.pyindex ddad307e690..f281188078c 100644--- a/hermes_cli/update_cmd_fleet_verify.py+++ b/hermes_cli/update_cmd_fleet_verify.py@@ -129,6 +129,61 @@ def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict:return live+def _gateway_unit_runs_one_of(unit: str, pids: set) -> bool:+ """Every scope that has ``hermes-gateway*`` *unit* loaded runs it as one of *pids*.++ The restart bookkeeping keeps bare unit names, so it cannot say which scope failed: a loaded twin+ in the other scope that is not proven keeps the failure. Anything unreadable proves nothing.+ """+ from hermes_cli import update_cmd_fleet as fleet+ if not pids or not (unit == "hermes-gateway" or unit.startswith("hermes-gateway-")):+ return False+ loaded = False+ for _scope, scope_cmd in fleet._SYSTEMD_SCOPES:+ try:+ stdout = fleet._systemctl(scope_cmd + ["show", "-p", "LoadState,MainPID", unit], timeout=5).stdout+ props = dict(line.split("=", 1) for line in (stdout or "").splitlines() if "=" in line)+ if props.get("LoadState") != "loaded":+ continue+ loaded = True+ if int(props.get("MainPID") or 0) not in pids:+ return False+ except (OSError, subprocess.TimeoutExpired, ValueError):+ return False+ return loaded+++def _settle_units_proven_current(restart, fleet_rows: list) -> bool:+ """Credit each failed gateway unit whose own MainPID the fleet probe shows serving this checkout.++ A restart call can time out while systemd still finishes the job (a unit whose stop takes+ minutes), so the restart phase's "failed" can be a gateway that came up on the new code moments+ later. Proof is the unit's own process: a ``current`` row whose pid is the unit's MainPID and was+ not running when the restart phase began. Another profile's current row never vouches for it.+ Returns True when a unit was credited; the phase's own incompleteness is then recomputed.+ """+ pre = restart.pre_restart_gateway_pids+ if pre is None or not restart.failed_or_stale_units:+ return False+ fresh = {row.get("pid") for row in fleet_rows if row.get("state") == "current"} - set(pre)+ proven = [unit for unit in dict.fromkeys(restart.failed_or_stale_units) if _gateway_unit_runs_one_of(unit, fresh)]+ if not proven:+ return False+ restart.failed_or_stale_units[:] = [u for u in restart.failed_or_stale_units if u not in proven]+ restart.unrestarted_units[:] = [u for u in restart.unrestarted_units if u not in proven]+ restart.restarted_services.extend(u for u in proven if u not in restart.restarted_services)+ for unit in proven:+ print(f" ✓ {unit} finished restarting onto the new code after the restart phase stopped waiting")+ restart.incomplete = bool(restart.failed_or_stale_units or restart.phase_errors)+ restart.record_receipt()+ if not restart.incomplete:+ # The restart phase withheld this stamp; every other profile's CLI reads it.+ from hermes_cli import update_cmd_fleet as fleet+ from hermes_cli.update_host_obligation import mark_host_restart_completed+ mark_host_restart_completed(fleet._restart_identity_sha())+ return True++def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None:"""Name the pre-update gateways on the obligation once their restart is owed.@@ -315,6 +370,9 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re_pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed,)_fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected)+ if _settle_units_proven_current(restart, _fleet_snapshot):+ _gateway_owed = _restart_phase_owes_gateway(restart)+ restart.incomplete = restart.incomplete or bool(_dashboards_down)if print_fleet_version_matrix(_fleet_snapshot):restart.incomplete = _gateway_owed = True# A proven-stale survivor must not keep running (its ticker yields every tick and@@ -335,6 +393,8 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re" gateway runtimes were expected — verification incomplete.")restart.incomplete = _gateway_owed = True+ # Printed after the probe, not by the restart phase: only units it did not prove current.+ fleet._warn_incomplete_gateway_fleet_restart(getattr(restart, "unrestarted_units", None) or [])# Every runtime the PLAN saw must appear in restart bookkeeping; an# unaccounted one is a silent miss and escalates like a STALE/DOWN row.diff --git a/hermes_cli/update_fleet_scope.py b/hermes_cli/update_fleet_scope.pyindex 10c5ad12608..f24d17157c7 100644--- a/hermes_cli/update_fleet_scope.py+++ b/hermes_cli/update_fleet_scope.py@@ -166,3 +166,20 @@ def describe_skipped_runtime(kind: str, name: str, home: str | None) -> str:if home is None:return f" ↷ {name}: {kind} whose Hermes home could not be read — left alone (not restarted)"return f" ↷ {name}: {kind} of another Hermes home ({home}) — left alone (not restarted)"+++def announce_skipped_runtime(kind: str, name: str, home: str | None) -> None:+ """Print :func:`describe_skipped_runtime` once per update run, and name the runtime on its receipt.++ The pre-swap pause and the post-swap restart phase each scope the same host-wide gateway list,+ and the restart phase runs in a fresh completion process, so only the run's receipt (handed to+ that process) knows what the pause already announced. No open receipt: every call prints.+ """+ from hermes_cli import update_receipt+ entry = [kind, name, home]+ current = update_receipt._current.get()+ announced = list((current.data if current is not None else {}).get("runtimes_left_alone") or [])+ if entry in announced:+ return+ print(describe_skipped_runtime(kind, name, home))+ update_receipt.record_fact("runtimes_left_alone", [*announced, entry])diff --git a/tests/hermes_cli/test_update_failed_unit_proven_current.py b/tests/hermes_cli/test_update_failed_unit_proven_current.pynew file mode 100644index 00000000000..88bbf3f862f--- /dev/null+++ b/tests/hermes_cli/test_update_failed_unit_proven_current.py@@ -0,0 +1,91 @@+"""A unit the restart phase recorded failed, but whose own gateway verification proves current.++A ``systemctl`` restart call can time out while systemd still finishes the job (a unit whose stop+takes ~3 minutes). The restart phase then records the unit failed, and the run used to print+"Update incomplete — some units were not restarted" and owe a ``gateway_restart`` follow-up even+after the fleet probe showed the unit's fresh gateway serving the checkout. Only the unit's OWN+process (its MainPID, not running when the restart began) may settle it; a current gateway of+another profile, or the pre-restart process, must not.+"""++from __future__ import annotations++import json+import types++import pytest++from hermes_cli.update_inventory import RuntimeRecord++pytestmark = pytest.mark.platforms("linux")++OLD_PID, NEW_PID, OTHER_PID = 500, 626640, 777+++ ("main_pid", "fleet_rows", "settled"),+ [+ (NEW_PID, [{"profile": "default", "pid": NEW_PID}], True),+ # The unit is dead; an unrelated profile's current gateway is no evidence for it.+ (0, [{"profile": "coder", "pid": OTHER_PID}], False),+ # The unit still runs the process that predates the restart phase.+ (OLD_PID, [{"profile": "default", "pid": OLD_PID}], False),+ ],+ ids=["own-fresh-mainpid", "other-profile-row", "pre-restart-pid"],+)+def test_failed_unit_settles_only_on_its_own_fresh_current_gateway(+ monkeypatch, capsys, main_pid, fleet_rows, settled):+ from hermes_cli import gateway, update_cmd, update_cmd_fleet as fleet, update_cmd_fleet_verify as verify+ from hermes_cli import update_receipt++ # Restart phase: the unit's restart call timed out, so it lands in the failed list.+ monkeypatch.setattr(gateway, "find_gateway_pids", lambda **kw: [OLD_PID])+ monkeypatch.setattr(fleet, "_scoped_manual_gateway_pids", lambda pids, **kw: list(pids))+ monkeypatch.setattr(fleet, "_gateway_drain_budget", lambda: 1.0)+ monkeypatch.setattr(fleet, "_restart_systemd_gateway_units",+ lambda restarted, failed, *args: failed.append("hermes-gateway"))+ monkeypatch.setattr(fleet, "_restart_manual_gateways", lambda out, budget: None)+ monkeypatch.setattr(fleet, "_force_kill_stuck_gateways", lambda killed: None)+ monkeypatch.setattr(fleet, "_restart_identity_sha", lambda: "new")++ # systemd: the unit exists in user scope only, with *main_pid* as its MainPID.+ def systemctl(cmd, timeout=None):+ user = "--user" in cmd+ stdout = f"LoadState=loaded\nMainPID={main_pid}\n" if user else "LoadState=not-found\nMainPID=0\n"+ return types.SimpleNamespace(stdout=stdout, stderr="", returncode=0)++ monkeypatch.setattr(fleet, "_systemctl", systemctl)++ # Verification: every row the probe returns is stamped with the checkout's code.+ rows = [{**row, "code_sha": "new", "code_version": "1", "state": "current"} for row in fleet_rows]+ monkeypatch.setattr(update_receipt, "collect_fleet_versions", lambda **kw: [dict(r) for r in rows])+ monkeypatch.setattr(verify, "_time", types.SimpleNamespace(monotonic=lambda: 0.0, sleep=lambda s: None))+ monkeypatch.setattr(verify, "_print_legacy_units_warning", lambda: None)+ monkeypatch.setattr("hermes_cli.update_cmd_maint._refresh_dashboard_after_update", lambda **kw: None)+ monkeypatch.setattr(update_cmd, "_surviving_pre_update_serve_runtimes", lambda plan: [])+ monkeypatch.setattr("hermes_cli.gateway_migrate.maybe_auto_migrate_after_update", lambda: None)+ cleared = []+ monkeypatch.setattr(fleet, "_clear_fleet_restart_pending_marker", lambda: cleared.append(True))++ plan = types.SimpleNamespace(runtimes=[+ RuntimeRecord(kind="gateway", profile="default", pid=OLD_PID, restart_via="systemd")])+ update_receipt.begin_update_receipt()+ restart = fleet._restart_gateway_fleet_after_update(plan, gateway_mode=False)+ verify._verify_fleet_after_update(restart, _pre_update_plan=plan, _windows_gateway_resume=None,+ update_complete=True)++ out = capsys.readouterr().out+ receipt = json.loads((update_cmd.get_hermes_home() / "logs/update_receipts/latest.json").read_text())+ followups = [f["step"] for f in receipt.get("followups", [])]+ if settled:+ assert "Update incomplete" not in out+ assert followups == []+ assert receipt["gateway_restart"]["incomplete"] is False+ assert receipt["gateway_restart"]["failed_units"] == []+ assert cleared == [True]+ else:+ assert "Update incomplete — some units were not restarted" in out+ assert " - hermes-gateway" in out+ assert followups == ["gateway_restart"]+ assert receipt["gateway_restart"]["incomplete"] is True+ assert cleared == []diff --git a/tests/hermes_cli/test_update_left_alone_notice_once.py b/tests/hermes_cli/test_update_left_alone_notice_once.pynew file mode 100644index 00000000000..e967e87ed0e--- /dev/null+++ b/tests/hermes_cli/test_update_left_alone_notice_once.py@@ -0,0 +1,37 @@+"""A gateway of another Hermes home is announced once per update run, not once per discovery pass.++The pre-swap pause and the post-swap restart phase both scope the host-wide gateway list, and the+restart phase runs in a fresh completion process, so every foreign gateway used to be listed twice+("↷ PID N: gateway process of another Hermes home (...) — left alone"). The run's receipt is the+only state both processes share (the completion request carries it as JSON).+"""++from __future__ import annotations++import contextvars+import json+++def test_left_alone_notice_prints_once_across_the_completion_handoff(monkeypatch, capsys):+ from hermes_cli import update_cmd_fleet as fleet, update_fleet_scope, update_receipt+ from hermes_cli.update_completion import _resume_receipt++ homes = {101: "/srv/other-a/.hermes", 102: "/srv/other-b/.hermes", 103: "/srv/other-c/.hermes"}+ monkeypatch.setattr(update_fleet_scope, "partition_gateway_pids_by_scope",+ lambda pids, scope=None: ([], [(pid, homes[pid]) for pid in pids]))++ update_receipt.begin_update_receipt()+ assert fleet._scoped_manual_gateway_pids([101, 102]) == [] # pause discovery, pre-swap process+ handed_over = json.loads(json.dumps(update_receipt._current.get().data))++ def completion_process():+ # A fresh interpreter: no receipt until it hydrates the one the request carries.+ assert update_receipt._current.get() is None+ _resume_receipt(handed_over)+ # Restart phase: the same two foreign gateways, plus one that appeared since the pause.+ return fleet._scoped_manual_gateway_pids([101, 102, 103])++ assert contextvars.Context().run(completion_process) == []+ out = capsys.readouterr().out+ for pid, home in homes.items():+ assert out.count(f"↷ PID {pid}: gateway process of another Hermes home ({home})") == 1, out