Skip to content

The agent that grows with you

fix(update): unit-owned serve reminders ride the receipt-backed flow; linux markers

kshitijk4poor338207240132
diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py
index 8917ce57c6b..6cc4b201c42 100644
--- a/hermes_cli/update_cmd_fleet_verify.py
+++ b/hermes_cli/update_cmd_fleet_verify.py
@@ -365,16 +365,11 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re
live_gateway_pids=_live_gateway_pids,
)
from dataclasses import asdict
- from hermes_cli.update_serve_obligations import defer_manual_serve, remind_unit_serve
+ from hermes_cli.update_serve_obligations import defer_manual_serve
- _stale_pids = {row.get("pid") for row in _stale_serve_rows or ()}
for runtime, outcome in zip(_pre_update_plan.runtimes, _runtime_outcomes):
if outcome["outcome"] == "unaccounted" and defer_manual_serve(asdict(runtime), require_alive=True):
outcome["outcome"] = "deferred"
- elif runtime.pid in _stale_pids and runtime.supervisor == "systemd":
- # A unit the update could not restart keeps a startup reminder with its exact
- # command; the gateway obligation never names it (#141021).
- remind_unit_serve(asdict(runtime))
if report_unaccounted_runtimes(_runtime_outcomes):
restart.incomplete = True
_missed_gateway, _missed_serves = _split_unaccounted(_runtime_outcomes)
diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py
index 3166b2fc867..f502b51113d 100644
--- a/hermes_cli/update_receipt.py
+++ b/hermes_cli/update_receipt.py
@@ -380,8 +380,9 @@ def begin_update_receipt(*, previous: dict | None = None, correlation_id: str |
rows = list((prior.get("plan") or {}).get("runtimes") or [])
rows += list(prior.get("pending_manual_serves") or [])
rows += list(prior.get("carried_manual_serves") or []) # a run killed before finalize
- receipt.data["carried_manual_serves"] = [
- row for row in rows if isinstance(row, dict) and row.get("supervisor") == "manual-serve"]
+ from hermes_cli.update_serve_obligations import is_reminder_row
+
+ receipt.data["carried_manual_serves"] = [row for row in rows if is_reminder_row(row)]
except Exception as exc: # pragma: no cover - defensive
logger.debug("Could not start update receipt: %s", exc)
return
diff --git a/hermes_cli/update_serve_obligations.py b/hermes_cli/update_serve_obligations.py
index cefee338338..fcd5871d72b 100644
--- a/hermes_cli/update_serve_obligations.py
+++ b/hermes_cli/update_serve_obligations.py
@@ -64,25 +64,42 @@ def _write_reminder(row: dict) -> None:
temporary.unlink(missing_ok=True)
-def remind_unit_serve(runtime: dict) -> bool:
- """File the same durable reminder for a stale serve/dashboard whose systemd unit this update could
- not restart (no root, #141021). The fleet obligation only names gateways, so without this the
- stale unit is never mentioned again after the update run. Carries the unit's restart command;
- cleared like a manual reminder once that incarnation exits. False when nothing was filed."""
+def is_unit_serve_row(row: object) -> bool:
+ """A serve/dashboard row whose systemd unit owns it (MainPID), with the unit recorded (#141021)."""
+ return (isinstance(row, dict) and row.get("kind") in ("serve", "dashboard") and row.get("supervisor") == "systemd"
+ and isinstance(row.get("detail"), dict) and bool(row["detail"].get("systemd_unit")))
+
+
+def is_reminder_row(row: object) -> bool:
+ """Rows the receipt-backed reminder flow carries: manual serves and unit-owned serves."""
+ return is_unit_serve_row(row) or (
+ isinstance(row, dict) and row.get("kind") in ("serve", "dashboard") and row.get("supervisor") == "manual-serve")
+
+
+def defer_unit_serve(runtime: dict) -> bool:
+ """The unit-owned twin of :func:`defer_manual_serve`: file a durable reminder carrying the unit's
+ restart command for a pre-update incarnation that is still alive. A unit-owned row moved off the
+ manual-serve reminder when it was classified ``systemd`` (#141021), and the fleet obligation names
+ only gateways, so without this a unit the update could not restart (no root, an aborted run,
+ ``--no-gateway-restart``) is never mentioned again. True when the row is discharged (process gone)
+ or the reminder was filed; False when it is still owed and nothing durable could be written."""
from hermes_cli.main_dashboard import _systemd_restart_commands
from hermes_cli.process_identity import _pid_alive_matches
- detail = runtime.get("detail") if isinstance(runtime.get("detail"), dict) else {}
- pid, created, unit = runtime.get("pid"), detail.get("create_time"), detail.get("systemd_unit")
- if (runtime.get("kind") not in ("serve", "dashboard") or runtime.get("supervisor") != "systemd" or not unit
- or type(pid) is not int or pid <= 0 or type(created) not in (int, float) or not math.isfinite(created)):
+ if not is_unit_serve_row(runtime):
+ return False
+ detail, pid = runtime["detail"], runtime.get("pid")
+ created = detail.get("create_time")
+ if type(pid) is not int or pid <= 0:
return False
+ if not (type(created) in (int, float) and math.isfinite(created) and created > 0):
+ return _pid_alive_matches(pid, None) is False
try:
- if _pid_alive_matches(pid, created) is not True:
- return False
+ if _pid_alive_matches(pid, created) is False:
+ return True
_write_reminder({"kind": runtime["kind"], "profile": runtime.get("profile", "unknown"), "pid": pid,
"create_time": created,
- "restart": _systemd_restart_commands(str(unit), detail.get("systemd_scope"))})
+ "restart": _systemd_restart_commands(str(detail["systemd_unit"]), detail.get("systemd_scope"))})
return True
except (OSError, ValueError, TypeError) as exc:
logger.debug("Could not preserve unit serve obligation: %s", exc)
@@ -92,6 +109,7 @@ def remind_unit_serve(runtime: dict) -> bool:
def retain_receipt_manual_serves(receipt: dict) -> list[dict]:
"""Return transfers still owed so receipt rotation cannot discard failed writes.
+ Covers manual serves and unit-owned serves (:func:`is_reminder_row`); the name predates the latter.
``carried_manual_serves`` is the durable running record's copy of the previous receipt's rows
(update_receipt.begin): read it too, or the warning vanishes while an update runs and after a
run killed before its ``plan`` stage.
@@ -101,9 +119,10 @@ def retain_receipt_manual_serves(receipt: dict) -> list[dict]:
rows += list(receipt.get("carried_manual_serves") or [])
pending = []
for row in rows:
- if not isinstance(row, dict) or row.get("kind") not in ("serve", "dashboard") or row.get("supervisor") != "manual-serve":
+ if not is_reminder_row(row):
continue
- if not defer_manual_serve(row) and row not in pending:
+ deferred = defer_unit_serve(row) if is_unit_serve_row(row) else defer_manual_serve(row)
+ if not deferred and row not in pending:
pending.append(row)
return pending
@@ -119,7 +138,12 @@ def warn_pending_manual_serves(*, startup: bool = False, pending_manual: list[di
for row in pending_manual:
print(f" ⚠ {row['kind']} [{row.get('profile', 'unknown')}] pid {row.get('pid', 'unknown')}: manual restart reminder could not be saved; restart remains pending in the update receipt.", file=stream)
detail = row.get("detail") if isinstance(row.get("detail"), dict) else {}
- if type(detail.get("create_time")) in (int, float):
+ if is_unit_serve_row(row):
+ from hermes_cli.main_dashboard import _systemd_restart_commands
+
+ commands = _systemd_restart_commands(str(detail["systemd_unit"]), detail.get("systemd_scope"))
+ print(f" Restart its unit: {' or '.join(f'`{c}`' for c in commands)}", file=stream)
+ elif type(detail.get("create_time")) in (int, float):
print(" Ask its owner to relaunch `hermes serve` / `hermes dashboard`; check reminder storage permissions and free space.", file=stream)
else:
# No usable creation time means identity, not storage, blocked the durable reminder.
diff --git a/tests/hermes_cli/test_update_system_dashboard_unit.py b/tests/hermes_cli/test_update_system_dashboard_unit.py
index 51881d5e5ff..8518f6e0ed9 100644
--- a/tests/hermes_cli/test_update_system_dashboard_unit.py
+++ b/tests/hermes_cli/test_update_system_dashboard_unit.py
@@ -58,6 +58,7 @@ def _systemd_owned_dashboard(monkeypatch, systemctl_calls: list):
monkeypatch.setattr(main_dashboard, "_find_stale_dashboard_pids", lambda **k: [DASH_PID])
[email protected]("linux") # systemd/cgroup paths are Linux-only (sys.platform gates)
def test_dashboard_pass_skips_a_unit_the_fleet_loop_already_declined(monkeypatch, capsys):
calls: list = []
_systemd_owned_dashboard(monkeypatch, calls)
@@ -74,6 +75,7 @@ def test_dashboard_pass_skips_a_unit_the_fleet_loop_already_declined(monkeypatch
assert "failed to restart" not in out and "Restarting managed dashboard" not in out
[email protected]("linux") # systemd/cgroup paths are Linux-only (sys.platform gates)
def test_dashboard_pass_still_restarts_the_unit_when_the_fleet_loop_never_saw_it(monkeypatch, capsys):
"""Negative control: without the fleet loop's verdict the managed restart still runs."""
calls: list = []
@@ -82,6 +84,7 @@ def test_dashboard_pass_still_restarts_the_unit_when_the_fleet_loop_never_saw_it
assert ["systemctl", "restart", UNIT] in calls
[email protected]("linux") # systemd/cgroup paths are Linux-only (sys.platform gates)
def test_systemd_owned_ledger_dashboard_is_classified_systemd(monkeypatch):
calls: list = []
_systemd_owned_dashboard(monkeypatch, calls)
@@ -96,6 +99,7 @@ def test_systemd_owned_ledger_dashboard_is_classified_systemd(monkeypatch):
assert (row.detail["systemd_unit"], row.detail["systemd_scope"]) == (UNIT, "system")
[email protected]("linux") # systemd/cgroup paths are Linux-only (sys.platform gates)
def test_hand_started_dashboard_in_a_foreign_cgroup_stays_manual(monkeypatch):
"""Started from a shell inside cron.service: the cgroup names a unit, but its MainPID is another
process, so the real ownership rule keeps the row manual-serve (its respawn reminder applies)."""
@@ -211,26 +215,71 @@ def _unit_dashboard_plan():
detail={"create_time": 1000.0, "systemd_unit": UNIT, "systemd_scope": "system"})])
-def test_unrestarted_unit_dashboard_keeps_a_startup_reminder_until_it_exits(monkeypatch, capsys):
- """The fleet obligation names only gateways, so once they are current the stale dashboard unit
- would never be mentioned again; it keeps its own reminder with the unit's command instead."""
+def _unit_row():
+ from dataclasses import asdict
+
+ return asdict(_unit_dashboard_plan().runtimes[0])
+
+
[email protected]("how", ["after-verification", "aborted-before-verification"])
+def test_unrestarted_unit_dashboard_keeps_a_receipt_backed_reminder(monkeypatch, capsys, how):
+ """The fleet obligation names only gateways, so once they are current a stale dashboard unit would
+ never be mentioned again. It rides the same receipt-backed reminder flow as a manual serve, so the
+ startup warning recovers it from the receipt's plan even when verification never ran (an aborted
+ run, ``--no-gateway-restart``), names the unit's command, and clears once the process exits."""
from hermes_cli import process_identity
- from hermes_cli.update_serve_obligations import warn_pending_manual_serves
monkeypatch.setattr(process_identity, "_pid_alive_matches", lambda *a: True)
- survivor = {"pid": DASH_PID, "kind": "dashboard", "profile": "default", "supervisor": "systemd",
- "systemd_unit": UNIT, "systemd_scope": "system"}
- _verify(monkeypatch, _restart(), plan=_unit_dashboard_plan(), survivors=[survivor])
+ if how == "after-verification":
+ survivor = {"pid": DASH_PID, "kind": "dashboard", "profile": "default", "supervisor": "systemd",
+ "systemd_unit": UNIT, "systemd_scope": "system"}
+ plan = _unit_dashboard_plan()
+ real_begin = update_receipt.begin_update_receipt
+
+ def begin_with_plan(*a, **k):
+ real_begin(*a, **k)
+ update_inventory.record_plan_in_receipt(plan)
+
+ monkeypatch.setattr(update_receipt, "begin_update_receipt", begin_with_plan)
+ _verify(monkeypatch, _restart(), plan=plan, survivors=[survivor])
+ else:
+ with update_receipt.update_receipt_scope():
+ update_receipt.begin_update_receipt()
+ update_inventory.record_plan_in_receipt(_unit_dashboard_plan())
+ update_receipt.finalize_update_receipt("failed")
capsys.readouterr()
- warn_pending_manual_serves(startup=True, pending_manual=[])
+ update_cmd_fleet._warn_pending_fleet_restart_on_startup()
err = capsys.readouterr().err
assert f"dashboard [default] pid {DASH_PID}: unit restart still pending" in err
assert f"`sudo systemctl restart {UNIT}`" in err and "Ask its owner" not in err
monkeypatch.setattr(process_identity, "_pid_alive_matches", lambda *a: False)
- warn_pending_manual_serves(startup=True, pending_manual=[])
+ update_cmd_fleet._warn_pending_fleet_restart_on_startup()
assert str(DASH_PID) not in capsys.readouterr().err
+def test_unit_reminder_that_cannot_be_saved_stays_in_the_receipt_with_its_command(monkeypatch, capsys):
+ from hermes_cli import process_identity
+ from hermes_cli.update_serve_obligations import retain_receipt_manual_serves, warn_pending_manual_serves
+ from hermes_constants import get_hermes_home
+
+ monkeypatch.setattr(process_identity, "_pid_alive_matches", lambda *a: True)
+ (get_hermes_home() / "serve_restart_pending").write_text("not a directory")
+ pending = retain_receipt_manual_serves({"plan": {"runtimes": [_unit_row()]}})
+ assert [row["pid"] for row in pending] == [DASH_PID]
+ warn_pending_manual_serves(startup=True, pending_manual=pending)
+ err = capsys.readouterr().err
+ assert f"`sudo systemctl restart {UNIT}`" in err and "Ask its owner" not in err
+
+
+def test_receipt_carries_unit_rows_forward_like_manual_ones():
+ from hermes_cli.update_serve_obligations import is_reminder_row
+
+ assert is_reminder_row(_unit_row())
+ assert is_reminder_row({"kind": "serve", "supervisor": "manual-serve"})
+ assert not is_reminder_row({"kind": "dashboard", "supervisor": "systemd", "detail": {}})
+ assert not is_reminder_row({"kind": "gateway", "supervisor": "systemd", "detail": {"systemd_unit": "x"}})
+
+
def test_unaccounted_unit_dashboard_hint_names_its_unit(capsys):
plan = UpdatePlan(runtimes=[RuntimeRecord(
kind="dashboard", profile="work", pid=DASH_PID, supervisor="systemd", restart_via="systemd",
@@ -241,3 +290,16 @@ def test_unaccounted_unit_dashboard_hint_names_its_unit(capsys):
assert update_inventory.report_unaccounted_runtimes(outcomes) is True
out = capsys.readouterr().out
assert f"sudo systemctl restart {UNIT}" in out and "hermes-serve.service" not in out
+
+
+def test_next_run_carries_a_unit_row_forward_from_the_previous_receipt(monkeypatch):
+ """A run killed before finalize must not lose the previous receipt's unit row: the next run's
+ running record snapshots it into ``carried_manual_serves`` like a manual one."""
+ with update_receipt.update_receipt_scope():
+ update_receipt.begin_update_receipt()
+ update_inventory.record_plan_in_receipt(_unit_dashboard_plan())
+ update_receipt.finalize_update_receipt("failed")
+ with update_receipt.update_receipt_scope():
+ update_receipt.begin_update_receipt()
+ carried = update_receipt._current.get().data["carried_manual_serves"]
+ assert [row["pid"] for row in carried] == [DASH_PID]