Skip to content

The agent that grows with you

fix(update): don't retry a dashboard unit the fleet loop already declined

kshitijk4poor5ddafd25465c
diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py
index 77a516f5423..06b0aac0259 100644
--- a/hermes_cli/dashboard_procs.py
+++ b/hermes_cli/dashboard_procs.py
@@ -604,19 +604,23 @@ def _kill_stale_dashboard_processes(
Manually-started dashboards are not auto-restarted because we don't know the original launch args
(--host, --port, --insecure, --tui, --no-open). See #68934.
- *already_restarted_units* names units (no ``.service`` suffix) the caller already restarted directly —
- e.g. ``hermes update``'s systemd fleet-restart loop, which restarts ``hermes-serve*`` units before this
- function runs. Without excluding them, a Serve-only install's freshly restarted process is found again
- here and restarted a second time for no benefit (review on #83595).
+ *already_restarted_units* names units (no ``.service`` suffix) the caller already handled directly —
+ e.g. ``hermes update``'s systemd fleet-restart loop, which restarts (or, without root, declines and
+ names) ``hermes-serve*``/``hermes-dashboard*`` units before this function runs. Without excluding them,
+ a Serve-only install's freshly restarted process is found again here and restarted a second time for
+ no benefit (review on #83595), and a declined system unit is retried into a polkit prompt (#141021).
"""
from hermes_cli import main_dashboard as _dash
- if restart_managed and _dash._restart_managed_dashboard_service(reason):
+ _dash_unit = str(getattr(_dash, "_DASHBOARD_SYSTEMD_UNIT", "hermes-dashboard.service")).removesuffix(".service")
+ # The update's fleet loop already handled the dashboard unit (restarted it, or declined for lack
+ # of root and printed the manual command): a second attempt here restarts a healthy unit twice,
+ # or retries without --no-ask-password and dumps a polkit/tty error under the same advice (#141021).
+ if (restart_managed and _dash_unit not in (already_restarted_units or ())
+ and _dash._restart_managed_dashboard_service(reason)):
# The dashboard unit is handled but other backends (e.g. hermes-serve.service) are not:
# mark the unit handled so the filter below drops its PIDs, and keep going.
- _dash_unit = getattr(_dash, "_DASHBOARD_SYSTEMD_UNIT", "hermes-dashboard.service")
- already_restarted_units = set(already_restarted_units or ()) | {
- str(_dash_unit).removesuffix(".service")}
+ already_restarted_units = set(already_restarted_units or ()) | {_dash_unit}
exclude = _exclude_pids_from_env()
if restart_managed:
# An SSH-owned backend belongs to an attached Desktop client; killing it strands that
diff --git a/hermes_cli/update_cmd_fleet_verify.py b/hermes_cli/update_cmd_fleet_verify.py
index 4ee3b2185db..9056843ebc8 100644
--- a/hermes_cli/update_cmd_fleet_verify.py
+++ b/hermes_cli/update_cmd_fleet_verify.py
@@ -200,9 +200,12 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re
# Restart a managed dashboard via systemd or stop stale manual ones (raw-killing
# a systemd-owned PID reads as clean stop and leaves the Cloudflare origin dead).
- # Already-restarted units aren't redone.
+ # Units the fleet loop already handled aren't redone — restarted ones, and failed ones too: those
+ # already printed their manual restart command, and a retry here (no root either) only adds a
+ # polkit error under the same advice (#141021).
# A dashboard it stopped and could not bring back is a promised restart that did not happen.
- _dashboards_down = _refresh_dashboard_after_update(already_restarted_units=set(restart.restarted_services))
+ _dashboards_down = _refresh_dashboard_after_update(
+ already_restarted_units=set(restart.restarted_services) | set(restart.failed_or_stale_units))
if _dashboards_down:
restart.incomplete = True
diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py
index d865d00419c..a9ad0e28a73 100644
--- a/hermes_cli/update_cmd_maint.py
+++ b/hermes_cli/update_cmd_maint.py
@@ -273,7 +273,8 @@ def _refresh_dashboard_after_update(*, already_restarted_units: set[str] | None
stopped and could not bring back, so the receipt records them ``failed`` (#109290).
*already_restarted_units*: systemd unit names (no ``.service``) the fleet-restart loop
- already restarted, so a Serve-only install isn't restarted a second time here.
+ already handled (restarted, or declined and named), so a Serve-only install isn't
+ restarted a second time here and a declined system unit isn't retried (#141021).
See #83595.
"""
diff --git a/tests/hermes_cli/test_update_system_dashboard_unit.py b/tests/hermes_cli/test_update_system_dashboard_unit.py
new file mode 100644
index 00000000000..673ea111a48
--- /dev/null
+++ b/tests/hermes_cli/test_update_system_dashboard_unit.py
@@ -0,0 +1,79 @@
+"""``hermes update`` on a system-scope ``hermes-dashboard`` unit without root (#141021).
+
+The fleet loop correctly declines to restart the unit (no non-interactive privilege path) and names
+the manual command. Three things then went wrong after it:
+
+* the post-update dashboard pass retried the same unit with a plain ``systemctl restart`` and
+ ``sudo -n``, dumping a polkit/tty error under identical advice;
+* the inventory labelled the unit's MainPID ``manual-serve``, so the user was told to "ask the
+ owner to relaunch `hermes dashboard`" for a process systemd owns;
+* the owed follow-up always said "gateways may still run pre-update code … `hermes gateway
+ restart`", even with every gateway verified current.
+"""
+
+from __future__ import annotations
+
+import sys
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from hermes_cli import dashboard_procs, main_dashboard, update_cmd, update_cmd_fleet
+from hermes_cli import update_cmd_fleet_verify as fleet_verify
+from hermes_cli import update_inventory, update_receipt
+from hermes_cli.update_inventory import RuntimeRecord, UpdatePlan
+
+DASH_PID = 961386
+UNIT = "hermes-dashboard.service"
+CGROUP = f"/system.slice/{UNIT}"
+
+
[email protected](autouse=True)
+def _no_real_launchd_jobs():
+ with patch.object(main_dashboard, "_loaded_launchd_backend_jobs", return_value=[]):
+ yield
+
+
+def _systemd_owned_dashboard(monkeypatch, systemctl_calls: list):
+ """Host shape from the report: PID 961386 is MainPID of the SYSTEM hermes-dashboard unit; any
+ manage verb would hit polkit (no TTY) and ``sudo -n`` would need a password."""
+ monkeypatch.setattr(main_dashboard, "_get_pid_cgroup_path", lambda pid: CGROUP if pid == DASH_PID else None)
+ monkeypatch.setattr(main_dashboard, "_get_systemd_service_for_pid", lambda pid: UNIT if pid == DASH_PID else None)
+
+ def run_probe(cmd, *, timeout):
+ systemctl_calls.append(list(cmd))
+ if "list-unit-files" in cmd:
+ listed = "--user" not in cmd
+ return SimpleNamespace(returncode=0 if listed else 1, stdout=f"{UNIT} enabled enabled\n" if listed else "", stderr="")
+ if "is-active" in cmd:
+ return SimpleNamespace(returncode=0, stdout="active\n", stderr="")
+ if "is-enabled" in cmd:
+ return SimpleNamespace(returncode=0, stdout="enabled\n", stderr="")
+ return SimpleNamespace(returncode=1, stdout="", stderr="Interactive authentication required.")
+
+ monkeypatch.setattr(main_dashboard, "_run_probe", run_probe)
+ monkeypatch.setattr(main_dashboard, "_find_stale_dashboard_pids", lambda **k: [DASH_PID])
+
+
+def test_dashboard_pass_skips_a_unit_the_fleet_loop_already_declined(monkeypatch, capsys):
+ calls: list = []
+ _systemd_owned_dashboard(monkeypatch, calls)
+ kills: list = []
+ monkeypatch.setattr(dashboard_procs, "_kill_pids_posix", lambda pids, killed, failed: kills.extend(pids))
+
+ result = dashboard_procs._kill_stale_dashboard_processes(
+ restart_managed=True, already_restarted_units={"hermes-gateway", "hermes-dashboard"})
+
+ assert not [c for c in calls if "restart" in c], "the declined unit was retried"
+ assert kills == [] and not result.get("unrecovered")
+ out = capsys.readouterr().out
+ assert "polkit" not in out and "failed to restart" not in out and "Restarting managed dashboard" not in out
+
+
+def test_dashboard_pass_still_restarts_the_unit_when_the_fleet_loop_never_saw_it(monkeypatch, capsys):
+ """Negative control: without the fleet loop's verdict the managed restart still runs."""
+ calls: list = []
+ _systemd_owned_dashboard(monkeypatch, calls)
+ dashboard_procs._kill_stale_dashboard_processes(restart_managed=True, already_restarted_units={"hermes-gateway"})
+ assert ["systemctl", "restart", UNIT] in calls