Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Refuse incomplete updater manager observations

pulse-triage[bot]8d8a1790ad83
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 7c4c4cb2b9..27151d2f66 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -7565,6 +7565,35 @@ remains required and is not claimed here.
### Verified unattended-update backup and owned rollback
+The unattended updater's service-state and running-PID reads require a successful
+bounded manager command and exactly one complete allowlisted property tuple.
+Both output streams belong to that observation; diagnostic text, blank records
+(including trailing stdout or stderr blanks), duplicates, extra properties and
+an unterminated final property are not update, start, rollback or success consent.
+A completion marker preserves those records through Bash command substitution
+and cannot appear twice or precede another record. Raw manager diagnostics are
+not echoed by the updater. The existing five-second read and one-second kill
+bounds, property-order independence and loaded/masked admission stay unchanged.
+
+An unknown initial observation stops before backup/download/installer mutation.
+An unknown post-installer or post-stop observation retains the saved executable
+and requires explicit service reconciliation, without an automatic second read,
+restart, restore or claimed success. Running identity needs complete reads on
+both sides of the unchanged `/proc` executable-byte comparison; a plausible
+first PID cannot compensate for an incomplete second observation.
+
+The registered `scripts/installtests/pulse_auto_update_test.go` verification
+requires the three manager-diagnostic caller controls in `UpdateServiceState`
+and the diagnostic-bearing first/second identity control in `RunningExecutable`
+from `test_auto_update_service_state.py`. They cover stderr text, a blank stderr
+record, a trailing stdout blank and an unterminated last property through the
+real conditional update/restart/rollback callers, preserving files, identity
+and recovery. Identity controls use a real private ordinary-user child and
+confirm its cleanup, not a live Pulse service. Existing signatures, actual file
+recovery, service deadlines and all original adverse controls remain mandatory.
+This source proof neither attributes #2785 nor establishes native/installed
+recovery; the original supplied-log and signed forward/reverse jobs remain.
+
`scripts/pulse-auto-update.sh` runs `perform_update` as a conditional, so Bash
`errexit` is not a substitute for explicit snapshot and restore checks. Before
executing the signed installer it must have a private unique rollback directory
diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go
index edf05eb6b9..63a0f401bf 100644
--- a/scripts/installtests/pulse_auto_update_test.go
+++ b/scripts/installtests/pulse_auto_update_test.go
@@ -170,10 +170,13 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {
if err != nil {
t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)
}
- // Keep the collector controls part of actual package acceptance, not just
+ // Keep the manager/collector controls part of actual package acceptance, not just
// an optional Python check. A green fixture with these methods removed
// would otherwise silently lose early-failure and final-line coverage.
required := []string{
+ "test_manager_diagnostics_refuse_initial_update_and_restart",
+ "test_manager_diagnostics_after_install_retain_backup_without_mutation",
+ "test_manager_diagnostics_after_stop_retain_backup_without_restart",
"test_success_cannot_claim_unverified_running_executable",
"test_rollback_keeps_backup_until_restored_running_executable_matches",
"test_early_collector_failure_survives_later_success_and_drains_installer",
@@ -237,6 +240,7 @@ func TestAutoUpdateRunningExecutableIdentity(t *testing.T) {
for _, name := range []string{
"test_actual_process_not_replaced_disk_path_is_checked",
"test_complete_stable_main_pid_tuple_is_required",
+ "test_manager_diagnostics_refuse_either_running_identity_read",
"test_identity_read_timeout_is_bounded_and_owned_process_is_reaped",
} {
passed := false
diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.sh
index d51ce53b91..9b1c534dcd 100755
--- a/scripts/pulse-auto-update.sh
+++ b/scripts/pulse-auto-update.sh
@@ -405,12 +405,18 @@ download_update_asset() {
# plausible partial result grants an update, restart or rollback.
read_update_service_state() {
local service_name=$1 observation="" line="" load="" state=""
- local seen_load=false seen_state=false
- if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState 2>/dev/null); then
+ local seen_load=false seen_state=false read_completed=false
+ # Keep diagnostics in the same observation and preserve trailing blank
+ # records. Command substitution otherwise discards them, turning an
+ # incomplete manager reply into consent for update/restart/rollback.
+ # The completion marker also requires the last property to end normally.
+ if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState 2>&1 &&
+ printf '%s' '__pulse_state_read_completed__'); then
log error "Cannot read Pulse service state ($service_name); reconcile service state before retrying" >&2
return 1
fi
while IFS= read -r line; do
+ [[ "$read_completed" == false ]] || return 1
case "$line" in
LoadState=*)
[[ "$seen_load" == false ]] || { log error "Duplicate Pulse service load state ($service_name)" >&2; return 1; }
@@ -418,12 +424,14 @@ read_update_service_state() {
ActiveState=*)
[[ "$seen_state" == false ]] || { log error "Duplicate Pulse service active state ($service_name)" >&2; return 1; }
seen_state=true; state=${line#ActiveState=} ;;
+ __pulse_state_read_completed__) read_completed=true ;;
*)
log error "Incomplete Pulse service observation ($service_name); no service action is safe" >&2
return 1 ;;
esac
done <<< "$observation"
- if [[ "$seen_load" != true || "$seen_state" != true || ( "$load" != loaded && "$load" != masked ) ]]; then
+ if [[ "$seen_load" != true || "$seen_state" != true || "$read_completed" != true ||
+ ( "$load" != loaded && "$load" != masked ) ]]; then
log error "Pulse service is unavailable ($service_name); no service action is safe" >&2
return 1
fi
@@ -437,12 +445,16 @@ read_update_service_state() {
# Read one complete tuple so a missing/recycled process is not update evidence.
read_update_service_main_pid() {
local service_name=$1 observation="" line="" load="" state="" pid=""
- local seen_load=false seen_state=false seen_pid=false
- if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState --property=MainPID 2>/dev/null); then
+ local seen_load=false seen_state=false seen_pid=false read_completed=false
+ # Both sides of the executable comparison require a complete read, just
+ # like update admission. Never hide a diagnostic behind a plausible PID.
+ if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState --property=MainPID 2>&1 &&
+ printf '%s' '__pulse_state_read_completed__'); then
log error "Cannot read running Pulse identity ($service_name); reconcile service state before recovery" >&2
return 1
fi
while IFS= read -r line; do
+ [[ "$read_completed" == false ]] || return 1
case "$line" in
LoadState=*)
[[ "$seen_load" == false ]] || return 1
@@ -453,10 +465,11 @@ read_update_service_main_pid() {
MainPID=*)
[[ "$seen_pid" == false ]] || return 1
seen_pid=true; pid=${line#MainPID=} ;;
+ __pulse_state_read_completed__) read_completed=true ;;
*) return 1 ;;
esac
done <<< "$observation"
- if [[ "$seen_load" != true || "$seen_state" != true || "$seen_pid" != true ||
+ if [[ "$seen_load" != true || "$seen_state" != true || "$seen_pid" != true || "$read_completed" != true ||
( "$load" != loaded && "$load" != masked ) || "$state" != active ||
! "$pid" =~ ^[1-9][0-9]*$ ]]; then
log error "Incomplete running Pulse identity ($service_name); reconcile service state before recovery" >&2
diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.py
index 007bf5fb42..55afde27ee 100644
--- a/scripts/tests/test_auto_update_service_state.py
+++ b/scripts/tests/test_auto_update_service_state.py
@@ -48,8 +48,15 @@ if args[0] == "show":
print("LoadState=loaded\nActiveState=inactive\nUnknown=active")
sys.exit(0)
# Reverse property order deliberately: systemd does not promise ordering.
- print("ActiveState=" + state)
- print("LoadState=" + os.environ["LOAD_STATE"])
+ payload = "ActiveState=" + state + "\nLoadState=" + os.environ["LOAD_STATE"] + "\n"
+ phase = "post-stop" if stopped else ("post-install" if after else "initial")
+ fault = os.environ.get("OBSERVATION_FAULT", "") if phase == os.environ.get("OBSERVATION_PHASE") else ""
+ if fault == "stdout-blank": payload += "\n"
+ if fault == "stdout-unterminated": payload = payload.rstrip("\n")
+ sys.stdout.write(payload)
+ sys.stdout.flush()
+ if fault == "stderr-warning": sys.stderr.write("private fixture manager diagnostic\n")
+ if fault == "stderr-blank": sys.stderr.write("\n")
sys.exit(0)
if args[0] == "is-active":
sys.exit(0 if state == "active" and not mode.startswith("initial-") else 1)
@@ -181,7 +188,8 @@ def read_logger_calls(path):
class UpdaterFixture(unittest.TestCase):
def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",
installer_exit=None, collector_exit=0, installer_output=None, collector_failures=None,
- real_log=False, runtime_exit=0, installer_state=None, download_fault=""):
+ real_log=False, runtime_exit=0, installer_state=None, download_fault="",
+ observation_fault="", observation_phase="initial"):
with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:
f = Path(directory)
for name in ("tools", "tmp", "install/bin", "config"):
@@ -222,6 +230,7 @@ class UpdaterFixture(unittest.TestCase):
COLLECTOR_EXIT=str(collector_exit),
REAL_LOG="true" if real_log else "false", RUNTIME_EXIT=str(runtime_exit),
DOWNLOAD_FAULT=download_fault,
+ OBSERVATION_FAULT=observation_fault, OBSERVATION_PHASE=observation_phase,
PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])
for line, status in (collector_failures or {}).items():
env[f"COLLECTOR_EXIT_{line}"] = str(status)
@@ -381,6 +390,58 @@ class UpdateLogSinks(UpdaterFixture):
class UpdateServiceState(UpdaterFixture):
+ def test_manager_diagnostics_refuse_initial_update_and_restart(self):
+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):
+ for flow, state in (("update", "active"), ("update", "inactive"), ("backstop", "inactive")):
+ with self.subTest(fault=fault, flow=flow, state=state):
+ r = self.run_fixture(observation_fault=fault, flow=flow, state=state, success=True)
+ # The backstop retains the caller's status, but no unknown
+ # observation is consent to install, restart or allocate.
+ self.assertEqual(r["exit"], 0 if flow == "backstop" else 1, r)
+ self.assertFalse(r["installed"] or r["downloaded"] or r["temporary"], r)
+ self.assertFalse(r["started"] or r["stopped"], r)
+ self.assertEqual(r["state"], state, r)
+ self.assertEqual(r["backups"], [], r)
+ self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n", r)
+ self.assertEqual(len(r["calls"]), 1, r) # No more favourable read.
+ self.assertNotIn("private fixture manager diagnostic", r["output"] + r["error"])
+
+ def test_manager_diagnostics_after_install_retain_backup_without_mutation(self):
+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):
+ for state in ("active", "inactive"):
+ with self.subTest(fault=fault, state=state):
+ r = self.run_fixture(observation_fault=fault, observation_phase="post-install",
+ state=state, success=True, installer_state="active")
+ self.assertEqual(r["exit"], 1, r)
+ self.assertTrue(r["installed"] and r["installer_finished"], r)
+ self.assertFalse(r["started"] or r["stopped"], r)
+ self.assertEqual(r["state"], "active", r)
+ self.assertEqual(r["version"], "v6.6.0\n", r)
+ self.assertEqual(r["backups"], [b"#!/bin/sh\necho 'Pulse v6.5.0'\n"], r)
+ self.assertEqual(r["runtime_probes"], [], r)
+ self.assertEqual(len(r["calls"]), 2, r)
+ self.assertNotIn("Update successfully installed and verified", r["output"])
+ self.assertNotIn("private fixture manager diagnostic", r["output"] + r["error"])
+
+ def test_manager_diagnostics_after_stop_retain_backup_without_restart(self):
+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):
+ for state in ("active", "inactive"):
+ with self.subTest(fault=fault, state=state):
+ r = self.run_fixture(observation_fault=fault, observation_phase="post-stop",
+ state=state, installer_state="active")
+ self.assertEqual(r["exit"], 1, r)
+ self.assertTrue(r["installed"] and r["stopped"], r)
+ self.assertFalse(r["started"], r)
+ self.assertEqual(r["state"], "inactive", r)
+ # The stop was authorised by the previous valid read;
+ # incomplete readback permits neither restore nor restart.
+ self.assertEqual(r["version"], "v6.6.0\n", r)
+ self.assertEqual(r["backups"], [b"#!/bin/sh\necho 'Pulse v6.5.0'\n"], r)
+ self.assertEqual(len(r["calls"]), 4, r)
+ self.assertIn("manual recovery required", r["output"])
+ self.assertNotIn("Previous installation restored", r["output"])
+ self.assertNotIn("private fixture manager diagnostic", r["output"] + r["error"])
+
def test_success_cannot_claim_unverified_running_executable(self):
# A matching new disk version and an active manager are insufficient.
# Preserve the currently active service; uncertainty grants no stop,
@@ -585,9 +646,19 @@ if mode == "pid-moved" and count > 1: pid = os.environ["DIFFERENT_PID"]
load = "not-found" if mode == "not-found" else ("masked" if mode == "masked" else "loaded")
state = "inactive" if mode == "inactive" or (mode == "state-moved" and count > 1) else "active"
# Deliberately reverse property order, as systemd does not promise an order.
-print("MainPID=" + pid)
-if mode != "missing": print("ActiveState=" + state)
-print("LoadState=" + load)
+payload = "MainPID=" + pid + "\n"
+if mode != "missing": payload += "ActiveState=" + state + "\n"
+payload += "LoadState=" + load + "\n"
+fault = ""
+if mode.startswith("diagnostic:"):
+ _, selected_fault, selected_read = mode.split(":")
+ if count == int(selected_read): fault = selected_fault
+if fault == "stdout-blank": payload += "\n"
+if fault == "stdout-unterminated": payload = payload.rstrip("\n")
+sys.stdout.write(payload)
+sys.stdout.flush()
+if fault == "stderr-warning": sys.stderr.write("private fixture manager diagnostic\n")
+if fault == "stderr-blank": sys.stderr.write("\n")
if mode == "duplicate": print("MainPID=" + pid)
if mode == "extra": print("Unknown=value")
'''
@@ -654,6 +725,18 @@ class RunningExecutable(unittest.TestCase):
self.assertEqual(r.returncode, 0 if accepted else 2, r.stderr)
self.assertEqual(observations, 2 if accepted or mode in ("pid-moved", "state-moved") else 1)
+ def test_manager_diagnostics_refuse_either_running_identity_read(self):
+ # These read the actual private child's /proc executable before and
+ # after the unchanged byte comparison, not a simulated version file.
+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):
+ for reading in (1, 2):
+ with self.subTest(fault=fault, reading=reading):
+ r, _, observations = self.run_identity(mode=f"diagnostic:{fault}:{reading}")
+ self.assertEqual(r.returncode, 2, r.stderr)
+ self.assertIn(b"IDENTITY_STATUS=2", r.stdout)
+ self.assertEqual(observations, reading)
+ self.assertNotIn(b"private fixture manager diagnostic", r.stdout + r.stderr)
+
def test_identity_read_timeout_is_bounded_and_owned_process_is_reaped(self):
r, elapsed, observations = self.run_identity(mode="hang")
self.assertEqual(r.returncode, 2)