Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Refuse incomplete updater manager observations
pulse-triage[bot]8d8a1790ad83
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 7c4c4cb2b9..27151d2f66 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -7565,6 +7565,35 @@ remains required and is not claimed here.### Verified unattended-update backup and owned rollback+The unattended updater's service-state and running-PID reads require a successful+bounded manager command and exactly one complete allowlisted property tuple.+Both output streams belong to that observation; diagnostic text, blank records+(including trailing stdout or stderr blanks), duplicates, extra properties and+an unterminated final property are not update, start, rollback or success consent.+A completion marker preserves those records through Bash command substitution+and cannot appear twice or precede another record. Raw manager diagnostics are+not echoed by the updater. The existing five-second read and one-second kill+bounds, property-order independence and loaded/masked admission stay unchanged.++An unknown initial observation stops before backup/download/installer mutation.+An unknown post-installer or post-stop observation retains the saved executable+and requires explicit service reconciliation, without an automatic second read,+restart, restore or claimed success. Running identity needs complete reads on+both sides of the unchanged `/proc` executable-byte comparison; a plausible+first PID cannot compensate for an incomplete second observation.++The registered `scripts/installtests/pulse_auto_update_test.go` verification+requires the three manager-diagnostic caller controls in `UpdateServiceState`+and the diagnostic-bearing first/second identity control in `RunningExecutable`+from `test_auto_update_service_state.py`. They cover stderr text, a blank stderr+record, a trailing stdout blank and an unterminated last property through the+real conditional update/restart/rollback callers, preserving files, identity+and recovery. Identity controls use a real private ordinary-user child and+confirm its cleanup, not a live Pulse service. Existing signatures, actual file+recovery, service deadlines and all original adverse controls remain mandatory.+This source proof neither attributes #2785 nor establishes native/installed+recovery; the original supplied-log and signed forward/reverse jobs remain.+`scripts/pulse-auto-update.sh` runs `perform_update` as a conditional, so Bash`errexit` is not a substitute for explicit snapshot and restore checks. Beforeexecuting the signed installer it must have a private unique rollback directorydiff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.goindex edf05eb6b9..63a0f401bf 100644--- a/scripts/installtests/pulse_auto_update_test.go+++ b/scripts/installtests/pulse_auto_update_test.go@@ -170,10 +170,13 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {if err != nil {t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)}- // Keep the collector controls part of actual package acceptance, not just+ // Keep the manager/collector controls part of actual package acceptance, not just// an optional Python check. A green fixture with these methods removed// would otherwise silently lose early-failure and final-line coverage.required := []string{+ "test_manager_diagnostics_refuse_initial_update_and_restart",+ "test_manager_diagnostics_after_install_retain_backup_without_mutation",+ "test_manager_diagnostics_after_stop_retain_backup_without_restart","test_success_cannot_claim_unverified_running_executable","test_rollback_keeps_backup_until_restored_running_executable_matches","test_early_collector_failure_survives_later_success_and_drains_installer",@@ -237,6 +240,7 @@ func TestAutoUpdateRunningExecutableIdentity(t *testing.T) {for _, name := range []string{"test_actual_process_not_replaced_disk_path_is_checked","test_complete_stable_main_pid_tuple_is_required",+ "test_manager_diagnostics_refuse_either_running_identity_read","test_identity_read_timeout_is_bounded_and_owned_process_is_reaped",} {passed := falsediff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.shindex d51ce53b91..9b1c534dcd 100755--- a/scripts/pulse-auto-update.sh+++ b/scripts/pulse-auto-update.sh@@ -405,12 +405,18 @@ download_update_asset() {# plausible partial result grants an update, restart or rollback.read_update_service_state() {local service_name=$1 observation="" line="" load="" state=""- local seen_load=false seen_state=false- if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState 2>/dev/null); then+ local seen_load=false seen_state=false read_completed=false+ # Keep diagnostics in the same observation and preserve trailing blank+ # records. Command substitution otherwise discards them, turning an+ # incomplete manager reply into consent for update/restart/rollback.+ # The completion marker also requires the last property to end normally.+ if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState 2>&1 &&+ printf '%s' '__pulse_state_read_completed__'); thenlog error "Cannot read Pulse service state ($service_name); reconcile service state before retrying" >&2return 1fiwhile IFS= read -r line; do+ [[ "$read_completed" == false ]] || return 1case "$line" inLoadState=*)[[ "$seen_load" == false ]] || { log error "Duplicate Pulse service load state ($service_name)" >&2; return 1; }@@ -418,12 +424,14 @@ read_update_service_state() {ActiveState=*)[[ "$seen_state" == false ]] || { log error "Duplicate Pulse service active state ($service_name)" >&2; return 1; }seen_state=true; state=${line#ActiveState=} ;;+ __pulse_state_read_completed__) read_completed=true ;;*)log error "Incomplete Pulse service observation ($service_name); no service action is safe" >&2return 1 ;;esacdone <<< "$observation"- if [[ "$seen_load" != true || "$seen_state" != true || ( "$load" != loaded && "$load" != masked ) ]]; then+ if [[ "$seen_load" != true || "$seen_state" != true || "$read_completed" != true ||+ ( "$load" != loaded && "$load" != masked ) ]]; thenlog error "Pulse service is unavailable ($service_name); no service action is safe" >&2return 1fi@@ -437,12 +445,16 @@ read_update_service_state() {# Read one complete tuple so a missing/recycled process is not update evidence.read_update_service_main_pid() {local service_name=$1 observation="" line="" load="" state="" pid=""- local seen_load=false seen_state=false seen_pid=false- if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState --property=MainPID 2>/dev/null); then+ local seen_load=false seen_state=false seen_pid=false read_completed=false+ # Both sides of the executable comparison require a complete read, just+ # like update admission. Never hide a diagnostic behind a plausible PID.+ if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState --property=MainPID 2>&1 &&+ printf '%s' '__pulse_state_read_completed__'); thenlog error "Cannot read running Pulse identity ($service_name); reconcile service state before recovery" >&2return 1fiwhile IFS= read -r line; do+ [[ "$read_completed" == false ]] || return 1case "$line" inLoadState=*)[[ "$seen_load" == false ]] || return 1@@ -453,10 +465,11 @@ read_update_service_main_pid() {MainPID=*)[[ "$seen_pid" == false ]] || return 1seen_pid=true; pid=${line#MainPID=} ;;+ __pulse_state_read_completed__) read_completed=true ;;*) return 1 ;;esacdone <<< "$observation"- if [[ "$seen_load" != true || "$seen_state" != true || "$seen_pid" != true ||+ if [[ "$seen_load" != true || "$seen_state" != true || "$seen_pid" != true || "$read_completed" != true ||( "$load" != loaded && "$load" != masked ) || "$state" != active ||! "$pid" =~ ^[1-9][0-9]*$ ]]; thenlog error "Incomplete running Pulse identity ($service_name); reconcile service state before recovery" >&2diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.pyindex 007bf5fb42..55afde27ee 100644--- a/scripts/tests/test_auto_update_service_state.py+++ b/scripts/tests/test_auto_update_service_state.py@@ -48,8 +48,15 @@ if args[0] == "show":print("LoadState=loaded\nActiveState=inactive\nUnknown=active")sys.exit(0)# Reverse property order deliberately: systemd does not promise ordering.- print("ActiveState=" + state)- print("LoadState=" + os.environ["LOAD_STATE"])+ payload = "ActiveState=" + state + "\nLoadState=" + os.environ["LOAD_STATE"] + "\n"+ phase = "post-stop" if stopped else ("post-install" if after else "initial")+ fault = os.environ.get("OBSERVATION_FAULT", "") if phase == os.environ.get("OBSERVATION_PHASE") else ""+ if fault == "stdout-blank": payload += "\n"+ if fault == "stdout-unterminated": payload = payload.rstrip("\n")+ sys.stdout.write(payload)+ sys.stdout.flush()+ if fault == "stderr-warning": sys.stderr.write("private fixture manager diagnostic\n")+ if fault == "stderr-blank": sys.stderr.write("\n")sys.exit(0)if args[0] == "is-active":sys.exit(0 if state == "active" and not mode.startswith("initial-") else 1)@@ -181,7 +188,8 @@ def read_logger_calls(path):class UpdaterFixture(unittest.TestCase):def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",installer_exit=None, collector_exit=0, installer_output=None, collector_failures=None,- real_log=False, runtime_exit=0, installer_state=None, download_fault=""):+ real_log=False, runtime_exit=0, installer_state=None, download_fault="",+ observation_fault="", observation_phase="initial"):with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:f = Path(directory)for name in ("tools", "tmp", "install/bin", "config"):@@ -222,6 +230,7 @@ class UpdaterFixture(unittest.TestCase):COLLECTOR_EXIT=str(collector_exit),REAL_LOG="true" if real_log else "false", RUNTIME_EXIT=str(runtime_exit),DOWNLOAD_FAULT=download_fault,+ OBSERVATION_FAULT=observation_fault, OBSERVATION_PHASE=observation_phase,PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])for line, status in (collector_failures or {}).items():env[f"COLLECTOR_EXIT_{line}"] = str(status)@@ -381,6 +390,58 @@ class UpdateLogSinks(UpdaterFixture):class UpdateServiceState(UpdaterFixture):+ def test_manager_diagnostics_refuse_initial_update_and_restart(self):+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):+ for flow, state in (("update", "active"), ("update", "inactive"), ("backstop", "inactive")):+ with self.subTest(fault=fault, flow=flow, state=state):+ r = self.run_fixture(observation_fault=fault, flow=flow, state=state, success=True)+ # The backstop retains the caller's status, but no unknown+ # observation is consent to install, restart or allocate.+ self.assertEqual(r["exit"], 0 if flow == "backstop" else 1, r)+ self.assertFalse(r["installed"] or r["downloaded"] or r["temporary"], r)+ self.assertFalse(r["started"] or r["stopped"], r)+ self.assertEqual(r["state"], state, r)+ self.assertEqual(r["backups"], [], r)+ self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n", r)+ self.assertEqual(len(r["calls"]), 1, r) # No more favourable read.+ self.assertNotIn("private fixture manager diagnostic", r["output"] + r["error"])++ def test_manager_diagnostics_after_install_retain_backup_without_mutation(self):+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):+ for state in ("active", "inactive"):+ with self.subTest(fault=fault, state=state):+ r = self.run_fixture(observation_fault=fault, observation_phase="post-install",+ state=state, success=True, installer_state="active")+ self.assertEqual(r["exit"], 1, r)+ self.assertTrue(r["installed"] and r["installer_finished"], r)+ self.assertFalse(r["started"] or r["stopped"], r)+ self.assertEqual(r["state"], "active", r)+ self.assertEqual(r["version"], "v6.6.0\n", r)+ self.assertEqual(r["backups"], [b"#!/bin/sh\necho 'Pulse v6.5.0'\n"], r)+ self.assertEqual(r["runtime_probes"], [], r)+ self.assertEqual(len(r["calls"]), 2, r)+ self.assertNotIn("Update successfully installed and verified", r["output"])+ self.assertNotIn("private fixture manager diagnostic", r["output"] + r["error"])++ def test_manager_diagnostics_after_stop_retain_backup_without_restart(self):+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):+ for state in ("active", "inactive"):+ with self.subTest(fault=fault, state=state):+ r = self.run_fixture(observation_fault=fault, observation_phase="post-stop",+ state=state, installer_state="active")+ self.assertEqual(r["exit"], 1, r)+ self.assertTrue(r["installed"] and r["stopped"], r)+ self.assertFalse(r["started"], r)+ self.assertEqual(r["state"], "inactive", r)+ # The stop was authorised by the previous valid read;+ # incomplete readback permits neither restore nor restart.+ self.assertEqual(r["version"], "v6.6.0\n", r)+ self.assertEqual(r["backups"], [b"#!/bin/sh\necho 'Pulse v6.5.0'\n"], r)+ self.assertEqual(len(r["calls"]), 4, r)+ self.assertIn("manual recovery required", r["output"])+ self.assertNotIn("Previous installation restored", r["output"])+ self.assertNotIn("private fixture manager diagnostic", r["output"] + r["error"])+def test_success_cannot_claim_unverified_running_executable(self):# A matching new disk version and an active manager are insufficient.# Preserve the currently active service; uncertainty grants no stop,@@ -585,9 +646,19 @@ if mode == "pid-moved" and count > 1: pid = os.environ["DIFFERENT_PID"]load = "not-found" if mode == "not-found" else ("masked" if mode == "masked" else "loaded")state = "inactive" if mode == "inactive" or (mode == "state-moved" and count > 1) else "active"# Deliberately reverse property order, as systemd does not promise an order.-print("MainPID=" + pid)-if mode != "missing": print("ActiveState=" + state)-print("LoadState=" + load)+payload = "MainPID=" + pid + "\n"+if mode != "missing": payload += "ActiveState=" + state + "\n"+payload += "LoadState=" + load + "\n"+fault = ""+if mode.startswith("diagnostic:"):+ _, selected_fault, selected_read = mode.split(":")+ if count == int(selected_read): fault = selected_fault+if fault == "stdout-blank": payload += "\n"+if fault == "stdout-unterminated": payload = payload.rstrip("\n")+sys.stdout.write(payload)+sys.stdout.flush()+if fault == "stderr-warning": sys.stderr.write("private fixture manager diagnostic\n")+if fault == "stderr-blank": sys.stderr.write("\n")if mode == "duplicate": print("MainPID=" + pid)if mode == "extra": print("Unknown=value")'''@@ -654,6 +725,18 @@ class RunningExecutable(unittest.TestCase):self.assertEqual(r.returncode, 0 if accepted else 2, r.stderr)self.assertEqual(observations, 2 if accepted or mode in ("pid-moved", "state-moved") else 1)+ def test_manager_diagnostics_refuse_either_running_identity_read(self):+ # These read the actual private child's /proc executable before and+ # after the unchanged byte comparison, not a simulated version file.+ for fault in ("stderr-warning", "stderr-blank", "stdout-blank", "stdout-unterminated"):+ for reading in (1, 2):+ with self.subTest(fault=fault, reading=reading):+ r, _, observations = self.run_identity(mode=f"diagnostic:{fault}:{reading}")+ self.assertEqual(r.returncode, 2, r.stderr)+ self.assertIn(b"IDENTITY_STATUS=2", r.stdout)+ self.assertEqual(observations, reading)+ self.assertNotIn(b"private fixture manager diagnostic", r.stdout + r.stderr)+def test_identity_read_timeout_is_bounded_and_owned_process_is_reaped(self):r, elapsed, observations = self.run_identity(mode="hang")self.assertEqual(r.returncode, 2)