Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep updater collector failures and final installer evidence
pulse-triage[bot]a04c246adfc8
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex bdabb1cec4..cccb5d6a0e 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -85,11 +85,17 @@ published installer trust, #2785's stop cause or installed acceptance.The helper records the installer and log collector's separate observed exitstatuses immediately after their pipeline, before logging or recovery can-overwrite them. A failed collector does not establish installer failure;-zero exits establish pipeline completion only, not version, service or update-acceptance. The existing signed transaction and confined service-state tests-cover installer-only, collector-only, simultaneous and successful exits with-unchanged prior-activity, mask, backup and restoration safeguards. These facts+overwrite them. The collector drains installer output even after a log-write+failure, retains the first failure across later successful writes, and includes+a final line without a newline. It must not close the installer pipe early or+admit an update because a later write hid missing diagnostic evidence.+A failed collector does not establish installer failure; zero exits establish+pipeline completion only, not version, service or update acceptance.+The existing signed transaction and confined service-state tests+cover installer-only, collector-only, simultaneous and successful exits,+early-write failures followed by success, complete pipe draining and+unterminated final evidence with unchanged prior-activity, mask, backup and+restoration safeguards. These factscannot reconstruct exits or signals from an earlier interrupted attempt, oridentify who stopped a host.diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.goindex 33ff734129..3369810ae9 100644--- a/scripts/installtests/pulse_auto_update_test.go+++ b/scripts/installtests/pulse_auto_update_test.go@@ -40,6 +40,23 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {if err != nil {t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)}+ // Keep the collector controls part of actual package acceptance, not just+ // an optional Python check. A green fixture with these methods removed+ // would otherwise silently lose early-failure and final-line coverage.+ for _, name := range []string{+ "test_early_collector_failure_survives_later_success_and_drains_installer",+ "test_unterminated_final_installer_line_is_collected_and_its_failure_retained",+ } {+ passed := false+ for _, line := range strings.Split(string(out), "\n") {+ if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") {+ passed = true+ }+ }+ if !passed {+ t.Fatalf("required updater collector control %s did not pass:\n%s", name, out)+ }+ }}func extractAutoUpdateServiceFunctions(t *testing.T) string {diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.shindex 71ee990020..94dd6095b8 100755--- a/scripts/pulse-auto-update.sh+++ b/scripts/pulse-auto-update.sh@@ -579,10 +579,25 @@ perform_update() {"PULSE_SERVICE_NAME=$service_name" \"PULSE_INSTALL_DIR=$INSTALL_DIR" \"PULSE_CONFIG_DIR=$CONFIG_DIR" \- bash "$installer_tmp" "${installer_args[@]}" 2>&1 | \- while IFS= read -r line; do- log info "installer: $line"- done; then+ bash "$installer_tmp" "${installer_args[@]}" 2>&1 | (+ collector_status=0+ line=""+ # Drain even after a failed log write: closing this pipe early can+ # interrupt an installer that is still replacing files. Retain the+ # first failure, not just the last successful write, and include a+ # final line without a newline in the completion evidence.+ while IFS= read -r line || [[ -n "$line" ]]; do+ if log info "installer: $line"; then+ :+ else+ log_status=$?+ if [[ "$collector_status" == 0 ]]; then+ collector_status=$log_status+ fi+ fi+ done+ exit "$collector_status"+ ); then# Capture both observed exits before any command overwrites PIPESTATUS.# Completion of this pipeline still requires binary/service verification.installer_pipeline_status=("${PIPESTATUS[@]}")diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.pyindex 55d5e0821d..5e7db24fca 100644--- a/scripts/tests/test_auto_update_service_state.py+++ b/scripts/tests/test_auto_update_service_state.py@@ -68,7 +68,14 @@ HARNESS = r'''source "$UPDATER"log() {printf '[%s] %s\n' "$1" "${*:2}"- if [[ "${2:-}" == 'installer: '* ]]; then return "$COLLECTOR_EXIT"; fi+ if [[ "${2:-}" == 'installer: '* ]]; then+ local count=0 status_name=""+ if [[ -f "$FIXTURE/collector-count" ]]; then read -r count < "$FIXTURE/collector-count"; fi+ ((count += 1))+ printf '%s\n' "$count" > "$FIXTURE/collector-count"+ status_name="COLLECTOR_EXIT_$count"+ return "${!status_name:-$COLLECTOR_EXIT}"+ fi}detect_service_name() { printf '%s\n' "$SERVICE_NAME"; }sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.@@ -109,7 +116,7 @@ exit "$status"class UpdateServiceState(unittest.TestCase):def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",- installer_exit=None, collector_exit=0):+ installer_exit=None, collector_exit=0, installer_output=None, collector_failures=None):with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:f = Path(directory)for name in ("tools", "tmp", "install/bin", "config"):@@ -125,13 +132,15 @@ class UpdateServiceState(unittest.TestCase):(f / "config/agent-id").write_text('original identity\n')(f / "state").write_text(state + "\n")(f / "new-pulse").write_bytes(new)+ (f / "installer-output").write_text("fixture installation output\n" if installer_output is None else installer_output)(f / "installer").write_text('set -eu\n''echo attempted > "$FIXTURE/installer-ran"\n''cp "$FIXTURE/new-pulse" "$PULSE_INSTALL_DIR/bin/pulse"\n''echo v6.6.0 > "$PULSE_INSTALL_DIR/VERSION"\n''echo "$INSTALLER_STATE" > "$FIXTURE/state"\n'- 'printf "fixture installation output\\n"\n'+ 'cat "$FIXTURE/installer-output"\n'+ 'echo completed > "$FIXTURE/installer-finished"\n''exit "$INSTALLER_EXIT"\n')env = dict(os.environ)env.update(FIXTURE=str(f), UPDATER=str(UPDATER), MODE=mode, FLOW=flow,@@ -141,6 +150,8 @@ class UpdateServiceState(unittest.TestCase):INSTALLER_EXIT=str(installer_exit) if installer_exit is not None else ("0" if success else "23"),COLLECTOR_EXIT=str(collector_exit),PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])+ for line, status in (collector_failures or {}).items():+ env[f"COLLECTOR_EXIT_{line}"] = str(status)start = time.monotonic()result = subprocess.run(["bash", "-c", HARNESS], env=env, capture_output=True, timeout=15)calls = [json.loads(line) for line in (f / "calls").read_text().splitlines()] if (f / "calls").exists() else []@@ -149,6 +160,8 @@ class UpdateServiceState(unittest.TestCase):elapsed=time.monotonic() - start, calls=calls,downloaded=(f / "downloads").exists(), temporary=(f / "temp-requests").exists(),installed=(f / "installer-ran").exists(), stopped=(f / "stop-ran").exists(),+ installer_finished=(f / "installer-finished").exists(),+ collected=int((f / "collector-count").read_text()) if (f / "collector-count").exists() else 0,started=(f / "start-ran").exists(), state=(f / "state").read_text().strip(),binary=(f / "install/bin/pulse").read_bytes(), version=(f / "install/VERSION").read_text(),backups=[(b / "pulse-bin").read_bytes() for b in backups],@@ -237,6 +250,43 @@ class UpdateServiceState(unittest.TestCase):self.assertEqual(r["started"], state == "active", r)self.assertEqual(r["backups"], [], r)+ def test_early_collector_failure_survives_later_success_and_drains_installer(self):+ # Exceed a pipe buffer so an early collector exit would interrupt the+ # installer rather than merely miss its final evidence. No host+ # installer or logging service is invoked by this confined fixture.+ lines = ["opening"] + ["x" * 2048 for _ in range(64)] + ["final installer evidence"]+ for state in ("active", "inactive"):+ for failures in ({1: 73}, {1: 73, 2: 42}):+ with self.subTest(state=state, failures=failures):+ r = self.run_fixture(state=state, success=True, installer_output="\n".join(lines) + "\n",+ collector_failures=failures)+ self.assertIn("Installer pipeline failed (installer exit: 0; log collector exit: 73)", r["output"], r)+ self.assertTrue(r["installer_finished"], r)+ self.assertEqual(r["collected"], len(lines), r)+ self.assertIn("installer: final installer evidence", r["output"], r)+ self.assertNotIn("Update successfully installed and verified", r["output"], r)+ self.assertEqual(r["exit"], 1, r)+ self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n", r)+ self.assertEqual(r["version"], "v6.5.0\n", r)+ self.assertEqual(r["started"], state == "active", r)+ self.assertEqual(r["backups"], [], r)++ def test_unterminated_final_installer_line_is_collected_and_its_failure_retained(self):+ for status in (0, 73):+ with self.subTest(status=status):+ r = self.run_fixture(state="inactive", success=True, installer_output="opening\nfinal evidence without newline",+ collector_failures={2: status})+ self.assertIn("installer: final evidence without newline", r["output"], r)+ self.assertTrue(r["installer_finished"], r)+ self.assertEqual(r["collected"], 2, r)+ self.assertEqual(r["exit"], 0 if status == 0 else 1, r)+ self.assertEqual(r["version"], "v6.6.0\n" if status == 0 else "v6.5.0\n", r)+ self.assertFalse(r["started"], r)+ self.assertEqual(r["backups"], [], r)+ expected = ("Installer completed; verifying update (installer exit: 0; log collector exit: 0)"+ if status == 0 else "Installer pipeline failed (installer exit: 0; log collector exit: 73)")+ self.assertIn(expected, r["output"], r)+def test_uncertain_rollback_stop_retains_backup_without_replacement_or_start(self):for mode in ("stop-noop", "stop-error", "readback-error", "readback-empty", "after-error"):with self.subTest(mode=mode):