Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Record unattended installer and log-collector exits separately
pulse-triage[bot]74a8d0d05ac2
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex d181a2898e..fcc9b02621 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -83,6 +83,16 @@ matrix and smoke controls retain their cryptographic, metadata and return-trapcoverage. These are source/fixture controls, not native systemd/LXC recovery,published installer trust, #2785's stop cause or installed acceptance.+The helper records the installer and log collector's separate observed exit+statuses immediately after their pipeline, before logging or recovery can+overwrite them. A failed collector does not establish installer failure;+zero exits establish pipeline completion only, not version, service or update+acceptance. The existing signed transaction and confined service-state tests+cover installer-only, collector-only, simultaneous and successful exits with+unchanged prior-activity, mask, backup and restoration safeguards. These facts+cannot reconstruct exits or signals from an earlier interrupted attempt, or+identify who stopped a host.+### Core E2E images are built once and admitted in the same runThe secret-free Core E2E workflow builds its existing `e2e_runtime` server anddiff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.goindex 484c97bf64..33ff734129 100644--- a/scripts/installtests/pulse_auto_update_test.go+++ b/scripts/installtests/pulse_auto_update_test.go@@ -1020,6 +1020,17 @@ printf 'TRANSACTION_ASSERTIONS_PASSED\n'if !strings.Contains(string(out), "TRANSACTION_ASSERTIONS_PASSED") {t.Fatalf("transaction assertions did not finish:\n%s", out)}+ if !preflight {+ installerExit := "0"+ switch outcome {+ case "failure", "symlink", "directory", "fifo":+ installerExit = "23"+ }+ observed := "(installer exit: " + installerExit + "; log collector exit: 0)"+ if !strings.Contains(string(out), observed) {+ t.Fatalf("missing observed pipeline exits %q after signed installer execution:\n%s", observed, out)+ }+ }}// Stable admission is object-bound, not a line-oriented tag/flag accumulator.diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.shindex e41683d968..71ee990020 100755--- a/scripts/pulse-auto-update.sh+++ b/scripts/pulse-auto-update.sh@@ -574,6 +574,7 @@ perform_update() {log info "Installer signature verified"local update_accepted="false"+ local -a installer_pipeline_status=()if env \"PULSE_SERVICE_NAME=$service_name" \"PULSE_INSTALL_DIR=$INSTALL_DIR" \@@ -582,8 +583,10 @@ perform_update() {while IFS= read -r line; dolog info "installer: $line"done; then-- log info "Installer completed; verifying update"+ # Capture both observed exits before any command overwrites PIPESTATUS.+ # Completion of this pipeline still requires binary/service verification.+ installer_pipeline_status=("${PIPESTATUS[@]}")+ log info "Installer completed; verifying update (installer exit: ${installer_pipeline_status[0]}; log collector exit: ${installer_pipeline_status[1]})"# Verify new versionlocal installed_version@@ -632,7 +635,10 @@ perform_update() {log error "Version mismatch after update. Expected: $new_version, Got: $installed_version"fielse- log error "Update installation failed"+ installer_pipeline_status=("${PIPESTATUS[@]}")+ # A collector failure is not evidence that the installer itself failed.+ # These are exit observations, not a diagnosis of a host interruption.+ log error "Installer pipeline failed (installer exit: ${installer_pipeline_status[0]}; log collector exit: ${installer_pipeline_status[1]})"fiif [[ "$update_accepted" == "true" ]]; thendiff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.pyindex 7950d7d2c5..55d5e0821d 100644--- a/scripts/tests/test_auto_update_service_state.py+++ b/scripts/tests/test_auto_update_service_state.py@@ -66,7 +66,10 @@ sys.exit(97)HARNESS = r'''source "$UPDATER"-log() { printf '[%s] %s\n' "$1" "${*:2}"; }+log() {+ printf '[%s] %s\n' "$1" "${*:2}"+ if [[ "${2:-}" == 'installer: '* ]]; then return "$COLLECTOR_EXIT"; fi+}detect_service_name() { printf '%s\n' "$SERVICE_NAME"; }sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.verify_release_signature() { :; }@@ -105,7 +108,8 @@ exit "$status"class UpdateServiceState(unittest.TestCase):- def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update"):+ def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",+ installer_exit=None, collector_exit=0):with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:f = Path(directory)for name in ("tools", "tmp", "install/bin", "config"):@@ -127,13 +131,15 @@ class UpdateServiceState(unittest.TestCase):'cp "$FIXTURE/new-pulse" "$PULSE_INSTALL_DIR/bin/pulse"\n''echo v6.6.0 > "$PULSE_INSTALL_DIR/VERSION"\n''echo "$INSTALLER_STATE" > "$FIXTURE/state"\n'+ 'printf "fixture installation output\\n"\n''exit "$INSTALLER_EXIT"\n')env = dict(os.environ)env.update(FIXTURE=str(f), UPDATER=str(UPDATER), MODE=mode, FLOW=flow,PULSE_INSTALL_DIR=str(f / "install"), PULSE_CONFIG_DIR=str(f / "config"),PULSE_SERVICE_NAME="pulse-custom@department", LOAD_STATE=load,INSTALLER_STATE="active" if mode in ("stop-noop", "stop-error", "stop-hang", "after-error", "readback-error", "readback-empty") else "inactive",- INSTALLER_EXIT="0" if success else "23",+ INSTALLER_EXIT=str(installer_exit) if installer_exit is not None else ("0" if success else "23"),+ COLLECTOR_EXIT=str(collector_exit),PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])start = time.monotonic()result = subprocess.run(["bash", "-c", HARNESS], env=env, capture_output=True, timeout=15)@@ -197,6 +203,40 @@ class UpdateServiceState(unittest.TestCase):self.assertFalse(r["stopped"], r)self.assertEqual(r["backups"], [], r)+ def test_pipeline_failure_records_both_exits_before_unchanged_recovery(self):+ # Distinct simultaneous failures detect swapped or overwritten statuses;+ # collector-only failure must not be misreported as installer failure.+ for installer, collector in ((23, 0), (0, 73), (42, 73)):+ for state, load in (("active", "loaded"), ("inactive", "loaded"),+ ("failed", "loaded"), ("inactive", "masked")):+ with self.subTest(installer=installer, collector=collector, state=state, load=load):+ r = self.run_fixture(state=state, load=load, installer_exit=installer,+ collector_exit=collector)+ observed = f"Installer pipeline failed (installer exit: {installer}; log collector exit: {collector})"+ self.assertIn(observed, r["output"], r)+ self.assertLess(r["output"].index(observed), r["output"].index("Restoring from backup"), r)+ self.assertNotIn("Installer completed", r["output"], r)+ self.assertNotIn("Update successfully installed and verified", r["output"], r)+ self.assertEqual(r["exit"], 1, r)+ self.assertTrue(r["installed"], r)+ self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n", r)+ self.assertEqual(r["version"], "v6.5.0\n", r)+ self.assertEqual(r["started"], state == "active", r)+ self.assertEqual(r["backups"], [], r)++ def test_completed_pipeline_records_exits_without_claiming_verification(self):+ for state in ("active", "inactive"):+ with self.subTest(state=state):+ r = self.run_fixture(state=state, success=True)+ observed = "Installer completed; verifying update (installer exit: 0; log collector exit: 0)"+ self.assertIn(observed, r["output"], r)+ self.assertLess(r["output"].index(observed), r["output"].index("Version verified:"), r)+ self.assertNotIn("Installer pipeline failed", r["output"], r)+ self.assertEqual(r["exit"], 0, r)+ self.assertIn(b"v6.6.0", r["binary"], r)+ self.assertEqual(r["started"], state == "active", r)+ self.assertEqual(r["backups"], [], r)+def test_uncertain_rollback_stop_retains_backup_without_replacement_or_start(self):for mode in ("stop-noop", "stop-error", "readback-error", "readback-empty", "after-error"):with self.subTest(mode=mode):