Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Record unattended installer and log-collector exits separately

pulse-triage[bot]74a8d0d05ac2
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index d181a2898e..fcc9b02621 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -83,6 +83,16 @@ matrix and smoke controls retain their cryptographic, metadata and return-trap
coverage. These are source/fixture controls, not native systemd/LXC recovery,
published installer trust, #2785's stop cause or installed acceptance.
+The helper records the installer and log collector's separate observed exit
+statuses immediately after their pipeline, before logging or recovery can
+overwrite them. A failed collector does not establish installer failure;
+zero exits establish pipeline completion only, not version, service or update
+acceptance. The existing signed transaction and confined service-state tests
+cover installer-only, collector-only, simultaneous and successful exits with
+unchanged prior-activity, mask, backup and restoration safeguards. These facts
+cannot reconstruct exits or signals from an earlier interrupted attempt, or
+identify who stopped a host.
+
### Core E2E images are built once and admitted in the same run
The secret-free Core E2E workflow builds its existing `e2e_runtime` server and
diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go
index 484c97bf64..33ff734129 100644
--- a/scripts/installtests/pulse_auto_update_test.go
+++ b/scripts/installtests/pulse_auto_update_test.go
@@ -1020,6 +1020,17 @@ printf 'TRANSACTION_ASSERTIONS_PASSED\n'
if !strings.Contains(string(out), "TRANSACTION_ASSERTIONS_PASSED") {
t.Fatalf("transaction assertions did not finish:\n%s", out)
}
+ if !preflight {
+ installerExit := "0"
+ switch outcome {
+ case "failure", "symlink", "directory", "fifo":
+ installerExit = "23"
+ }
+ observed := "(installer exit: " + installerExit + "; log collector exit: 0)"
+ if !strings.Contains(string(out), observed) {
+ t.Fatalf("missing observed pipeline exits %q after signed installer execution:\n%s", observed, out)
+ }
+ }
}
// Stable admission is object-bound, not a line-oriented tag/flag accumulator.
diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.sh
index e41683d968..71ee990020 100755
--- a/scripts/pulse-auto-update.sh
+++ b/scripts/pulse-auto-update.sh
@@ -574,6 +574,7 @@ perform_update() {
log info "Installer signature verified"
local update_accepted="false"
+ local -a installer_pipeline_status=()
if env \
"PULSE_SERVICE_NAME=$service_name" \
"PULSE_INSTALL_DIR=$INSTALL_DIR" \
@@ -582,8 +583,10 @@ perform_update() {
while IFS= read -r line; do
log info "installer: $line"
done; then
-
- log info "Installer completed; verifying update"
+ # Capture both observed exits before any command overwrites PIPESTATUS.
+ # Completion of this pipeline still requires binary/service verification.
+ installer_pipeline_status=("${PIPESTATUS[@]}")
+ log info "Installer completed; verifying update (installer exit: ${installer_pipeline_status[0]}; log collector exit: ${installer_pipeline_status[1]})"
# Verify new version
local installed_version
@@ -632,7 +635,10 @@ perform_update() {
log error "Version mismatch after update. Expected: $new_version, Got: $installed_version"
fi
else
- log error "Update installation failed"
+ installer_pipeline_status=("${PIPESTATUS[@]}")
+ # A collector failure is not evidence that the installer itself failed.
+ # These are exit observations, not a diagnosis of a host interruption.
+ log error "Installer pipeline failed (installer exit: ${installer_pipeline_status[0]}; log collector exit: ${installer_pipeline_status[1]})"
fi
if [[ "$update_accepted" == "true" ]]; then
diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.py
index 7950d7d2c5..55d5e0821d 100644
--- a/scripts/tests/test_auto_update_service_state.py
+++ b/scripts/tests/test_auto_update_service_state.py
@@ -66,7 +66,10 @@ sys.exit(97)
HARNESS = r'''
source "$UPDATER"
-log() { printf '[%s] %s\n' "$1" "${*:2}"; }
+log() {
+ printf '[%s] %s\n' "$1" "${*:2}"
+ if [[ "${2:-}" == 'installer: '* ]]; then return "$COLLECTOR_EXIT"; fi
+}
detect_service_name() { printf '%s\n' "$SERVICE_NAME"; }
sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.
verify_release_signature() { :; }
@@ -105,7 +108,8 @@ exit "$status"
class UpdateServiceState(unittest.TestCase):
- def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update"):
+ def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",
+ installer_exit=None, collector_exit=0):
with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:
f = Path(directory)
for name in ("tools", "tmp", "install/bin", "config"):
@@ -127,13 +131,15 @@ class UpdateServiceState(unittest.TestCase):
'cp "$FIXTURE/new-pulse" "$PULSE_INSTALL_DIR/bin/pulse"\n'
'echo v6.6.0 > "$PULSE_INSTALL_DIR/VERSION"\n'
'echo "$INSTALLER_STATE" > "$FIXTURE/state"\n'
+ 'printf "fixture installation output\\n"\n'
'exit "$INSTALLER_EXIT"\n')
env = dict(os.environ)
env.update(FIXTURE=str(f), UPDATER=str(UPDATER), MODE=mode, FLOW=flow,
PULSE_INSTALL_DIR=str(f / "install"), PULSE_CONFIG_DIR=str(f / "config"),
PULSE_SERVICE_NAME="pulse-custom@department", LOAD_STATE=load,
INSTALLER_STATE="active" if mode in ("stop-noop", "stop-error", "stop-hang", "after-error", "readback-error", "readback-empty") else "inactive",
- INSTALLER_EXIT="0" if success else "23",
+ INSTALLER_EXIT=str(installer_exit) if installer_exit is not None else ("0" if success else "23"),
+ COLLECTOR_EXIT=str(collector_exit),
PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])
start = time.monotonic()
result = subprocess.run(["bash", "-c", HARNESS], env=env, capture_output=True, timeout=15)
@@ -197,6 +203,40 @@ class UpdateServiceState(unittest.TestCase):
self.assertFalse(r["stopped"], r)
self.assertEqual(r["backups"], [], r)
+ def test_pipeline_failure_records_both_exits_before_unchanged_recovery(self):
+ # Distinct simultaneous failures detect swapped or overwritten statuses;
+ # collector-only failure must not be misreported as installer failure.
+ for installer, collector in ((23, 0), (0, 73), (42, 73)):
+ for state, load in (("active", "loaded"), ("inactive", "loaded"),
+ ("failed", "loaded"), ("inactive", "masked")):
+ with self.subTest(installer=installer, collector=collector, state=state, load=load):
+ r = self.run_fixture(state=state, load=load, installer_exit=installer,
+ collector_exit=collector)
+ observed = f"Installer pipeline failed (installer exit: {installer}; log collector exit: {collector})"
+ self.assertIn(observed, r["output"], r)
+ self.assertLess(r["output"].index(observed), r["output"].index("Restoring from backup"), r)
+ self.assertNotIn("Installer completed", r["output"], r)
+ self.assertNotIn("Update successfully installed and verified", r["output"], r)
+ self.assertEqual(r["exit"], 1, r)
+ self.assertTrue(r["installed"], r)
+ self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n", r)
+ self.assertEqual(r["version"], "v6.5.0\n", r)
+ self.assertEqual(r["started"], state == "active", r)
+ self.assertEqual(r["backups"], [], r)
+
+ def test_completed_pipeline_records_exits_without_claiming_verification(self):
+ for state in ("active", "inactive"):
+ with self.subTest(state=state):
+ r = self.run_fixture(state=state, success=True)
+ observed = "Installer completed; verifying update (installer exit: 0; log collector exit: 0)"
+ self.assertIn(observed, r["output"], r)
+ self.assertLess(r["output"].index(observed), r["output"].index("Version verified:"), r)
+ self.assertNotIn("Installer pipeline failed", r["output"], r)
+ self.assertEqual(r["exit"], 0, r)
+ self.assertIn(b"v6.6.0", r["binary"], r)
+ self.assertEqual(r["started"], state == "active", r)
+ self.assertEqual(r["backups"], [], r)
+
def test_uncertain_rollback_stop_retains_backup_without_replacement_or_start(self):
for mode in ("stop-noop", "stop-error", "readback-error", "readback-empty", "after-error"):
with self.subTest(mode=mode):