Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep updater collector failures and final installer evidence

pulse-triage[bot]a04c246adfc8
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index bdabb1cec4..cccb5d6a0e 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -85,11 +85,17 @@ published installer trust, #2785's stop cause or installed acceptance.
The helper records the installer and log collector's separate observed exit
statuses immediately after their pipeline, before logging or recovery can
-overwrite them. A failed collector does not establish installer failure;
-zero exits establish pipeline completion only, not version, service or update
-acceptance. The existing signed transaction and confined service-state tests
-cover installer-only, collector-only, simultaneous and successful exits with
-unchanged prior-activity, mask, backup and restoration safeguards. These facts
+overwrite them. The collector drains installer output even after a log-write
+failure, retains the first failure across later successful writes, and includes
+a final line without a newline. It must not close the installer pipe early or
+admit an update because a later write hid missing diagnostic evidence.
+A failed collector does not establish installer failure; zero exits establish
+pipeline completion only, not version, service or update acceptance.
+The existing signed transaction and confined service-state tests
+cover installer-only, collector-only, simultaneous and successful exits,
+early-write failures followed by success, complete pipe draining and
+unterminated final evidence with unchanged prior-activity, mask, backup and
+restoration safeguards. These facts
cannot reconstruct exits or signals from an earlier interrupted attempt, or
identify who stopped a host.
diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go
index 33ff734129..3369810ae9 100644
--- a/scripts/installtests/pulse_auto_update_test.go
+++ b/scripts/installtests/pulse_auto_update_test.go
@@ -40,6 +40,23 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {
if err != nil {
t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)
}
+ // Keep the collector controls part of actual package acceptance, not just
+ // an optional Python check. A green fixture with these methods removed
+ // would otherwise silently lose early-failure and final-line coverage.
+ for _, name := range []string{
+ "test_early_collector_failure_survives_later_success_and_drains_installer",
+ "test_unterminated_final_installer_line_is_collected_and_its_failure_retained",
+ } {
+ passed := false
+ for _, line := range strings.Split(string(out), "\n") {
+ if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") {
+ passed = true
+ }
+ }
+ if !passed {
+ t.Fatalf("required updater collector control %s did not pass:\n%s", name, out)
+ }
+ }
}
func extractAutoUpdateServiceFunctions(t *testing.T) string {
diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.sh
index 71ee990020..94dd6095b8 100755
--- a/scripts/pulse-auto-update.sh
+++ b/scripts/pulse-auto-update.sh
@@ -579,10 +579,25 @@ perform_update() {
"PULSE_SERVICE_NAME=$service_name" \
"PULSE_INSTALL_DIR=$INSTALL_DIR" \
"PULSE_CONFIG_DIR=$CONFIG_DIR" \
- bash "$installer_tmp" "${installer_args[@]}" 2>&1 | \
- while IFS= read -r line; do
- log info "installer: $line"
- done; then
+ bash "$installer_tmp" "${installer_args[@]}" 2>&1 | (
+ collector_status=0
+ line=""
+ # Drain even after a failed log write: closing this pipe early can
+ # interrupt an installer that is still replacing files. Retain the
+ # first failure, not just the last successful write, and include a
+ # final line without a newline in the completion evidence.
+ while IFS= read -r line || [[ -n "$line" ]]; do
+ if log info "installer: $line"; then
+ :
+ else
+ log_status=$?
+ if [[ "$collector_status" == 0 ]]; then
+ collector_status=$log_status
+ fi
+ fi
+ done
+ exit "$collector_status"
+ ); then
# Capture both observed exits before any command overwrites PIPESTATUS.
# Completion of this pipeline still requires binary/service verification.
installer_pipeline_status=("${PIPESTATUS[@]}")
diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.py
index 55d5e0821d..5e7db24fca 100644
--- a/scripts/tests/test_auto_update_service_state.py
+++ b/scripts/tests/test_auto_update_service_state.py
@@ -68,7 +68,14 @@ HARNESS = r'''
source "$UPDATER"
log() {
printf '[%s] %s\n' "$1" "${*:2}"
- if [[ "${2:-}" == 'installer: '* ]]; then return "$COLLECTOR_EXIT"; fi
+ if [[ "${2:-}" == 'installer: '* ]]; then
+ local count=0 status_name=""
+ if [[ -f "$FIXTURE/collector-count" ]]; then read -r count < "$FIXTURE/collector-count"; fi
+ ((count += 1))
+ printf '%s\n' "$count" > "$FIXTURE/collector-count"
+ status_name="COLLECTOR_EXIT_$count"
+ return "${!status_name:-$COLLECTOR_EXIT}"
+ fi
}
detect_service_name() { printf '%s\n' "$SERVICE_NAME"; }
sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.
@@ -109,7 +116,7 @@ exit "$status"
class UpdateServiceState(unittest.TestCase):
def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",
- installer_exit=None, collector_exit=0):
+ installer_exit=None, collector_exit=0, installer_output=None, collector_failures=None):
with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:
f = Path(directory)
for name in ("tools", "tmp", "install/bin", "config"):
@@ -125,13 +132,15 @@ class UpdateServiceState(unittest.TestCase):
(f / "config/agent-id").write_text('original identity\n')
(f / "state").write_text(state + "\n")
(f / "new-pulse").write_bytes(new)
+ (f / "installer-output").write_text("fixture installation output\n" if installer_output is None else installer_output)
(f / "installer").write_text(
'set -eu\n'
'echo attempted > "$FIXTURE/installer-ran"\n'
'cp "$FIXTURE/new-pulse" "$PULSE_INSTALL_DIR/bin/pulse"\n'
'echo v6.6.0 > "$PULSE_INSTALL_DIR/VERSION"\n'
'echo "$INSTALLER_STATE" > "$FIXTURE/state"\n'
- 'printf "fixture installation output\\n"\n'
+ 'cat "$FIXTURE/installer-output"\n'
+ 'echo completed > "$FIXTURE/installer-finished"\n'
'exit "$INSTALLER_EXIT"\n')
env = dict(os.environ)
env.update(FIXTURE=str(f), UPDATER=str(UPDATER), MODE=mode, FLOW=flow,
@@ -141,6 +150,8 @@ class UpdateServiceState(unittest.TestCase):
INSTALLER_EXIT=str(installer_exit) if installer_exit is not None else ("0" if success else "23"),
COLLECTOR_EXIT=str(collector_exit),
PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])
+ for line, status in (collector_failures or {}).items():
+ env[f"COLLECTOR_EXIT_{line}"] = str(status)
start = time.monotonic()
result = subprocess.run(["bash", "-c", HARNESS], env=env, capture_output=True, timeout=15)
calls = [json.loads(line) for line in (f / "calls").read_text().splitlines()] if (f / "calls").exists() else []
@@ -149,6 +160,8 @@ class UpdateServiceState(unittest.TestCase):
elapsed=time.monotonic() - start, calls=calls,
downloaded=(f / "downloads").exists(), temporary=(f / "temp-requests").exists(),
installed=(f / "installer-ran").exists(), stopped=(f / "stop-ran").exists(),
+ installer_finished=(f / "installer-finished").exists(),
+ collected=int((f / "collector-count").read_text()) if (f / "collector-count").exists() else 0,
started=(f / "start-ran").exists(), state=(f / "state").read_text().strip(),
binary=(f / "install/bin/pulse").read_bytes(), version=(f / "install/VERSION").read_text(),
backups=[(b / "pulse-bin").read_bytes() for b in backups],
@@ -237,6 +250,43 @@ class UpdateServiceState(unittest.TestCase):
self.assertEqual(r["started"], state == "active", r)
self.assertEqual(r["backups"], [], r)
+ def test_early_collector_failure_survives_later_success_and_drains_installer(self):
+ # Exceed a pipe buffer so an early collector exit would interrupt the
+ # installer rather than merely miss its final evidence. No host
+ # installer or logging service is invoked by this confined fixture.
+ lines = ["opening"] + ["x" * 2048 for _ in range(64)] + ["final installer evidence"]
+ for state in ("active", "inactive"):
+ for failures in ({1: 73}, {1: 73, 2: 42}):
+ with self.subTest(state=state, failures=failures):
+ r = self.run_fixture(state=state, success=True, installer_output="\n".join(lines) + "\n",
+ collector_failures=failures)
+ self.assertIn("Installer pipeline failed (installer exit: 0; log collector exit: 73)", r["output"], r)
+ self.assertTrue(r["installer_finished"], r)
+ self.assertEqual(r["collected"], len(lines), r)
+ self.assertIn("installer: final installer evidence", r["output"], r)
+ self.assertNotIn("Update successfully installed and verified", r["output"], r)
+ self.assertEqual(r["exit"], 1, r)
+ self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n", r)
+ self.assertEqual(r["version"], "v6.5.0\n", r)
+ self.assertEqual(r["started"], state == "active", r)
+ self.assertEqual(r["backups"], [], r)
+
+ def test_unterminated_final_installer_line_is_collected_and_its_failure_retained(self):
+ for status in (0, 73):
+ with self.subTest(status=status):
+ r = self.run_fixture(state="inactive", success=True, installer_output="opening\nfinal evidence without newline",
+ collector_failures={2: status})
+ self.assertIn("installer: final evidence without newline", r["output"], r)
+ self.assertTrue(r["installer_finished"], r)
+ self.assertEqual(r["collected"], 2, r)
+ self.assertEqual(r["exit"], 0 if status == 0 else 1, r)
+ self.assertEqual(r["version"], "v6.6.0\n" if status == 0 else "v6.5.0\n", r)
+ self.assertFalse(r["started"], r)
+ self.assertEqual(r["backups"], [], r)
+ expected = ("Installer completed; verifying update (installer exit: 0; log collector exit: 0)"
+ if status == 0 else "Installer pipeline failed (installer exit: 0; log collector exit: 73)")
+ self.assertIn(expected, r["output"], r)
+
def test_uncertain_rollback_stop_retains_backup_without_replacement_or_start(self):
for mode in ("stop-noop", "stop-error", "readback-error", "readback-empty", "after-error"):
with self.subTest(mode=mode):