Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Retain real unattended updater log-sink failures
pulse-triage[bot]a8fadc30e46e
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 70eda1305e..b31edf9c11 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -113,6 +113,16 @@ overwrite them. The collector drains installer output even after a log-writefailure, retains the first failure across later successful writes, and includesa final line without a newline. It must not close the installer pipe early oradmit an update because a later write hid missing diagnostic evidence.++The production `log()` primitive attempts both its system logger and stdout+sinks and returns the first failed sink's exit even under a conditional caller+that disables Bash errexit. A successful stdout write cannot conceal a failed+logger write, nor a successful logger conceal failed stdout. Confined controls+keep `log()` intact, substitute only the external logger and exercise closed+stdout; the connected collector control drains more than a pipe buffer through+an early, later or final unterminated-line logger failure, records both pipeline+exits and preserves the existing verified restoration/prior-active rules. These+controls do not establish a host journal's availability or #2785's stop cause.A failed collector does not establish installer failure; zero exits establishpipeline completion only, not version, service or update acceptance.The existing signed transaction and confined service-state testsdiff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.goindex 03d4bb4dba..e45363efcf 100644--- a/scripts/installtests/pulse_auto_update_test.go+++ b/scripts/installtests/pulse_auto_update_test.go@@ -143,6 +143,8 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {for _, name := range []string{"test_early_collector_failure_survives_later_success_and_drains_installer","test_unterminated_final_installer_line_is_collected_and_its_failure_retained",+ "test_real_log_retains_each_sink_failure_in_conditional_calls",+ "test_real_logger_failure_drains_installer_and_preserves_recovery",} {passed := falsefor _, line := range strings.Split(string(out), "\n") {diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.shindex 94dd6095b8..667ed35ed0 100755--- a/scripts/pulse-auto-update.sh+++ b/scripts/pulse-auto-update.sh@@ -22,8 +22,16 @@ PINNED_RELEASE_SSH_PUBLIC_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMZd/DaH+Bldzlog() {local level=$1shift- logger -t "$LOG_TAG" -p "user.$level" "$@"- echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $@"+ local logger_status=0 output_status=0+ # Conditional callers disable errexit throughout this function. Try both+ # sinks, but never let successful stdout hide a failed journal write (or+ # vice versa). The installer collector owns draining and the first error.+ logger -t "$LOG_TAG" -p "user.$level" "$@" || logger_status=$?+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $@" || output_status=$?+ if [[ "$logger_status" != 0 ]]; then+ return "$logger_status"+ fi+ return "$output_status"}release_signature_key_available() {diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.pyindex 5e7db24fca..46a0fa2d06 100644--- a/scripts/tests/test_auto_update_service_state.py+++ b/scripts/tests/test_auto_update_service_state.py@@ -66,6 +66,7 @@ sys.exit(97)HARNESS = r'''source "$UPDATER"+if [[ "$REAL_LOG" != true ]]; thenlog() {printf '[%s] %s\n' "$1" "${*:2}"if [[ "${2:-}" == 'installer: '* ]]; then@@ -77,6 +78,7 @@ log() {return "${!status_name:-$COLLECTOR_EXIT}"fi}+fidetect_service_name() { printf '%s\n' "$SERVICE_NAME"; }sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.verify_release_signature() { :; }@@ -114,15 +116,39 @@ exit "$status"'''+LOGGER = r'''#!/usr/bin/env python3+import json, os, sys+from pathlib import Path+f = Path(os.environ["FIXTURE"])+args = sys.argv[1:]+with (f / "logger-calls").open("a") as out:+ out.write(json.dumps(args) + "\n")+if (args[:3] != ["-t", "pulse-custom@department-auto-update", "-p"]+ or len(args) < 5 or args[3] not in ("user.info", "user.warn", "user.error")):+ sys.exit(97)+if args[4:5] and args[4].startswith("installer: "):+ count_path = f / "collector-count"+ count = int(count_path.read_text()) + 1 if count_path.exists() else 1+ count_path.write_text(str(count))+ sys.exit(int(os.environ.get("COLLECTOR_EXIT_" + str(count), os.environ["COLLECTOR_EXIT"])))+sys.exit(0)+'''++class UpdateServiceState(unittest.TestCase):def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",- installer_exit=None, collector_exit=0, installer_output=None, collector_failures=None):+ installer_exit=None, collector_exit=0, installer_output=None, collector_failures=None,+ real_log=False):with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:f = Path(directory)for name in ("tools", "tmp", "install/bin", "config"):(f / name).mkdir(parents=True)(f / "tools/systemctl").write_text(MANAGER)(f / "tools/systemctl").chmod(0o755)+ # The production log() remains intact in real-log controls. Only+ # its external sink is confined; no host journal is contacted.+ (f / "tools/logger").write_text(LOGGER)+ (f / "tools/logger").chmod(0o755)old = b"#!/bin/sh\necho 'Pulse v6.5.0'\n"new = b"#!/bin/sh\necho 'Pulse v6.6.0'\n"(f / "install/bin/pulse").write_bytes(old)@@ -149,6 +175,7 @@ class UpdateServiceState(unittest.TestCase):INSTALLER_STATE="active" if mode in ("stop-noop", "stop-error", "stop-hang", "after-error", "readback-error", "readback-empty") else "inactive",INSTALLER_EXIT=str(installer_exit) if installer_exit is not None else ("0" if success else "23"),COLLECTOR_EXIT=str(collector_exit),+ REAL_LOG="true" if real_log else "false",PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])for line, status in (collector_failures or {}).items():env[f"COLLECTOR_EXIT_{line}"] = str(status)@@ -162,6 +189,8 @@ class UpdateServiceState(unittest.TestCase):installed=(f / "installer-ran").exists(), stopped=(f / "stop-ran").exists(),installer_finished=(f / "installer-finished").exists(),collected=int((f / "collector-count").read_text()) if (f / "collector-count").exists() else 0,+ logger_calls=[json.loads(line) for line in (f / "logger-calls").read_text().splitlines()]+ if (f / "logger-calls").exists() else [],started=(f / "start-ran").exists(), state=(f / "state").read_text().strip(),binary=(f / "install/bin/pulse").read_bytes(), version=(f / "install/VERSION").read_text(),backups=[(b / "pulse-bin").read_bytes() for b in backups],@@ -174,6 +203,61 @@ class UpdateServiceState(unittest.TestCase):self.assertIn("pulse-custom@department", args)return observation+ def test_real_log_retains_each_sink_failure_in_conditional_calls(self):+ for logger_exit, closed_stdout, expected in ((0, False, 0), (73, False, 73),+ (0, True, 1), (73, True, 73)):+ with self.subTest(logger_exit=logger_exit, closed_stdout=closed_stdout):+ with tempfile.TemporaryDirectory(prefix="pulse-updater-log-") as directory:+ f = Path(directory)+ (f / "tools").mkdir()+ # Only the external logger is substituted, not log().+ (f / "tools/logger").write_text(+ '#!/bin/sh\n'+ 'printf "%s\\n" "$*" > "$FIXTURE/logger-args"\n'+ 'exit "$LOGGER_EXIT"\n')+ (f / "tools/logger").chmod(0o755)+ env = dict(os.environ, FIXTURE=str(f), UPDATER=str(UPDATER),+ PULSE_SERVICE_NAME="pulse-custom@department", LOGGER_EXIT=str(logger_exit),+ PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])+ redirect = " 1>&-" if closed_stdout else ""+ script = ('set -euo pipefail\nsource "$UPDATER"\nstatus=0\n'+ 'log info "sink control"' + redirect + ' || status=$?\n'+ 'printf "OBSERVED_STATUS=%s\\n" "$status"\n')+ r = subprocess.run(["bash", "-c", script], env=env, capture_output=True, timeout=5)+ self.assertEqual(r.returncode, 0)+ self.assertIn(f"OBSERVED_STATUS={expected}\n", r.stdout.decode())+ self.assertEqual((f / "logger-args").read_text(),+ "-t pulse-custom@department-auto-update -p user.info sink control\n")+ self.assertEqual("[info] sink control" in r.stdout.decode(), not closed_stdout)++ def test_real_logger_failure_drains_installer_and_preserves_recovery(self):+ # More than a pipe buffer, plus an unterminated final line. These run+ # production log(), perform_update and its collector together: a green+ # log() double cannot establish that a real sink failure is retained.+ lines = ["opening"] + ["x" * 2048 for _ in range(64)] + ["final installer evidence"]+ for state in ("active", "inactive"):+ for failures in ({}, {1: 73}, {1: 73, 2: 42}, {len(lines): 73}):+ with self.subTest(state=state, failures=failures):+ r = self.run_fixture(state=state, success=True, real_log=True,+ installer_output="\n".join(lines), collector_failures=failures)+ self.assertTrue(r["installer_finished"])+ self.assertEqual(r["collected"], len(lines))+ self.assertTrue("installer: final installer evidence" in r["output"])+ installer_calls = [args for args in r["logger_calls"] if args[4].startswith("installer: ")]+ self.assertEqual(len(installer_calls), len(lines))+ self.assertTrue(all(args[:4] == ["-t", "pulse-custom@department-auto-update", "-p", "user.info"]+ for args in installer_calls))+ expected = ("Installer pipeline failed (installer exit: 0; log collector exit: 73)"+ if failures else "Installer completed; verifying update (installer exit: 0; log collector exit: 0)")+ self.assertTrue(expected in r["output"], "real logger exit was not retained by the collector")+ self.assertEqual(r["exit"], 1 if failures else 0)+ self.assertEqual("Update successfully installed and verified" in r["output"], not failures)+ self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n" if failures+ else b"#!/bin/sh\necho 'Pulse v6.6.0'\n")+ self.assertEqual(r["version"], "v6.5.0\n" if failures else "v6.6.0\n")+ self.assertEqual(r["started"], state == "active")+ self.assertEqual(r["backups"], [])+def test_unknown_initial_state_refuses_before_backup_download_or_installer(self):cases = [(m, "active", "loaded") for m in("initial-error", "initial-empty", "initial-duplicate", "initial-extra")]