Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve native updater failures across cleanup
pulse-triage[bot]e1cd6843e9d9
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 2826ba8d24..c59dedcfed 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -369,6 +369,23 @@ cleanup removes only objects created by this attempt. Compiler and root-commandfailures stay gating under the existing job's deadline; a seven-day closed receiptcontains source/binary/output hashes, all phases and cleanup, never signer keys,persistent contents or raw application/installer logs.++Failed commands and assertions name their fixed preparation or transaction+operation. The original transaction's `command_failure` is preserved before+cleanup runs, and a cleanup command failure is retained separately under+`cleanup.command_failure`. A successful cleanup must not label the earlier+transaction failure as its own; an assertion or cleanup-only failure must not+invent an executed transaction command. Timeout facts remain distinct from a+later numeric cleanup exit, with no stale output hash attached. Neither these+labels nor completed owned cleanup diagnose why an update failed or accept a+native phase. `test_primary_command_survives_successful_or_failed_cleanup`,+`test_primary_timeout_is_not_replaced_by_cleanup_exit`,+`test_assertion_and_cleanup_only_errors_do_not_invent_primary_command` and+`test_successful_receipt_retains_no_failure_record` drive the actual receipt+producer with real confined failing/timeout children and no native manager;+the registered Go harness requires every portable verdict. Raw command output,+private fixture contents, argv and paths remain outside these observations.+The private fixture disables telemetry and binds the existing service to loopback;it exposes no test server on the hosted VM's external interface.Before compiling the native pair, CI requires the exact checkout head and nodiff --git a/scripts/installtests/auto_update_systemd.py b/scripts/installtests/auto_update_systemd.pyindex 51e853eb1c..424d440d10 100644--- a/scripts/installtests/auto_update_systemd.py+++ b/scripts/installtests/auto_update_systemd.py@@ -146,6 +146,7 @@ class NativeLab:self.owned_unit = Falseself.owned_root = Falseself.owned_user = False+ self.operation = "prepare-admission"def run(self, *args, timeout=120, ok=(0,)):process = subprocess.Popen(args, env=self.env, stdin=subprocess.DEVNULL,@@ -162,7 +163,8 @@ class NativeLab:process.communicate(timeout=2)self.receipt["command_failure"] = {"command": Path(args[0]).name,"deadline_exceeded": True,- "observed_exit": process.returncode}+ "observed_exit": process.returncode,+ "operation": self.operation}raise RuntimeError("native command deadline exceeded") from None# Private in-memory diagnostic for confined controls, never a receipt field.self._last_output = output@@ -170,6 +172,7 @@ class NativeLab:if result.returncode not in ok:self.receipt["command_failure"] = {"command": Path(args[0]).name,"observed_exit": result.returncode,+ "operation": self.operation,"output_bytes": len(output),"output_sha256": hashlib.sha256(output).hexdigest()}raise RuntimeError(f"native command failed ({result.returncode}): {Path(args[0]).name}")@@ -193,6 +196,7 @@ class NativeLab:return pairsdef prepare(self):+ self.operation = "prepare-admission"for path in (self.unit, Path("/run/systemd/system") / self.unit.name):if path.exists() or path.is_symlink():raise ValueError("native fixture unit already exists")@@ -205,6 +209,7 @@ class NativeLab:raise ValueError("native fixture unit is already known to systemd")for name in ("tools", "tmp", "archives", "signer"):(self.root / name).mkdir(mode=0o700 if name == "signer" else 0o755)+ self.operation = "prepare-source"source = (self.repo / "install.sh").read_text()(self.root / "installer-definitions").write_text(installer_definitions(source))self.receipt["sources"] = {p: digest(self.repo / p) for p in@@ -214,10 +219,12 @@ class NativeLab:if len(set(self.receipt["binaries"].values())) != 2:raise ValueError("same executable bytes prove no replacement")for tag, binary in self.binaries.items():+ self.operation = "verify-binary-" + tagif self.run(str(binary), "--version").stdout.decode().splitlines()[0] != "Pulse " + tag:raise ValueError("synthetic binary identity mismatch")self.create_signer()for tag, binary in self.binaries.items():+ self.operation = "prepare-archive-" + tagversion = self.root / "VERSION"version.write_text(tag + "\n")archive = self.root / "archives" / f"pulse-{tag}-linux-amd64.tar.gz"@@ -225,27 +232,35 @@ class NativeLab:tar.add(binary, arcname="bin/pulse")tar.add(version, arcname="VERSION")self.sign(archive)+ self.operation = "prepare-installer"(self.root / "installer").write_text(INSTALLER)self.sign(self.root / "installer")+ self.operation = "prepare-transport"for name in ("curl", "mktemp"):path = self.root / "tools" / namepath.write_text(TRANSPORT)path.chmod(0o755)# Establish the baseline using real source staging and unit/helper writers.+ self.operation = "baseline-account-admission"self.owned_user = self.run("id", "-u", "pulse", ok=(0, 1)).returncode == 1self.owned_unit = True # Any partial writer output is ours to clean up.+ self.operation = "baseline-install"self.shell('source "$PULSE_NATIVE_ROOT/installer-definitions"; ''PINNED_RELEASE_SSH_PUBLIC_KEY=$(cat "$PULSE_NATIVE_ROOT/signer/key.pub"); ''create_user; setup_directories; ''install_pulse_archive "$PULSE_NATIVE_ROOT/archives/pulse-v0.0.1-linux-amd64.tar.gz" v0.0.1; ''install_systemd_service; setup_update_command')+ self.operation = "baseline-configure"data = self.root / "data"(data / ".env").write_text("PULSE_TELEMETRY=false\nBIND_ADDRESS=127.0.0.1\n")(data / "system.json").write_text('{"autoUpdateEnabled":false,"updateChannel":"stable"}\n')(data / "native-persistence-sentinel").write_text("original persistent data\n")+ self.operation = "baseline-start"self.run("systemctl", "start", self.unit.name)+ self.operation = "baseline-readback"self.assert_runtime(TAGS[0], "active")time.sleep(2) # Let asynchronous initial persistence settle once.+ self.operation = "baseline-persistence"files = (".env", "system.json", ".encryption.key", "native-persistence-sentinel")self.baseline_data = {p: digest(data / p) for p in files}self.baseline_mode = (self.root / "install/bin/pulse").stat().st_mode & 0o777@@ -274,6 +289,7 @@ class NativeLab:why a hosted OpenSSH invocation failed before the first native phase.Diagnostic labels below mean observed text, not a causal diagnosis."""+ self.operation = operationtry:return self.run("ssh-keygen", *arguments)except RuntimeError:@@ -373,6 +389,7 @@ class NativeLab:return statedef phase(self, name, tag, expected_exit=0, active="active", fault="", retain=False, manual=False):+ self.operation = namebefore_state = self.state()self.env.update(PULSE_NATIVE_TARGET=tag, PULSE_NATIVE_FAULT=fault)result = self.run("/bin/bash", "-c", '"$PULSE_UPDATE_HELPER_PATH" --version "$PULSE_NATIVE_TARGET"'@@ -404,18 +421,22 @@ class NativeLab:self.phase("forward", TAGS[1])self.phase("manual-reverse", TAGS[0], manual=True)self.phase("failed-active", TAGS[1], 1, fault="installer")+ self.operation = "prepare-inactive"self.run("systemctl", "stop", self.unit.name)self.phase("failed-inactive", TAGS[1], 1, active="inactive", fault="installer")# A real /dev/null mask, not a manager double returning 'masked'.+ self.operation = "prepare-masked"saved_unit = self.unit.read_bytes()self.unit.unlink()self.run("systemctl", "mask", self.unit.name)self.run("systemctl", "daemon-reload")self.phase("failed-masked", TAGS[1], 1, active="inactive", fault="installer")+ self.operation = "restore-unmasked"self.run("systemctl", "unmask", self.unit.name)self.unit.write_bytes(saved_unit)self.run("systemctl", "daemon-reload")self.run("systemctl", "start", self.unit.name)+ self.operation = "prepare-bad-signature"installer = self.root / "installer"original = installer.read_bytes()calls = (self.root / "installer-calls").read_bytes()@@ -430,6 +451,7 @@ class NativeLab:self.phase("incomplete-restore", TAGS[1], 1, active="inactive", fault="restore", retain=True)def cleanup(self):+ self.operation = "cleanup"if self.owned_unit:state = self.state()if state["LoadState"] != "not-found":@@ -479,13 +501,23 @@ def main():lab.exercise()except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error:failed = True- lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error)}+ lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error),+ "operation": lab.operation}finally:+ # Cleanup runs its own commands. Preserve the original transaction+ # observation before any later failure can replace it or be mistaken+ # for that transaction. A logical assertion has no command observation.+ transaction_failure = lab.receipt.pop("command_failure", None)try:lab.cleanup()except (OSError, RuntimeError, subprocess.SubprocessError) as error:failed = Truelab.receipt["cleanup"] = {"complete": False, "error_type": type(error).__name__}+ cleanup_failure = lab.receipt.pop("command_failure", None)+ if cleanup_failure is not None:+ lab.receipt["cleanup"]["command_failure"] = cleanup_failure+ if transaction_failure is not None:+ lab.receipt["command_failure"] = transaction_failurereceipt.write_text(json.dumps(lab.receipt, indent=2) + "\n")if failed or len(lab.receipt["phases"]) != 7:raise SystemExit("Native updater proof incomplete; inspect the bounded receipt.")diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.goindex c9bc2ff6e3..edf05eb6b9 100644--- a/scripts/installtests/pulse_auto_update_test.go+++ b/scripts/installtests/pulse_auto_update_test.go@@ -37,6 +37,10 @@ func TestAutoUpdateNativeSystemdHarnessContracts(t *testing.T) {"test_phase_cannot_accept_wrong_bytes_state_mask_persistence_or_backup","test_managed_process_identity_follows_served_readiness_and_rejects_residuals","test_runtime_readback_rejects_post_comparison_drift",+ "test_primary_command_survives_successful_or_failed_cleanup",+ "test_primary_timeout_is_not_replaced_by_cleanup_exit",+ "test_assertion_and_cleanup_only_errors_do_not_invent_primary_command",+ "test_successful_receipt_retains_no_failure_record","test_connected_linux_control_is_required_without_macho_or_validator_bypass",} {if results[name] != "ok" {diff --git a/scripts/tests/test_auto_update_systemd.py b/scripts/tests/test_auto_update_systemd.pyindex 7dc8754030..b934388e40 100644--- a/scripts/tests/test_auto_update_systemd.py+++ b/scripts/tests/test_auto_update_systemd.py@@ -1,5 +1,6 @@#!/usr/bin/env python3"""Discriminating controls for the native CI recipe; no host manager/root use."""+import contextlibimport importlib.utilimport ioimport json@@ -299,6 +300,7 @@ class NativeContracts(unittest.TestCase):else:lab.phase("forward", "v0.0.2", active="inactive")self.assertTrue(lab.receipt["phases"][0]["passed"])+ self.assertEqual(lab.operation, "forward")def test_managed_process_identity_follows_served_readiness_and_rejects_residuals(self):for mutation in ("", "process-bytes", "stopped-after-readiness", "inactive-pid"):@@ -665,6 +667,93 @@ else: sys.exit(97)source.replace("runs-on: ubuntu-24.04", "runs-on: self-hosted")):with self.assertRaises(AssertionError): recipe(changed)+ def run_receipt(self, primary, cleanup_fails):+ # Execute the actual receipt producer, with real unprivileged failing+ # children. Substitute only native admission and service/account work.+ # These controls cannot provide a native transaction or cleanup proof.+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ receipt = root / "receipt.json"+ lab = native.NativeLab(root, ROOT, root / "a", root / "b", "a" * 40)+ owned_cleanup = lab.cleanup+ def prepare():+ lab.operation = "baseline-install"+ if primary == "command":+ lab.run("/bin/bash", "-c", "printf 'private-transaction-fixture'; exit 23")+ if primary == "timeout":+ lab.run(sys.executable, "-c", "import time; time.sleep(60)", timeout=0.05)+ if primary == "assertion":+ raise RuntimeError("confined persistence assertion")+ def exercise():+ lab.receipt["phases"] = [{"name": "confined-control"}] * 7+ def cleanup():+ lab.operation = "cleanup"+ if cleanup_fails:+ lab.run(sys.executable, "-c", "print('private-cleanup-fixture'); raise SystemExit(31)")+ owned_cleanup() # Owns nothing; touches no manager.+ argv = ["native-receipt-control", "--before", "unused-before", "--after", "unused-after",+ "--receipt", str(receipt)]+ with patch.dict(os.environ, {"EXPECTED_SOURCE_SHA": "a" * 40, "RUNNER_TEMP": str(root)}), \+ patch.object(sys, "argv", argv), patch.object(native, "hosted_admission"), \+ patch.object(native, "NativeLab", return_value=lab), \+ patch.object(lab, "prepare", side_effect=prepare), \+ patch.object(lab, "exercise", side_effect=exercise), \+ patch.object(lab, "cleanup", side_effect=cleanup), \+ contextlib.redirect_stdout(io.StringIO()):+ if primary or cleanup_fails:+ with self.assertRaises(SystemExit): native.main()+ else:+ native.main()+ result = json.loads(receipt.read_text())+ for private in ("private-transaction-fixture", "private-cleanup-fixture", str(root)):+ self.assertNotIn(private, json.dumps(result))+ return result++ def test_primary_command_survives_successful_or_failed_cleanup(self):+ for cleanup_fails in (False, True):+ with self.subTest(cleanup_fails=cleanup_fails):+ result = self.run_receipt("command", cleanup_fails)+ self.assertEqual(result["command_failure"]["observed_exit"], 23)+ self.assertEqual(result["command_failure"]["command"], "bash")+ self.assertEqual(result["command_failure"]["operation"], "baseline-install")+ self.assertEqual(result["failure"]["operation"], "baseline-install")+ self.assertEqual(result["phases"], [])+ self.assertEqual(result["cleanup"]["complete"], not cleanup_fails)+ if cleanup_fails:+ self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)+ self.assertEqual(result["cleanup"]["command_failure"]["operation"], "cleanup")+ else:+ self.assertNotIn("command_failure", result["cleanup"])++ def test_primary_timeout_is_not_replaced_by_cleanup_exit(self):+ result = self.run_receipt("timeout", True)+ self.assertTrue(result["command_failure"]["deadline_exceeded"])+ self.assertLess(result["command_failure"]["observed_exit"], 0)+ self.assertEqual(result["command_failure"]["operation"], "baseline-install")+ self.assertNotIn("output_sha256", result["command_failure"])+ self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)+ self.assertEqual(result["phases"], [])++ def test_assertion_and_cleanup_only_errors_do_not_invent_primary_command(self):+ for primary in ("assertion", ""):+ with self.subTest(primary=primary):+ result = self.run_receipt(primary, True)+ self.assertNotIn("command_failure", result)+ self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)+ self.assertFalse(result["cleanup"]["complete"])+ if primary:+ self.assertEqual(result["failure"]["operation"], "baseline-install")+ self.assertEqual(result["phases"], [])+ else:+ self.assertNotIn("failure", result)++ def test_successful_receipt_retains_no_failure_record(self):+ result = self.run_receipt("", False)+ self.assertNotIn("failure", result)+ self.assertNotIn("command_failure", result)+ self.assertNotIn("command_failure", result["cleanup"])+ self.assertTrue(result["cleanup"]["complete"])+if __name__ == "__main__":unittest.main()