Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve native updater failures across cleanup

pulse-triage[bot]e1cd6843e9d9
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 2826ba8d24..c59dedcfed 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -369,6 +369,23 @@ cleanup removes only objects created by this attempt. Compiler and root-command
failures stay gating under the existing job's deadline; a seven-day closed receipt
contains source/binary/output hashes, all phases and cleanup, never signer keys,
persistent contents or raw application/installer logs.
+
+Failed commands and assertions name their fixed preparation or transaction
+operation. The original transaction's `command_failure` is preserved before
+cleanup runs, and a cleanup command failure is retained separately under
+`cleanup.command_failure`. A successful cleanup must not label the earlier
+transaction failure as its own; an assertion or cleanup-only failure must not
+invent an executed transaction command. Timeout facts remain distinct from a
+later numeric cleanup exit, with no stale output hash attached. Neither these
+labels nor completed owned cleanup diagnose why an update failed or accept a
+native phase. `test_primary_command_survives_successful_or_failed_cleanup`,
+`test_primary_timeout_is_not_replaced_by_cleanup_exit`,
+`test_assertion_and_cleanup_only_errors_do_not_invent_primary_command` and
+`test_successful_receipt_retains_no_failure_record` drive the actual receipt
+producer with real confined failing/timeout children and no native manager;
+the registered Go harness requires every portable verdict. Raw command output,
+private fixture contents, argv and paths remain outside these observations.
+
The private fixture disables telemetry and binds the existing service to loopback;
it exposes no test server on the hosted VM's external interface.
Before compiling the native pair, CI requires the exact checkout head and no
diff --git a/scripts/installtests/auto_update_systemd.py b/scripts/installtests/auto_update_systemd.py
index 51e853eb1c..424d440d10 100644
--- a/scripts/installtests/auto_update_systemd.py
+++ b/scripts/installtests/auto_update_systemd.py
@@ -146,6 +146,7 @@ class NativeLab:
self.owned_unit = False
self.owned_root = False
self.owned_user = False
+ self.operation = "prepare-admission"
def run(self, *args, timeout=120, ok=(0,)):
process = subprocess.Popen(args, env=self.env, stdin=subprocess.DEVNULL,
@@ -162,7 +163,8 @@ class NativeLab:
process.communicate(timeout=2)
self.receipt["command_failure"] = {"command": Path(args[0]).name,
"deadline_exceeded": True,
- "observed_exit": process.returncode}
+ "observed_exit": process.returncode,
+ "operation": self.operation}
raise RuntimeError("native command deadline exceeded") from None
# Private in-memory diagnostic for confined controls, never a receipt field.
self._last_output = output
@@ -170,6 +172,7 @@ class NativeLab:
if result.returncode not in ok:
self.receipt["command_failure"] = {"command": Path(args[0]).name,
"observed_exit": result.returncode,
+ "operation": self.operation,
"output_bytes": len(output),
"output_sha256": hashlib.sha256(output).hexdigest()}
raise RuntimeError(f"native command failed ({result.returncode}): {Path(args[0]).name}")
@@ -193,6 +196,7 @@ class NativeLab:
return pairs
def prepare(self):
+ self.operation = "prepare-admission"
for path in (self.unit, Path("/run/systemd/system") / self.unit.name):
if path.exists() or path.is_symlink():
raise ValueError("native fixture unit already exists")
@@ -205,6 +209,7 @@ class NativeLab:
raise ValueError("native fixture unit is already known to systemd")
for name in ("tools", "tmp", "archives", "signer"):
(self.root / name).mkdir(mode=0o700 if name == "signer" else 0o755)
+ self.operation = "prepare-source"
source = (self.repo / "install.sh").read_text()
(self.root / "installer-definitions").write_text(installer_definitions(source))
self.receipt["sources"] = {p: digest(self.repo / p) for p in
@@ -214,10 +219,12 @@ class NativeLab:
if len(set(self.receipt["binaries"].values())) != 2:
raise ValueError("same executable bytes prove no replacement")
for tag, binary in self.binaries.items():
+ self.operation = "verify-binary-" + tag
if self.run(str(binary), "--version").stdout.decode().splitlines()[0] != "Pulse " + tag:
raise ValueError("synthetic binary identity mismatch")
self.create_signer()
for tag, binary in self.binaries.items():
+ self.operation = "prepare-archive-" + tag
version = self.root / "VERSION"
version.write_text(tag + "\n")
archive = self.root / "archives" / f"pulse-{tag}-linux-amd64.tar.gz"
@@ -225,27 +232,35 @@ class NativeLab:
tar.add(binary, arcname="bin/pulse")
tar.add(version, arcname="VERSION")
self.sign(archive)
+ self.operation = "prepare-installer"
(self.root / "installer").write_text(INSTALLER)
self.sign(self.root / "installer")
+ self.operation = "prepare-transport"
for name in ("curl", "mktemp"):
path = self.root / "tools" / name
path.write_text(TRANSPORT)
path.chmod(0o755)
# Establish the baseline using real source staging and unit/helper writers.
+ self.operation = "baseline-account-admission"
self.owned_user = self.run("id", "-u", "pulse", ok=(0, 1)).returncode == 1
self.owned_unit = True # Any partial writer output is ours to clean up.
+ self.operation = "baseline-install"
self.shell('source "$PULSE_NATIVE_ROOT/installer-definitions"; '
'PINNED_RELEASE_SSH_PUBLIC_KEY=$(cat "$PULSE_NATIVE_ROOT/signer/key.pub"); '
'create_user; setup_directories; '
'install_pulse_archive "$PULSE_NATIVE_ROOT/archives/pulse-v0.0.1-linux-amd64.tar.gz" v0.0.1; '
'install_systemd_service; setup_update_command')
+ self.operation = "baseline-configure"
data = self.root / "data"
(data / ".env").write_text("PULSE_TELEMETRY=false\nBIND_ADDRESS=127.0.0.1\n")
(data / "system.json").write_text('{"autoUpdateEnabled":false,"updateChannel":"stable"}\n')
(data / "native-persistence-sentinel").write_text("original persistent data\n")
+ self.operation = "baseline-start"
self.run("systemctl", "start", self.unit.name)
+ self.operation = "baseline-readback"
self.assert_runtime(TAGS[0], "active")
time.sleep(2) # Let asynchronous initial persistence settle once.
+ self.operation = "baseline-persistence"
files = (".env", "system.json", ".encryption.key", "native-persistence-sentinel")
self.baseline_data = {p: digest(data / p) for p in files}
self.baseline_mode = (self.root / "install/bin/pulse").stat().st_mode & 0o777
@@ -274,6 +289,7 @@ class NativeLab:
why a hosted OpenSSH invocation failed before the first native phase.
Diagnostic labels below mean observed text, not a causal diagnosis.
"""
+ self.operation = operation
try:
return self.run("ssh-keygen", *arguments)
except RuntimeError:
@@ -373,6 +389,7 @@ class NativeLab:
return state
def phase(self, name, tag, expected_exit=0, active="active", fault="", retain=False, manual=False):
+ self.operation = name
before_state = self.state()
self.env.update(PULSE_NATIVE_TARGET=tag, PULSE_NATIVE_FAULT=fault)
result = self.run("/bin/bash", "-c", '"$PULSE_UPDATE_HELPER_PATH" --version "$PULSE_NATIVE_TARGET"'
@@ -404,18 +421,22 @@ class NativeLab:
self.phase("forward", TAGS[1])
self.phase("manual-reverse", TAGS[0], manual=True)
self.phase("failed-active", TAGS[1], 1, fault="installer")
+ self.operation = "prepare-inactive"
self.run("systemctl", "stop", self.unit.name)
self.phase("failed-inactive", TAGS[1], 1, active="inactive", fault="installer")
# A real /dev/null mask, not a manager double returning 'masked'.
+ self.operation = "prepare-masked"
saved_unit = self.unit.read_bytes()
self.unit.unlink()
self.run("systemctl", "mask", self.unit.name)
self.run("systemctl", "daemon-reload")
self.phase("failed-masked", TAGS[1], 1, active="inactive", fault="installer")
+ self.operation = "restore-unmasked"
self.run("systemctl", "unmask", self.unit.name)
self.unit.write_bytes(saved_unit)
self.run("systemctl", "daemon-reload")
self.run("systemctl", "start", self.unit.name)
+ self.operation = "prepare-bad-signature"
installer = self.root / "installer"
original = installer.read_bytes()
calls = (self.root / "installer-calls").read_bytes()
@@ -430,6 +451,7 @@ class NativeLab:
self.phase("incomplete-restore", TAGS[1], 1, active="inactive", fault="restore", retain=True)
def cleanup(self):
+ self.operation = "cleanup"
if self.owned_unit:
state = self.state()
if state["LoadState"] != "not-found":
@@ -479,13 +501,23 @@ def main():
lab.exercise()
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error:
failed = True
- lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error)}
+ lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error),
+ "operation": lab.operation}
finally:
+ # Cleanup runs its own commands. Preserve the original transaction
+ # observation before any later failure can replace it or be mistaken
+ # for that transaction. A logical assertion has no command observation.
+ transaction_failure = lab.receipt.pop("command_failure", None)
try:
lab.cleanup()
except (OSError, RuntimeError, subprocess.SubprocessError) as error:
failed = True
lab.receipt["cleanup"] = {"complete": False, "error_type": type(error).__name__}
+ cleanup_failure = lab.receipt.pop("command_failure", None)
+ if cleanup_failure is not None:
+ lab.receipt["cleanup"]["command_failure"] = cleanup_failure
+ if transaction_failure is not None:
+ lab.receipt["command_failure"] = transaction_failure
receipt.write_text(json.dumps(lab.receipt, indent=2) + "\n")
if failed or len(lab.receipt["phases"]) != 7:
raise SystemExit("Native updater proof incomplete; inspect the bounded receipt.")
diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go
index c9bc2ff6e3..edf05eb6b9 100644
--- a/scripts/installtests/pulse_auto_update_test.go
+++ b/scripts/installtests/pulse_auto_update_test.go
@@ -37,6 +37,10 @@ func TestAutoUpdateNativeSystemdHarnessContracts(t *testing.T) {
"test_phase_cannot_accept_wrong_bytes_state_mask_persistence_or_backup",
"test_managed_process_identity_follows_served_readiness_and_rejects_residuals",
"test_runtime_readback_rejects_post_comparison_drift",
+ "test_primary_command_survives_successful_or_failed_cleanup",
+ "test_primary_timeout_is_not_replaced_by_cleanup_exit",
+ "test_assertion_and_cleanup_only_errors_do_not_invent_primary_command",
+ "test_successful_receipt_retains_no_failure_record",
"test_connected_linux_control_is_required_without_macho_or_validator_bypass",
} {
if results[name] != "ok" {
diff --git a/scripts/tests/test_auto_update_systemd.py b/scripts/tests/test_auto_update_systemd.py
index 7dc8754030..b934388e40 100644
--- a/scripts/tests/test_auto_update_systemd.py
+++ b/scripts/tests/test_auto_update_systemd.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Discriminating controls for the native CI recipe; no host manager/root use."""
+import contextlib
import importlib.util
import io
import json
@@ -299,6 +300,7 @@ class NativeContracts(unittest.TestCase):
else:
lab.phase("forward", "v0.0.2", active="inactive")
self.assertTrue(lab.receipt["phases"][0]["passed"])
+ self.assertEqual(lab.operation, "forward")
def test_managed_process_identity_follows_served_readiness_and_rejects_residuals(self):
for mutation in ("", "process-bytes", "stopped-after-readiness", "inactive-pid"):
@@ -665,6 +667,93 @@ else: sys.exit(97)
source.replace("runs-on: ubuntu-24.04", "runs-on: self-hosted")):
with self.assertRaises(AssertionError): recipe(changed)
+ def run_receipt(self, primary, cleanup_fails):
+ # Execute the actual receipt producer, with real unprivileged failing
+ # children. Substitute only native admission and service/account work.
+ # These controls cannot provide a native transaction or cleanup proof.
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ receipt = root / "receipt.json"
+ lab = native.NativeLab(root, ROOT, root / "a", root / "b", "a" * 40)
+ owned_cleanup = lab.cleanup
+ def prepare():
+ lab.operation = "baseline-install"
+ if primary == "command":
+ lab.run("/bin/bash", "-c", "printf 'private-transaction-fixture'; exit 23")
+ if primary == "timeout":
+ lab.run(sys.executable, "-c", "import time; time.sleep(60)", timeout=0.05)
+ if primary == "assertion":
+ raise RuntimeError("confined persistence assertion")
+ def exercise():
+ lab.receipt["phases"] = [{"name": "confined-control"}] * 7
+ def cleanup():
+ lab.operation = "cleanup"
+ if cleanup_fails:
+ lab.run(sys.executable, "-c", "print('private-cleanup-fixture'); raise SystemExit(31)")
+ owned_cleanup() # Owns nothing; touches no manager.
+ argv = ["native-receipt-control", "--before", "unused-before", "--after", "unused-after",
+ "--receipt", str(receipt)]
+ with patch.dict(os.environ, {"EXPECTED_SOURCE_SHA": "a" * 40, "RUNNER_TEMP": str(root)}), \
+ patch.object(sys, "argv", argv), patch.object(native, "hosted_admission"), \
+ patch.object(native, "NativeLab", return_value=lab), \
+ patch.object(lab, "prepare", side_effect=prepare), \
+ patch.object(lab, "exercise", side_effect=exercise), \
+ patch.object(lab, "cleanup", side_effect=cleanup), \
+ contextlib.redirect_stdout(io.StringIO()):
+ if primary or cleanup_fails:
+ with self.assertRaises(SystemExit): native.main()
+ else:
+ native.main()
+ result = json.loads(receipt.read_text())
+ for private in ("private-transaction-fixture", "private-cleanup-fixture", str(root)):
+ self.assertNotIn(private, json.dumps(result))
+ return result
+
+ def test_primary_command_survives_successful_or_failed_cleanup(self):
+ for cleanup_fails in (False, True):
+ with self.subTest(cleanup_fails=cleanup_fails):
+ result = self.run_receipt("command", cleanup_fails)
+ self.assertEqual(result["command_failure"]["observed_exit"], 23)
+ self.assertEqual(result["command_failure"]["command"], "bash")
+ self.assertEqual(result["command_failure"]["operation"], "baseline-install")
+ self.assertEqual(result["failure"]["operation"], "baseline-install")
+ self.assertEqual(result["phases"], [])
+ self.assertEqual(result["cleanup"]["complete"], not cleanup_fails)
+ if cleanup_fails:
+ self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)
+ self.assertEqual(result["cleanup"]["command_failure"]["operation"], "cleanup")
+ else:
+ self.assertNotIn("command_failure", result["cleanup"])
+
+ def test_primary_timeout_is_not_replaced_by_cleanup_exit(self):
+ result = self.run_receipt("timeout", True)
+ self.assertTrue(result["command_failure"]["deadline_exceeded"])
+ self.assertLess(result["command_failure"]["observed_exit"], 0)
+ self.assertEqual(result["command_failure"]["operation"], "baseline-install")
+ self.assertNotIn("output_sha256", result["command_failure"])
+ self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)
+ self.assertEqual(result["phases"], [])
+
+ def test_assertion_and_cleanup_only_errors_do_not_invent_primary_command(self):
+ for primary in ("assertion", ""):
+ with self.subTest(primary=primary):
+ result = self.run_receipt(primary, True)
+ self.assertNotIn("command_failure", result)
+ self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)
+ self.assertFalse(result["cleanup"]["complete"])
+ if primary:
+ self.assertEqual(result["failure"]["operation"], "baseline-install")
+ self.assertEqual(result["phases"], [])
+ else:
+ self.assertNotIn("failure", result)
+
+ def test_successful_receipt_retains_no_failure_record(self):
+ result = self.run_receipt("", False)
+ self.assertNotIn("failure", result)
+ self.assertNotIn("command_failure", result)
+ self.assertNotIn("command_failure", result["cleanup"])
+ self.assertTrue(result["cleanup"]["complete"])
+
if __name__ == "__main__":
unittest.main()