Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stage source-built server binaries before atomic replacement
pulse-triage[bot]1e6629c03c71
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex cd2f359277..7fb6b92e13 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -35,7 +35,7 @@ with the existing five-second command bound and read back; only `inactive`admits replacement. A failed unit does not imply prior workload liveness.Failed/timed-out reads or stops, transitional states and malformed/empty results-refuse replacement without an automatic restart or retry. The archive path+refuse replacement without an automatic restart or retry. Both replacement pathsatomically renames its fully prepared executable over the live path, rather thanmoving away the old executable or retrying extraction after deleting it. Afailed rename leaves the previous binary and metadata intact; only a service@@ -44,12 +44,25 @@ readback. An inactive/failed service is not started by this failure recovery.Recovery cannot turn the failed update into success. Successful replacementpreserves prior-active attribution for the existing post-update liveness check.+The opt-in source-build path also stages the compiled executable, its mode and+owner on the destination filesystem before stopping Pulse. It requires a+successful five-second-bounded, nonblank `--version` result and a nonempty+source revision before admission; development versions need not match a release+tag. Build or staging faults preserve the running service and installed bytes,+modes, VERSION, source-build marker and configuration. Its single atomic rename+uses the same prior-active-only failure recovery. It must not move aside the+live executable, copy over it or remove a pre-existing `pulse.old` recovery file.+Source-build temporary directories are removed on admission, stop and rename+failures. This does not add signed-release trust to an opt-in source build.+`scripts/tests/test_server_installer_stop.py` now executes the actual archiveinstaller with real tar/copy/rename bytes and fixture-only signing/architecture,service and ownership boundaries. Download/local admission failures, partialstaging copies, version exits/timeouts, uncertain stops, atomic rename faults,prior-active-only recovery/readback and retained bytes/modes/configuration are-checked, along with source-build and all three manual staging-failure flows.+checked, along with source-build admission/atomic-rename/recovery faults and all+three manual staging-failure flows. The source ordering and no-live-copy+obligations are also checked in `TestRootInstallScriptStagesUpdateBeforeStoppingService`.The existing root-installer ordering and update-resilience tests remainindependent controls. These ordinary-user fixtures do not establish signaturecryptography, native systemd/installed upgrade acceptance, ancillary-agentdiff --git a/install.sh b/install.shindex 97672c02a5..1e3b05be86 100755--- a/install.sh+++ b/install.sh@@ -3732,6 +3732,9 @@ build_from_source() {local arch=""local go_arch=""local service_name=""+ local binary_stage=""+ local version_output=""+ local source_revision=""print_info "Building Pulse from source (branch: $branch)..."@@ -3848,27 +3851,52 @@ build_from_source() {return 1fi- service_name=$(detect_service_name)- if ! stop_pulse_for_replacement "$service_name"; then+ # A successful build command alone does not admit its executable. Prepare+ # it on the destination filesystem before stopping the running service,+ # just as for signed archives. Source versions may be development builds.+ if ! mkdir -p "$INSTALL_DIR/bin" || ! binary_stage=$(mktemp -d "$INSTALL_DIR/bin/.pulse-stage-XXXXXX"); then+ print_error "Could not stage the built Pulse binary; Pulse has not been stopped"cd "$original_dir" >/dev/null 2>&1 || truerm -rf "$temp_build"return 1fi+ if ! cp pulse "$binary_stage/pulse" || ! chmod 755 "$binary_stage/pulse" || ! chown pulse:pulse "$binary_stage/pulse"; then+ print_error "Could not prepare the built Pulse binary; Pulse has not been stopped"+ cd "$original_dir" >/dev/null 2>&1 || true+ rm -rf "$binary_stage" "$temp_build"+ return 1+ fi+ if ! version_output=$(timeout 5 "$binary_stage/pulse" --version 2>/dev/null) || [[ -z "${version_output//[[:space:]]/}" ]]; then+ print_error "Built Pulse binary could not report its version; Pulse has not been stopped"+ cd "$original_dir" >/dev/null 2>&1 || true+ rm -rf "$binary_stage" "$temp_build"+ return 1+ fi+ if ! source_revision=$(git rev-parse --short HEAD) || [[ -z "$source_revision" ]]; then+ print_error "Could not identify the source build; Pulse has not been stopped"+ cd "$original_dir" >/dev/null 2>&1 || true+ rm -rf "$binary_stage" "$temp_build"+ return 1+ fi- mkdir -p "$INSTALL_DIR/bin" "$INSTALL_DIR/scripts"-- if [[ -f "$INSTALL_DIR/bin/pulse" ]]; then- mv "$INSTALL_DIR/bin/pulse" "$INSTALL_DIR/bin/pulse.old" 2>/dev/null || true+ service_name=$(detect_service_name)+ PULSE_WAS_ACTIVE="false"+ if ! stop_pulse_for_replacement "$service_name"; then+ cd "$original_dir" >/dev/null 2>&1 || true+ rm -rf "$binary_stage" "$temp_build"+ return 1fi- if ! cp pulse "$INSTALL_DIR/bin/pulse"; then- print_error "Failed to copy built Pulse binary"- [[ -f "$INSTALL_DIR/bin/pulse.old" ]] && mv "$INSTALL_DIR/bin/pulse.old" "$INSTALL_DIR/bin/pulse"+ if ! mv -fT "$binary_stage/pulse" "$INSTALL_DIR/bin/pulse"; then+ print_error "Failed to replace built Pulse; the previous binary is unchanged"cd "$original_dir" >/dev/null 2>&1 || true- rm -rf "$temp_build"+ rm -rf "$binary_stage" "$temp_build"+ recover_pulse_after_failed_replacement "$service_name"return 1fi- chmod +x "$INSTALL_DIR/bin/pulse"+ rm -rf "$binary_stage"++ mkdir -p "$INSTALL_DIR/scripts"for script_name in install-container-agent.sh install-docker.sh install.sh install.ps1; doif [[ -f "scripts/$script_name" ]]; then@@ -3879,12 +3907,11 @@ build_from_source() {install_binary_symlink "$INSTALL_DIR/bin/pulse" "$BINARY_LINK_PATH"- echo "$branch-$(git rev-parse --short HEAD)" > "$INSTALL_DIR/VERSION"+ echo "$branch-$source_revision" > "$INSTALL_DIR/VERSION"echo "$branch" > "$BUILD_FROM_SOURCE_MARKER"chown -R pulse:pulse "$INSTALL_DIR" 2>/dev/null || truechown pulse:pulse "$BUILD_FROM_SOURCE_MARKER" 2>/dev/null || true- rm -f "$INSTALL_DIR/bin/pulse.old"cd "$original_dir" >/dev/null 2>&1 || truerm -rf "$temp_build"diff --git a/scripts/installtests/root_install_sh_test.go b/scripts/installtests/root_install_sh_test.goindex 9d1bb38749..49ccec9b28 100644--- a/scripts/installtests/root_install_sh_test.go+++ b/scripts/installtests/root_install_sh_test.go@@ -149,6 +149,31 @@ func TestRootInstallScriptStagesUpdateBeforeStoppingService(t *testing.T) {if strings.Contains(archiveInstall, "pulse.old") || strings.Contains(archiveInstall, "temp_extract2") {t.Fatal("archive replacement must not move/delete the old binary or blindly re-extract")}+ sourceInstall := extractRootInstallShellFunction(t, "build_from_source")+ previous = -1+ for _, step := range []string{+ `make build`,+ `mktemp -d "$INSTALL_DIR/bin/.pulse-stage-XXXXXX"`,+ `cp pulse "$binary_stage/pulse"`,+ `chmod 755 "$binary_stage/pulse"`,+ `chown pulse:pulse "$binary_stage/pulse"`,+ `timeout 5 "$binary_stage/pulse" --version`,+ `source_revision=$(git rev-parse --short HEAD)`,+ `stop_pulse_for_replacement "$service_name"`,+ `mv -fT "$binary_stage/pulse" "$INSTALL_DIR/bin/pulse"`,+ } {+ position := strings.Index(sourceInstall, step)+ if position <= previous {+ t.Fatalf("source admission/replacement steps are out of order or missing at %s", step)+ }+ previous = position+ }+ if strings.Contains(sourceInstall, "pulse.old") || strings.Contains(sourceInstall, `cp pulse "$INSTALL_DIR/bin/pulse"`) {+ t.Fatal("source replacement must not move/delete the old binary or copy onto the live path")+ }+ if !strings.Contains(sourceInstall, `recover_pulse_after_failed_replacement "$service_name"`) {+ t.Fatal("failed source rename must use prior-active-only recovery")+ }}func TestRootInstallScriptInstallsSignatureVerificationDependencies(t *testing.T) {diff --git a/scripts/tests/test_server_installer_stop.py b/scripts/tests/test_server_installer_stop.pyindex aa1dae482b..f0d7e2fd5e 100644--- a/scripts/tests/test_server_installer_stop.py+++ b/scripts/tests/test_server_installer_stop.py@@ -6,6 +6,7 @@ These are ordinary-user installer controls, not native systemd acceptance.from pathlib import Pathimport os+import itertoolsimport subprocessimport tempfileimport tarfile@@ -13,7 +14,7 @@ import unittestROOT = Path(__file__).resolve().parents[2]-INSTALLER = ROOT / "install.sh"+INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh"))SYSTEMCTL = r'''#!/usr/bin/env bashset -eu@@ -123,8 +124,10 @@ git() {if [[ "$1" == clone ]]; thenlocal target="${@: -1}"mkdir -p "$target/frontend-modern"- cp "$FIXTURE/new-binary" "$target/pulse"+ [[ "$FAULT" == missing_binary ]] || cp "$FIXTURE/new-binary" "$target/pulse"else+ [[ "$FAULT" != source_revision ]] || return 1+ [[ "$FAULT" != source_revision_empty ]] || return 0echo abc1234fi}@@ -180,6 +183,7 @@ class ServerInstallerStopTest(unittest.TestCase):(fixture / "config").mkdir()originals = {installed / "bin/pulse": b"old executable\n",+ installed / "bin/pulse.old": b"pre-existing recovery binary\n",installed / "VERSION": b"v6.5.0\n",installed / "BUILD_FROM_SOURCE": b"old branch\n",fixture / "config/private-state": b"fixture persistent state\n",@@ -192,6 +196,8 @@ class ServerInstallerStopTest(unittest.TestCase):new_binary += b"sleep 10\n"elif fault == "version_exit":new_binary += b"echo v6.6.0; exit 1\n"+ elif fault == "version_empty":+ new_binary += b"printf ' \\t\\n'\n"else:version = "v6.5.0" if fault == "version_mismatch" else "unknown" if fault == "version_unknown" else "v6.6.0"new_binary += ("echo " + version + "\n").encode()@@ -277,25 +283,58 @@ class ServerInstallerStopTest(unittest.TestCase):self.assertTrue(observation["local_archive_retained"])def test_atomic_rename_failure_restores_only_a_previously_active_service(self):- for state in ("active", "inactive", "failed"):- for fault in ("rename", "recovery_start", "recovery_inactive"):- with self.subTest(state=state, fault=fault):- observation = self.exercise(REPLACEMENT, "archive", state, fault)- result = observation["result"]- self.assertNotEqual(result.returncode, 0)- self.assertEqual(observation["files"], observation["originals"])- self.assertTrue(all(m == 0o600 for m in observation["modes"].values()))- self.assertEqual(observation["calls"].count("start pulse-custom"), int(state == "active"))- self.assertNotIn("AGENTS", observation["calls"])- self.assertNotIn("SCRIPTS", observation["calls"])- if state == "active":- if fault == "rename":- self.assertIn("Previous Pulse binary is running again; the update failed", result.stdout)- else:- self.assertIn("could not be confirmed running", result.stderr)- self.assertEqual(observation["stage_paths"], [])- self.assertEqual(observation["temporary_paths_remaining"], [])- self.assertFalse(observation["archive_retained"])+ for flow, state, fault in itertools.product(+ ("archive", "source"), ("active", "inactive", "failed"),+ ("rename", "recovery_start", "recovery_inactive")+ ):+ with self.subTest(flow=flow, state=state, fault=fault):+ observation = self.exercise(REPLACEMENT, flow, state, fault)+ result = observation["result"]+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(observation["files"], observation["originals"])+ self.assertTrue(all(m == 0o600 for m in observation["modes"].values()))+ self.assertEqual(observation["calls"].count("start pulse-custom"), int(state == "active"))+ self.assertNotIn("AGENTS", observation["calls"])+ self.assertNotIn("SCRIPTS", observation["calls"])+ if state == "active":+ if fault == "rename":+ self.assertIn("Previous Pulse binary is running again; the update failed", result.stdout)+ else:+ self.assertIn("could not be confirmed running", result.stderr)+ self.assertEqual(observation["stage_paths"], [])+ self.assertEqual(observation["temporary_paths_remaining"], [])+ self.assertFalse(observation["archive_retained"])++ def test_source_admission_failures_never_stop_or_mutate_pulse(self):+ for fault in ("missing_binary", "stage_directory", "copy", "permissions",+ "ownership", "version_exit", "version_timeout", "version_empty",+ "source_revision", "source_revision_empty"):+ with self.subTest(fault=fault):+ observation = self.exercise(REPLACEMENT, "source", "active", fault)+ self.assertNotEqual(observation["result"].returncode, 0)+ self.assertEqual(observation["files"], observation["originals"])+ self.assertTrue(all(m == 0o600 for m in observation["modes"].values()))+ self.assertNotRegex(observation["calls"], r"(?m)^(show|stop|start|restart|RENAME|AGENTS|SCRIPTS)\b")+ self.assertEqual(observation["stage_paths"], [])+ self.assertEqual(observation["temporary_paths_remaining"], [])++ def test_source_success_admits_bytes_before_stop_and_commits_once(self):+ observation = self.exercise(REPLACEMENT, "source", "active")+ self.assertEqual(observation["result"].returncode, 0, observation["result"].stderr)+ calls = observation["calls"]+ self.assertLess(calls.index("VERSION_PROBE"), calls.index("stop pulse-custom"))+ self.assertLess(calls.index("stop pulse-custom"), calls.index("RENAME"))+ self.assertEqual(calls.count("RENAME"), 1)+ self.assertEqual(calls.count("VERSION_PROBE"), 1)+ self.assertEqual(observation["files"]["install/bin/pulse"], observation["new_binary"])+ self.assertEqual(observation["modes"]["install/bin/pulse"], 0o755)+ self.assertEqual(observation["files"]["install/bin/pulse.old"], observation["originals"]["install/bin/pulse.old"])+ self.assertEqual(observation["files"]["install/VERSION"], b"fixture-abc1234\n")+ self.assertEqual(observation["files"]["install/BUILD_FROM_SOURCE"], b"fixture\n")+ self.assertEqual(observation["files"]["config/private-state"], b"fixture persistent state\n")+ self.assertEqual(observation["stage_paths"], [])+ self.assertEqual(observation["temporary_paths_remaining"], [])+ self.assertNotIn("start pulse-custom", calls)def test_archive_success_admits_bytes_before_stop_and_commits_once(self):observation = self.exercise(REPLACEMENT, "archive", "active")