Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Refuse server replacement until Pulse is confirmed stopped
pulse-triage[bot]407da3addb1a
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 8a9771e941..4ee954dbc9 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -15,6 +15,30 @@## Purpose+### Server replacement requires a confirmed inactive service++The server installer's archive and source-build paths stage their inputs before+stopping Pulse. Manual version changes, menu updates and reinstalls must not stop+the service before staging, so a download/build failure does not create an outage.+Immediately before binary replacement, the detected service must return an+explicit `inactive` state. An `active` or `failed` unit is stopped with the+existing five-second command bound and read back; only an `inactive` result+admits replacement. A failed unit does not imply prior workload liveness.++Failed/timed-out reads or stops, transitional states and malformed/empty results+refuse replacement without an automatic restart or retry. Retain the installed+binary, version and source-build marker on this refusal; temporary downloaded+archives are cleaned up. Preserve the prior-active flag across a successful stop+for the existing post-update liveness check. This does not establish restoration+after a later installation failure, filesystem/data rollback or native systemd+acceptance.++`scripts/tests/test_server_installer_stop.py` exercises both real replacement+paths with fixture-only external producers and a local service-command shim,+including actual command timeouts, bytes/modes, liveness attribution and all+three manual staging-failure flows. The existing root-installer ordering and+update-resilience tests remain independent controls.+### Shipped documentation is checked before CI dependency workBuild and Test checks every shipped Markdown mirror immediately after thediff --git a/install.sh b/install.shindex 6fe81754df..2e92d00cb9 100755--- a/install.sh+++ b/install.sh@@ -590,6 +590,44 @@ safe_systemctl() {}}+# A failed/timed-out is-active query is not evidence that replacement is safe.+# Require an explicit stopped state, and read it back after a successful stop.+# Do not restart after an uncertain stop: the operator must reconcile that state.+stop_pulse_for_replacement() {+ local service_name="$1"+ local state=""+ if ! state=$(timeout 5 systemctl show "$service_name" --property=ActiveState --value 2>/dev/null); then+ print_error "Could not read Pulse service state ($service_name); refusing binary replacement"+ return 1+ fi+ case "$state" in+ inactive) return 0 ;;+ active) PULSE_WAS_ACTIVE="true" ;;+ failed) ;; # A failed unit can retain processes; stop it without claiming prior liveness.+ *)+ print_error "Pulse service state is not settled ($service_name); refusing binary replacement"+ return 1+ ;;+ esac++ print_info "Stopping existing Pulse service ($service_name)..."+ if ! safe_systemctl stop "$service_name"; then+ print_error "Could not stop Pulse ($service_name); refusing binary replacement. Check service state before retrying."+ return 1+ fi+ if ! state=$(timeout 5 systemctl show "$service_name" --property=ActiveState --value 2>/dev/null); then+ print_error "Could not confirm Pulse stopped ($service_name); refusing binary replacement"+ return 1+ fi+ case "$state" in+ inactive) return 0 ;;+ *)+ print_error "Pulse has not reached a stopped state ($service_name); refusing binary replacement"+ return 1+ ;;+ esac+}+# Detect existing service name (pulse or pulse-backend)detect_service_name() {if [[ "$SERVICE_NAME_EXPLICIT" == "true" ]]; then@@ -3447,8 +3485,6 @@ download_pulse() {local archive_from_temp=falselocal inferred_release=""- rm -f "$BUILD_FROM_SOURCE_MARKER"-if ! ensure_update_disk_headroom "/tmp" "$INSTALL_DIR"; then# The configuration snapshot taken earlier is not needed: the update# never reached staging. Remove it so a retry loop on a low-space@@ -3507,12 +3543,13 @@ download_pulse() {exit 1fi- # Detect and stop existing service after the archive is available but before replacing the binary.+ # Stage first, then require a confirmed stop before any binary replacement.EXISTING_SERVICE=$(detect_service_name)- if timeout 5 systemctl is-active --quiet "$EXISTING_SERVICE" 2>/dev/null; then- print_info "Stopping existing Pulse service ($EXISTING_SERVICE)..."- safe_systemctl stop "$EXISTING_SERVICE" || true- sleep 2+ if ! stop_pulse_for_replacement "$EXISTING_SERVICE"; then+ if [[ "$archive_from_temp" == "true" ]]; then+ rm -f "$archive_path" "${archive_path}.sshsig"+ fi+ exit 1fiif ! install_pulse_archive "$archive_path" "$expected_release"; then@@ -3522,6 +3559,7 @@ download_pulse() {exit 1fi+ rm -f "$BUILD_FROM_SOURCE_MARKER"if [[ "$archive_from_temp" == "true" ]]; thenrm -f "$archive_path" "${archive_path}.sshsig"fi@@ -3815,10 +3853,10 @@ build_from_source() {fiservice_name=$(detect_service_name)- if timeout 5 systemctl is-active --quiet "$service_name" 2>/dev/null; then- print_info "Stopping existing Pulse service ($service_name)..."- safe_systemctl stop "$service_name" || true- sleep 2+ if ! stop_pulse_for_replacement "$service_name"; then+ cd "$original_dir" >/dev/null 2>&1 || true+ rm -rf "$temp_build"+ return 1fimkdir -p "$INSTALL_DIR/bin" "$INSTALL_DIR/scripts"@@ -4648,10 +4686,7 @@ wait_for_service_active() {# guarantee it comes back up afterward (#1323).stop_pulse_for_update() {PULSE_WAS_ACTIVE="false"- if timeout 5 systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then- PULSE_WAS_ACTIVE="true"- fi- systemctl stop "$SERVICE_NAME" 2>/dev/null || true+ stop_pulse_for_replacement "$SERVICE_NAME"}# After an update, if Pulse was running beforehand, make sure it is running again:@@ -4912,7 +4947,6 @@ main() {fibackup_existing- stop_pulse_for_updatecreate_userdownload_pulse# A half-removed installation (binary present, /etc/pulse or the@@ -5084,7 +5118,6 @@ main() {fibackup_existing- stop_pulse_for_updatecreate_userdownload_pulse# Same repair as the --version path: a half-removed@@ -5112,7 +5145,6 @@ main() {offer_existing_auto_updatesbackup_existing- stop_pulse_for_updatecreate_userdownload_pulsesetup_directoriesdiff --git a/scripts/installtests/root_install_sh_test.go b/scripts/installtests/root_install_sh_test.goindex b19400c6ef..be92bb2678 100644--- a/scripts/installtests/root_install_sh_test.go+++ b/scripts/installtests/root_install_sh_test.go@@ -112,7 +112,7 @@ func TestRootInstallScriptStagesUpdateBeforeStoppingService(t *testing.T) {`ensure_update_disk_headroom "/tmp" "$INSTALL_DIR"`,`download_release_archive "$LATEST_RELEASE" "$pulse_arch" "$archive_path"`,`run_upgrade_readiness_preflight "$CURRENT_VERSION" "$expected_release"`,- `safe_systemctl stop "$EXISTING_SERVICE"`,+ `stop_pulse_for_replacement "$EXISTING_SERVICE"`,`install_pulse_archive "$archive_path" "$expected_release"`,}diff --git a/scripts/tests/test-install-update-resilience.sh b/scripts/tests/test-install-update-resilience.shindex 081cc7879a..189df560fd 100755--- a/scripts/tests/test-install-update-resilience.sh+++ b/scripts/tests/test-install-update-resilience.sh@@ -24,6 +24,7 @@ SYSTEMCTL_ACTIVE="no"START_ATTEMPTS=0systemctl() {case "$1" in+ show) if [[ "$SYSTEMCTL_ACTIVE" == "yes" ]]; then echo active; else echo inactive; fi ;;is-active) [[ "$SYSTEMCTL_ACTIVE" == "yes" ]] && return 0 || return 1 ;;stop) SYSTEMCTL_ACTIVE="no"; return 0 ;;start) START_ATTEMPTS=$((START_ATTEMPTS + 1)); return 0 ;;diff --git a/scripts/tests/test_server_installer_stop.py b/scripts/tests/test_server_installer_stop.pynew file mode 100644index 0000000000..6697b0a394--- /dev/null+++ b/scripts/tests/test_server_installer_stop.py@@ -0,0 +1,202 @@+#!/usr/bin/env python3+"""Exercise real replacement paths; external services/builds use local fixtures.++These are ordinary-user installer controls, not native systemd acceptance.+"""++from pathlib import Path+import os+import subprocess+import tempfile+import unittest+++ROOT = Path(__file__).resolve().parents[2]+INSTALLER = ROOT / "install.sh"++SYSTEMCTL = r'''#!/usr/bin/env bash+set -eu+printf '%s\n' "$*" >> "$FIXTURE/calls"+case "$1" in+ show)+ if [[ -f "$FIXTURE/stopped" ]]; then+ [[ "$CASE" != post_query_error ]] || exit 1+ case "$CASE" in+ still_active) echo active ;;+ post_transition) echo deactivating ;;+ post_failed) echo failed ;;+ *) echo inactive ;;+ esac+ else+ case "$CASE" in+ query_error) exit 1 ;;+ query_timeout) sleep 10 ;;+ empty) : ;;+ malformed) printf 'active\ninactive\n' ;;+ inactive|failed|activating|deactivating|reloading|unknown) echo "$CASE" ;;+ *) echo active ;;+ esac+ fi+ ;;+ is-active) [[ "$CASE" != inactive && "$CASE" != failed ]] ;;+ stop)+ case "$CASE" in+ stop_error) exit 1 ;;+ stop_timeout) sleep 10 ;;+ esac+ touch "$FIXTURE/stopped"+ ;;+ *) exit 97 ;;+esac+'''++REPLACEMENT = r'''+source "$INSTALLER"+INSTALL_DIR="$FIXTURE/install"+CONFIG_DIR="$FIXTURE/config"+BUILD_FROM_SOURCE_MARKER="$INSTALL_DIR/BUILD_FROM_SOURCE"+BINARY_LINK_PATH="$FIXTURE/link"+CURRENT_VERSION=v6.5.0+PULSE_WAS_ACTIVE=false+print_info() { :; }+print_success() { :; }+print_error() { echo "$*" >&2; }+detect_service_name() { echo pulse-custom; }+# No network, package installation, signature bypass in production or root writes:+# these fixture-only producers supply the inputs to the actual replacement path.+ensure_update_disk_headroom() { return 0; }+detect_pulse_architecture() { echo amd64; }+run_upgrade_readiness_preflight() { return 0; }+resolve_target_release() { LATEST_RELEASE=v6.6.0; }+create_temp_archive_path() { printf '%s\n' "$FIXTURE/archive.tgz"; }+download_release_archive() { touch "$3" "${3}.sshsig"; }+install_pulse_archive() {+ echo REPLACED >> "$FIXTURE/calls"+ cp "$FIXTURE/new-binary" "$INSTALL_DIR/bin/pulse"+ echo v6.6.0 > "$INSTALL_DIR/VERSION"+}+apt-get() { return 0; }+go() { echo 'go version go1.99.0 linux/amd64'; }+npm() { return 0; }+make() { return 0; }+git() {+ if [[ "$1" == clone ]]; then+ local target="${@: -1}"+ mkdir -p "$target/frontend-modern"+ cp "$FIXTURE/new-binary" "$target/pulse"+ else+ echo abc1234+ fi+}+chown() { return 0; }+install_binary_symlink() { echo REPLACED >> "$FIXTURE/calls"; }+if [[ "$FLOW" == archive ]]; then+ download_pulse+else+ build_from_source fixture+fi+printf 'WAS_ACTIVE=%s\n' "$PULSE_WAS_ACTIVE"+'''++STAGING_FAILURE = r'''+source "$INSTALLER"+INSTALL_DIR="$FIXTURE/install"+CONFIG_DIR="$FIXTURE/config"+CURRENT_VERSION=v6.5.0+LATEST_RELEASE=v6.6.0+FORCE_VERSION=""+[[ "$FLOW" != version ]] || FORCE_VERSION=v6.6.0+for fn in print_header print_info check_root detect_os check_docker_environment backup_existing create_user offer_existing_auto_updates; do+ eval "$fn() { :; }"+done+check_proxmox_host() { return 1; }+check_existing_installation() { return 0; }+detect_service_name() { echo pulse-custom; }+resolve_latest_release_tag_for_channel() { echo v6.6.0; }+read_configured_update_channel() { echo stable; }+run_upgrade_readiness_preflight() { return 0; }+safe_read() {+ local choice=1+ [[ "$FLOW" != reinstall ]] || choice=2+ printf -v "$2" '%s' "$choice"+}+curl() { return 0; }+download_pulse() { echo 'staging failed' >&2; exit 42; }+main+'''+++class ServerInstallerStopTest(unittest.TestCase):+ def exercise(self, script, flow, case):+ with tempfile.TemporaryDirectory() as temporary:+ fixture = Path(temporary)+ (fixture / "bin").mkdir()+ installed = fixture / "install"+ (installed / "bin").mkdir(parents=True)+ (fixture / "config").mkdir()+ originals = {+ installed / "bin/pulse": b"old executable\n",+ installed / "VERSION": b"v6.5.0\n",+ installed / "BUILD_FROM_SOURCE": b"old branch\n",+ fixture / "config/private-state": b"fixture persistent state\n",+ }+ for path, content in originals.items():+ path.write_bytes(content)+ path.chmod(0o600)+ (fixture / "new-binary").write_bytes(b"new executable\n")+ (fixture / "bin/systemctl").write_text(SYSTEMCTL)+ (fixture / "bin/systemctl").chmod(0o700)+ env = {**os.environ, "FIXTURE": str(fixture), "CASE": case,+ "FLOW": flow, "INSTALLER": str(INSTALLER),+ "PATH": str(fixture / "bin") + ":" + os.environ["PATH"]}+ result = subprocess.run(["bash", "-c", script], env=env,+ capture_output=True, text=True, timeout=30)+ return {+ "result": result,+ "files": {str(p.relative_to(fixture)): p.read_bytes() if p.exists() else None for p in originals},+ "modes": {str(p.relative_to(fixture)): p.stat().st_mode & 0o777 if p.exists() else None for p in originals},+ "originals": {str(p.relative_to(fixture)): b for p, b in originals.items()},+ "calls": (fixture / "calls").read_text() if (fixture / "calls").exists() else "",+ "archive_retained": (fixture / "archive.tgz").exists(),+ }++ def test_refused_replacements_preserve_installed_bytes_and_modes(self):+ for flow in ("archive", "source"):+ for case in ("stop_error", "stop_timeout", "query_error", "query_timeout",+ "still_active", "post_query_error", "post_transition", "post_failed",+ "activating", "deactivating", "reloading", "unknown", "empty", "malformed"):+ with self.subTest(flow=flow, case=case):+ observation = self.exercise(REPLACEMENT, flow, case)+ result = observation["result"]+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("refusing binary replacement", result.stderr)+ self.assertEqual(observation["files"], observation["originals"])+ self.assertTrue(all(m == 0o600 for m in observation["modes"].values()))+ self.assertNotIn("REPLACED", observation["calls"])+ self.assertNotRegex(observation["calls"], r"(?m)^(start|restart)\b")+ self.assertFalse(observation["archive_retained"])++ def test_confirmed_stop_and_inactive_installs_keep_liveness_attribution(self):+ for flow in ("archive", "source"):+ for case in ("active", "inactive", "failed"):+ with self.subTest(flow=flow, case=case):+ observation = self.exercise(REPLACEMENT, flow, case)+ result = observation["result"]+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(observation["files"]["install/bin/pulse"], b"new executable\n")+ self.assertEqual(observation["files"]["config/private-state"], b"fixture persistent state\n")+ self.assertIn("WAS_ACTIVE=" + str(case == "active").lower(), result.stdout)+ self.assertEqual(observation["calls"].count("stop pulse-custom"), int(case != "inactive"))+ self.assertFalse(observation["archive_retained"])++ def test_all_manual_update_flows_leave_service_running_when_staging_fails(self):+ for flow in ("version", "update", "reinstall"):+ with self.subTest(flow=flow):+ observation = self.exercise(STAGING_FAILURE, flow, "active")+ self.assertEqual(observation["result"].returncode, 42)+ self.assertEqual(observation["files"], observation["originals"])+ self.assertNotRegex(observation["calls"], r"(?m)^stop\b")+++if __name__ == "__main__":+ unittest.main()