Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Refuse server replacement until Pulse is confirmed stopped

pulse-triage[bot]407da3addb1a
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 8a9771e941..4ee954dbc9 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -15,6 +15,30 @@
## Purpose
+### Server replacement requires a confirmed inactive service
+
+The server installer's archive and source-build paths stage their inputs before
+stopping Pulse. Manual version changes, menu updates and reinstalls must not stop
+the service before staging, so a download/build failure does not create an outage.
+Immediately before binary replacement, the detected service must return an
+explicit `inactive` state. An `active` or `failed` unit is stopped with the
+existing five-second command bound and read back; only an `inactive` result
+admits replacement. A failed unit does not imply prior workload liveness.
+
+Failed/timed-out reads or stops, transitional states and malformed/empty results
+refuse replacement without an automatic restart or retry. Retain the installed
+binary, version and source-build marker on this refusal; temporary downloaded
+archives are cleaned up. Preserve the prior-active flag across a successful stop
+for the existing post-update liveness check. This does not establish restoration
+after a later installation failure, filesystem/data rollback or native systemd
+acceptance.
+
+`scripts/tests/test_server_installer_stop.py` exercises both real replacement
+paths with fixture-only external producers and a local service-command shim,
+including actual command timeouts, bytes/modes, liveness attribution and all
+three manual staging-failure flows. The existing root-installer ordering and
+update-resilience tests remain independent controls.
+
### Shipped documentation is checked before CI dependency work
Build and Test checks every shipped Markdown mirror immediately after the
diff --git a/install.sh b/install.sh
index 6fe81754df..2e92d00cb9 100755
--- a/install.sh
+++ b/install.sh
@@ -590,6 +590,44 @@ safe_systemctl() {
}
}
+# A failed/timed-out is-active query is not evidence that replacement is safe.
+# Require an explicit stopped state, and read it back after a successful stop.
+# Do not restart after an uncertain stop: the operator must reconcile that state.
+stop_pulse_for_replacement() {
+ local service_name="$1"
+ local state=""
+ if ! state=$(timeout 5 systemctl show "$service_name" --property=ActiveState --value 2>/dev/null); then
+ print_error "Could not read Pulse service state ($service_name); refusing binary replacement"
+ return 1
+ fi
+ case "$state" in
+ inactive) return 0 ;;
+ active) PULSE_WAS_ACTIVE="true" ;;
+ failed) ;; # A failed unit can retain processes; stop it without claiming prior liveness.
+ *)
+ print_error "Pulse service state is not settled ($service_name); refusing binary replacement"
+ return 1
+ ;;
+ esac
+
+ print_info "Stopping existing Pulse service ($service_name)..."
+ if ! safe_systemctl stop "$service_name"; then
+ print_error "Could not stop Pulse ($service_name); refusing binary replacement. Check service state before retrying."
+ return 1
+ fi
+ if ! state=$(timeout 5 systemctl show "$service_name" --property=ActiveState --value 2>/dev/null); then
+ print_error "Could not confirm Pulse stopped ($service_name); refusing binary replacement"
+ return 1
+ fi
+ case "$state" in
+ inactive) return 0 ;;
+ *)
+ print_error "Pulse has not reached a stopped state ($service_name); refusing binary replacement"
+ return 1
+ ;;
+ esac
+}
+
# Detect existing service name (pulse or pulse-backend)
detect_service_name() {
if [[ "$SERVICE_NAME_EXPLICIT" == "true" ]]; then
@@ -3447,8 +3485,6 @@ download_pulse() {
local archive_from_temp=false
local inferred_release=""
- rm -f "$BUILD_FROM_SOURCE_MARKER"
-
if ! ensure_update_disk_headroom "/tmp" "$INSTALL_DIR"; then
# The configuration snapshot taken earlier is not needed: the update
# never reached staging. Remove it so a retry loop on a low-space
@@ -3507,12 +3543,13 @@ download_pulse() {
exit 1
fi
- # Detect and stop existing service after the archive is available but before replacing the binary.
+ # Stage first, then require a confirmed stop before any binary replacement.
EXISTING_SERVICE=$(detect_service_name)
- if timeout 5 systemctl is-active --quiet "$EXISTING_SERVICE" 2>/dev/null; then
- print_info "Stopping existing Pulse service ($EXISTING_SERVICE)..."
- safe_systemctl stop "$EXISTING_SERVICE" || true
- sleep 2
+ if ! stop_pulse_for_replacement "$EXISTING_SERVICE"; then
+ if [[ "$archive_from_temp" == "true" ]]; then
+ rm -f "$archive_path" "${archive_path}.sshsig"
+ fi
+ exit 1
fi
if ! install_pulse_archive "$archive_path" "$expected_release"; then
@@ -3522,6 +3559,7 @@ download_pulse() {
exit 1
fi
+ rm -f "$BUILD_FROM_SOURCE_MARKER"
if [[ "$archive_from_temp" == "true" ]]; then
rm -f "$archive_path" "${archive_path}.sshsig"
fi
@@ -3815,10 +3853,10 @@ build_from_source() {
fi
service_name=$(detect_service_name)
- if timeout 5 systemctl is-active --quiet "$service_name" 2>/dev/null; then
- print_info "Stopping existing Pulse service ($service_name)..."
- safe_systemctl stop "$service_name" || true
- sleep 2
+ if ! stop_pulse_for_replacement "$service_name"; then
+ cd "$original_dir" >/dev/null 2>&1 || true
+ rm -rf "$temp_build"
+ return 1
fi
mkdir -p "$INSTALL_DIR/bin" "$INSTALL_DIR/scripts"
@@ -4648,10 +4686,7 @@ wait_for_service_active() {
# guarantee it comes back up afterward (#1323).
stop_pulse_for_update() {
PULSE_WAS_ACTIVE="false"
- if timeout 5 systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
- PULSE_WAS_ACTIVE="true"
- fi
- systemctl stop "$SERVICE_NAME" 2>/dev/null || true
+ stop_pulse_for_replacement "$SERVICE_NAME"
}
# After an update, if Pulse was running beforehand, make sure it is running again:
@@ -4912,7 +4947,6 @@ main() {
fi
backup_existing
- stop_pulse_for_update
create_user
download_pulse
# A half-removed installation (binary present, /etc/pulse or the
@@ -5084,7 +5118,6 @@ main() {
fi
backup_existing
- stop_pulse_for_update
create_user
download_pulse
# Same repair as the --version path: a half-removed
@@ -5112,7 +5145,6 @@ main() {
offer_existing_auto_updates
backup_existing
- stop_pulse_for_update
create_user
download_pulse
setup_directories
diff --git a/scripts/installtests/root_install_sh_test.go b/scripts/installtests/root_install_sh_test.go
index b19400c6ef..be92bb2678 100644
--- a/scripts/installtests/root_install_sh_test.go
+++ b/scripts/installtests/root_install_sh_test.go
@@ -112,7 +112,7 @@ func TestRootInstallScriptStagesUpdateBeforeStoppingService(t *testing.T) {
`ensure_update_disk_headroom "/tmp" "$INSTALL_DIR"`,
`download_release_archive "$LATEST_RELEASE" "$pulse_arch" "$archive_path"`,
`run_upgrade_readiness_preflight "$CURRENT_VERSION" "$expected_release"`,
- `safe_systemctl stop "$EXISTING_SERVICE"`,
+ `stop_pulse_for_replacement "$EXISTING_SERVICE"`,
`install_pulse_archive "$archive_path" "$expected_release"`,
}
diff --git a/scripts/tests/test-install-update-resilience.sh b/scripts/tests/test-install-update-resilience.sh
index 081cc7879a..189df560fd 100755
--- a/scripts/tests/test-install-update-resilience.sh
+++ b/scripts/tests/test-install-update-resilience.sh
@@ -24,6 +24,7 @@ SYSTEMCTL_ACTIVE="no"
START_ATTEMPTS=0
systemctl() {
case "$1" in
+ show) if [[ "$SYSTEMCTL_ACTIVE" == "yes" ]]; then echo active; else echo inactive; fi ;;
is-active) [[ "$SYSTEMCTL_ACTIVE" == "yes" ]] && return 0 || return 1 ;;
stop) SYSTEMCTL_ACTIVE="no"; return 0 ;;
start) START_ATTEMPTS=$((START_ATTEMPTS + 1)); return 0 ;;
diff --git a/scripts/tests/test_server_installer_stop.py b/scripts/tests/test_server_installer_stop.py
new file mode 100644
index 0000000000..6697b0a394
--- /dev/null
+++ b/scripts/tests/test_server_installer_stop.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Exercise real replacement paths; external services/builds use local fixtures.
+
+These are ordinary-user installer controls, not native systemd acceptance.
+"""
+
+from pathlib import Path
+import os
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+INSTALLER = ROOT / "install.sh"
+
+SYSTEMCTL = r'''#!/usr/bin/env bash
+set -eu
+printf '%s\n' "$*" >> "$FIXTURE/calls"
+case "$1" in
+ show)
+ if [[ -f "$FIXTURE/stopped" ]]; then
+ [[ "$CASE" != post_query_error ]] || exit 1
+ case "$CASE" in
+ still_active) echo active ;;
+ post_transition) echo deactivating ;;
+ post_failed) echo failed ;;
+ *) echo inactive ;;
+ esac
+ else
+ case "$CASE" in
+ query_error) exit 1 ;;
+ query_timeout) sleep 10 ;;
+ empty) : ;;
+ malformed) printf 'active\ninactive\n' ;;
+ inactive|failed|activating|deactivating|reloading|unknown) echo "$CASE" ;;
+ *) echo active ;;
+ esac
+ fi
+ ;;
+ is-active) [[ "$CASE" != inactive && "$CASE" != failed ]] ;;
+ stop)
+ case "$CASE" in
+ stop_error) exit 1 ;;
+ stop_timeout) sleep 10 ;;
+ esac
+ touch "$FIXTURE/stopped"
+ ;;
+ *) exit 97 ;;
+esac
+'''
+
+REPLACEMENT = r'''
+source "$INSTALLER"
+INSTALL_DIR="$FIXTURE/install"
+CONFIG_DIR="$FIXTURE/config"
+BUILD_FROM_SOURCE_MARKER="$INSTALL_DIR/BUILD_FROM_SOURCE"
+BINARY_LINK_PATH="$FIXTURE/link"
+CURRENT_VERSION=v6.5.0
+PULSE_WAS_ACTIVE=false
+print_info() { :; }
+print_success() { :; }
+print_error() { echo "$*" >&2; }
+detect_service_name() { echo pulse-custom; }
+# No network, package installation, signature bypass in production or root writes:
+# these fixture-only producers supply the inputs to the actual replacement path.
+ensure_update_disk_headroom() { return 0; }
+detect_pulse_architecture() { echo amd64; }
+run_upgrade_readiness_preflight() { return 0; }
+resolve_target_release() { LATEST_RELEASE=v6.6.0; }
+create_temp_archive_path() { printf '%s\n' "$FIXTURE/archive.tgz"; }
+download_release_archive() { touch "$3" "${3}.sshsig"; }
+install_pulse_archive() {
+ echo REPLACED >> "$FIXTURE/calls"
+ cp "$FIXTURE/new-binary" "$INSTALL_DIR/bin/pulse"
+ echo v6.6.0 > "$INSTALL_DIR/VERSION"
+}
+apt-get() { return 0; }
+go() { echo 'go version go1.99.0 linux/amd64'; }
+npm() { return 0; }
+make() { return 0; }
+git() {
+ if [[ "$1" == clone ]]; then
+ local target="${@: -1}"
+ mkdir -p "$target/frontend-modern"
+ cp "$FIXTURE/new-binary" "$target/pulse"
+ else
+ echo abc1234
+ fi
+}
+chown() { return 0; }
+install_binary_symlink() { echo REPLACED >> "$FIXTURE/calls"; }
+if [[ "$FLOW" == archive ]]; then
+ download_pulse
+else
+ build_from_source fixture
+fi
+printf 'WAS_ACTIVE=%s\n' "$PULSE_WAS_ACTIVE"
+'''
+
+STAGING_FAILURE = r'''
+source "$INSTALLER"
+INSTALL_DIR="$FIXTURE/install"
+CONFIG_DIR="$FIXTURE/config"
+CURRENT_VERSION=v6.5.0
+LATEST_RELEASE=v6.6.0
+FORCE_VERSION=""
+[[ "$FLOW" != version ]] || FORCE_VERSION=v6.6.0
+for fn in print_header print_info check_root detect_os check_docker_environment backup_existing create_user offer_existing_auto_updates; do
+ eval "$fn() { :; }"
+done
+check_proxmox_host() { return 1; }
+check_existing_installation() { return 0; }
+detect_service_name() { echo pulse-custom; }
+resolve_latest_release_tag_for_channel() { echo v6.6.0; }
+read_configured_update_channel() { echo stable; }
+run_upgrade_readiness_preflight() { return 0; }
+safe_read() {
+ local choice=1
+ [[ "$FLOW" != reinstall ]] || choice=2
+ printf -v "$2" '%s' "$choice"
+}
+curl() { return 0; }
+download_pulse() { echo 'staging failed' >&2; exit 42; }
+main
+'''
+
+
+class ServerInstallerStopTest(unittest.TestCase):
+ def exercise(self, script, flow, case):
+ with tempfile.TemporaryDirectory() as temporary:
+ fixture = Path(temporary)
+ (fixture / "bin").mkdir()
+ installed = fixture / "install"
+ (installed / "bin").mkdir(parents=True)
+ (fixture / "config").mkdir()
+ originals = {
+ installed / "bin/pulse": b"old executable\n",
+ installed / "VERSION": b"v6.5.0\n",
+ installed / "BUILD_FROM_SOURCE": b"old branch\n",
+ fixture / "config/private-state": b"fixture persistent state\n",
+ }
+ for path, content in originals.items():
+ path.write_bytes(content)
+ path.chmod(0o600)
+ (fixture / "new-binary").write_bytes(b"new executable\n")
+ (fixture / "bin/systemctl").write_text(SYSTEMCTL)
+ (fixture / "bin/systemctl").chmod(0o700)
+ env = {**os.environ, "FIXTURE": str(fixture), "CASE": case,
+ "FLOW": flow, "INSTALLER": str(INSTALLER),
+ "PATH": str(fixture / "bin") + ":" + os.environ["PATH"]}
+ result = subprocess.run(["bash", "-c", script], env=env,
+ capture_output=True, text=True, timeout=30)
+ return {
+ "result": result,
+ "files": {str(p.relative_to(fixture)): p.read_bytes() if p.exists() else None for p in originals},
+ "modes": {str(p.relative_to(fixture)): p.stat().st_mode & 0o777 if p.exists() else None for p in originals},
+ "originals": {str(p.relative_to(fixture)): b for p, b in originals.items()},
+ "calls": (fixture / "calls").read_text() if (fixture / "calls").exists() else "",
+ "archive_retained": (fixture / "archive.tgz").exists(),
+ }
+
+ def test_refused_replacements_preserve_installed_bytes_and_modes(self):
+ for flow in ("archive", "source"):
+ for case in ("stop_error", "stop_timeout", "query_error", "query_timeout",
+ "still_active", "post_query_error", "post_transition", "post_failed",
+ "activating", "deactivating", "reloading", "unknown", "empty", "malformed"):
+ with self.subTest(flow=flow, case=case):
+ observation = self.exercise(REPLACEMENT, flow, case)
+ result = observation["result"]
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("refusing binary replacement", result.stderr)
+ self.assertEqual(observation["files"], observation["originals"])
+ self.assertTrue(all(m == 0o600 for m in observation["modes"].values()))
+ self.assertNotIn("REPLACED", observation["calls"])
+ self.assertNotRegex(observation["calls"], r"(?m)^(start|restart)\b")
+ self.assertFalse(observation["archive_retained"])
+
+ def test_confirmed_stop_and_inactive_installs_keep_liveness_attribution(self):
+ for flow in ("archive", "source"):
+ for case in ("active", "inactive", "failed"):
+ with self.subTest(flow=flow, case=case):
+ observation = self.exercise(REPLACEMENT, flow, case)
+ result = observation["result"]
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(observation["files"]["install/bin/pulse"], b"new executable\n")
+ self.assertEqual(observation["files"]["config/private-state"], b"fixture persistent state\n")
+ self.assertIn("WAS_ACTIVE=" + str(case == "active").lower(), result.stdout)
+ self.assertEqual(observation["calls"].count("stop pulse-custom"), int(case != "inactive"))
+ self.assertFalse(observation["archive_retained"])
+
+ def test_all_manual_update_flows_leave_service_running_when_staging_fails(self):
+ for flow in ("version", "update", "reinstall"):
+ with self.subTest(flow=flow):
+ observation = self.exercise(STAGING_FAILURE, flow, "active")
+ self.assertEqual(observation["result"].returncode, 42)
+ self.assertEqual(observation["files"], observation["originals"])
+ self.assertNotRegex(observation["calls"], r"(?m)^stop\b")
+
+
+if __name__ == "__main__":
+ unittest.main()