Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Refuse unattended recovery on uncertain systemd service state

pulse-triage[bot]9060dbd54de3
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 1c4a607d02..d181a2898e 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -52,6 +52,37 @@ execute the actual unattended helper's selection and
`perform_update` entry/handoff with a fake installer, including timeout and
failed/partial inventory before any service observation or installer call.
+### Unattended restoration requires observed service state
+
+Before backup, download, installer invocation or restart-trap setup, the
+standalone helper requires one complete successful observation of the exact
+selected unit's LoadState and ActiveState. Loaded and masked units are admitted
+only in settled active, inactive or failed states; missing, partial, duplicate,
+failed, timed-out, unavailable or transitional observations refuse the update.
+A failed is-active call is no longer treated as evidence of prior inactivity.
+Discovery, explicit identities, masks, consent and stable/signature admission
+retain their existing contracts.
+
+After an attempted installation, rollback replacement requires an observed
+inactive unit, or a bounded successful stop followed by a fresh inactive
+readback. Successful stop submission alone does not permit file replacement.
+Unknown/unsettled state retains the verified private backup, preserves the
+current destination bytes, suppresses restart and reports manual reconciliation.
+A post-install observation failure is not permission to start or roll back.
+The prior-active backstop starts only a freshly observed stopped/failed unit;
+it never starts a unit on a failed read. Restoration still stages and compares
+all saved files, uses atomic destination replacement and restarts only when
+prior activity was established. Incomplete restoration/start leaves the backup
+and fails; no new automatic retry, unmasking or config/data rollback is added.
+
+`TestAutoUpdateServiceStateFailsClosed` executes the production helper with
+private versioned file fixtures and an external confined manager, including
+real query/stop deadlines, successful/no-op/refused stops, failed readback,
+prior-state restoration and late unknown state. The existing signed transaction
+matrix and smoke controls retain their cryptographic, metadata and return-trap
+coverage. These are source/fixture controls, not native systemd/LXC recovery,
+published installer trust, #2785's stop cause or installed acceptance.
+
### Core E2E images are built once and admitted in the same run
The secret-free Core E2E workflow builds its existing `e2e_runtime` server and
diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go
index 3401b84de4..484c97bf64 100644
--- a/scripts/installtests/pulse_auto_update_test.go
+++ b/scripts/installtests/pulse_auto_update_test.go
@@ -29,6 +29,28 @@ func TestAutoUpdateServiceDiscoveryAndInstallerHandoff(t *testing.T) {
}
}
+// Service-state admission is independent of the signed/metadata transaction
+// matrix below; no native service or published input is used by this fixture.
+func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v")
+ cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)
+ }
+}
+
+func extractAutoUpdateServiceFunctions(t *testing.T) string {
+ t.Helper()
+ var functions []string
+ for _, name := range []string{"read_update_service_state", "start_update_service_if_stopped", "stop_update_service_for_rollback", "wait_for_service_active", "ensure_service_restarted"} {
+ functions = append(functions, extractAutoUpdateFunction(t, name))
+ }
+ return strings.Join(functions, "\n")
+}
+
// TestIsPrereleaseTagRecognizesPrereleases asserts that any semver tag with
// a hyphen after the patch component is flagged as a prerelease. This is the
// pattern used for Pulse RCs (`-rc.N`), betas (`-beta.N`), alphas, nightlies,
@@ -142,6 +164,9 @@ INSTALL_DIR="/tmp/pulse-nonexistent-test-install"
log() { echo "[$1] $2"; }
detect_service_name() { echo pulse; }
get_current_version() { echo v5.1.27; }
+# Pure-function manager doubles stay in this shell; the Python fixture uses
+# real timeout with an external confined manager to test actual deadlines.
+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }
systemctl() { return 0; }
# If any of these get called, the guard has failed.
curl() { echo "FAIL: curl invoked during refused update"; exit 99; }
@@ -449,7 +474,7 @@ curl() {
if [[ -n "$out" ]]; then
case "$out" in
*.sig.*) printf 'dummy-signature\n' > "$out" ;;
- *) printf '#!/usr/bin/env bash\nexit 1\n' > "$out" ;;
+ *) printf '#!/usr/bin/env bash\necho inactive > "${PULSE_INSTALL_DIR}/service-state"\nexit 1\n' > "$out" ;;
esac
fi
return 0
@@ -458,10 +483,15 @@ curl() {
# Service is running at the was-active capture, then down (the installer
# stopped it before failing) until an explicit start/restart.
IS_ACTIVE_CALLS=0
+printf 'active\n' > "$INSTALL_DIR/service-state"
SERVICE_UP="no"
STARTS=0
+# Pure-function manager doubles stay in this shell; the Python fixture uses
+# real timeout with an external confined manager to test actual deadlines.
+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }
systemctl() {
case "$1" in
+ show) printf 'LoadState=loaded\nActiveState=%s\n' "$(cat "$INSTALL_DIR/service-state")"; return 0 ;;
is-active)
((IS_ACTIVE_CALLS += 1))
if (( IS_ACTIVE_CALLS == 1 )); then return 0; fi
@@ -470,16 +500,16 @@ systemctl() {
start|restart)
((STARTS += 1))
SERVICE_UP="yes"
+ printf 'active\n' > "$INSTALL_DIR/service-state"
return 0
;;
- stop) SERVICE_UP="no"; return 0 ;;
+ stop) SERVICE_UP="no"; printf 'inactive\n' > "$INSTALL_DIR/service-state"; return 0 ;;
esac
return 1
}
` + extractAutoUpdateFunction(t, "is_prerelease_tag") + `
` + extractAutoUpdateFunction(t, "resolve_install_script_url") + `
-` + extractAutoUpdateFunction(t, "wait_for_service_active") + `
-` + extractAutoUpdateFunction(t, "ensure_service_restarted") + `
+` + extractAutoUpdateServiceFunctions(t) + `
` + extractAutoUpdateFunction(t, "perform_update") + `
if perform_update v5.1.25; then
echo "RESULT:succeeded"
@@ -514,8 +544,12 @@ func TestEnsureServiceRestartedHonorsPriorServiceState(t *testing.T) {
set -uo pipefail
SERVICE_UP="no"
STARTS=0
+# Pure-function manager doubles stay in this shell; the Python fixture uses
+# real timeout with an external confined manager to test actual deadlines.
+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }
systemctl() {
case "$1" in
+ show) printf 'LoadState=loaded\n'; if [[ "$SERVICE_UP" == yes ]]; then echo ActiveState=active; else echo ActiveState=inactive; fi; return 0 ;;
is-active) [[ "$SERVICE_UP" == "yes" ]] && return 0 || return 1 ;;
start|restart) ((STARTS += 1)); SERVICE_UP="yes"; return 0 ;;
esac
@@ -523,8 +557,7 @@ systemctl() {
}
sleep() { :; }
log() { echo "[$1] ${*:2}"; }
-` + extractAutoUpdateFunction(t, "wait_for_service_active") + `
-` + extractAutoUpdateFunction(t, "ensure_service_restarted") + `
+` + extractAutoUpdateServiceFunctions(t) + `
ensure_service_restarted pulse false || echo "INACTIVE_PATH_FAILED"
echo "STARTS_AFTER_INACTIVE:$STARTS"
ensure_service_restarted pulse true || echo "ACTIVE_PATH_FAILED"
@@ -601,14 +634,16 @@ INSTALLER
}
# Pulse was running and stays running, so perform_update succeeds.
+# Pure-function manager doubles stay in this shell; the Python fixture uses
+# real timeout with an external confined manager to test actual deadlines.
+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }
systemctl() {
- if [[ "$1" == "is-active" ]]; then return 0; fi
+ if [[ "$1" == show ]]; then printf 'LoadState=loaded\nActiveState=active\n'; fi
return 0
}
` + extractAutoUpdateFunction(t, "is_prerelease_tag") + `
` + extractAutoUpdateFunction(t, "resolve_install_script_url") + `
-` + extractAutoUpdateFunction(t, "wait_for_service_active") + `
-` + extractAutoUpdateFunction(t, "ensure_service_restarted") + `
+` + extractAutoUpdateServiceFunctions(t) + `
` + extractAutoUpdateFunction(t, "perform_update") + `
if perform_update v5.1.25; then echo "RESULT:succeeded"; else echo "RESULT:failed"; fi
echo "LEAKED:$(trap -p RETURN)"
@@ -768,6 +803,7 @@ cat > "$FIXTURE_DIR/installer" <<'INSTALLER'
set -euo pipefail
printf 'executed\n' > "$FIXTURE_DIR/installer-executed"
printf 'false\n' > "$FIXTURE_DIR/active"
+[[ "$FAULT" != stop ]] || printf 'true\n' > "$FIXTURE_DIR/active"
mkdir -p "$PULSE_INSTALL_DIR/bin"
new_version=v6.5.0
[[ "$OUTCOME" != mismatch ]] || new_version=v6.4.6
@@ -873,8 +909,12 @@ curl() {
command cp "$FIXTURE_DIR/installer" "$out"
fi
}
+# Pure-function manager doubles stay in this shell; the Python fixture uses
+# real timeout with an external confined manager to test actual deadlines.
+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }
systemctl() {
case "$1" in
+ show) printf 'LoadState=loaded\n'; if [[ "$(cat "$FIXTURE_DIR/active")" == true ]]; then echo ActiveState=active; else echo ActiveState=inactive; fi ;;
is-active) [[ "$(cat "$FIXTURE_DIR/active")" == true ]] ;;
stop)
printf 'stop\n' >> "$FIXTURE_DIR/service-operations"
diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.sh
index a166d4e6d4..e41683d968 100755
--- a/scripts/pulse-auto-update.sh
+++ b/scripts/pulse-auto-update.sh
@@ -365,14 +365,74 @@ resolve_install_script_url() {
printf 'https://github.com/%s/releases/download/%s/install.sh\n' "$GITHUB_REPO" "$target_version"
}
-# Perform the update
+# Read a complete, bounded manager observation. An is-active failure conflates
+# an inactive unit with an unavailable manager; neither missing text nor a
+# plausible partial result grants an update, restart or rollback.
+read_update_service_state() {
+ local service_name=$1 observation="" line="" load="" state=""
+ local seen_load=false seen_state=false
+ if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState 2>/dev/null); then
+ log error "Cannot read Pulse service state ($service_name); reconcile service state before retrying" >&2
+ return 1
+ fi
+ while IFS= read -r line; do
+ case "$line" in
+ LoadState=*)
+ [[ "$seen_load" == false ]] || { log error "Duplicate Pulse service load state ($service_name)" >&2; return 1; }
+ seen_load=true; load=${line#LoadState=} ;;
+ ActiveState=*)
+ [[ "$seen_state" == false ]] || { log error "Duplicate Pulse service active state ($service_name)" >&2; return 1; }
+ seen_state=true; state=${line#ActiveState=} ;;
+ *)
+ log error "Incomplete Pulse service observation ($service_name); no service action is safe" >&2
+ return 1 ;;
+ esac
+ done <<< "$observation"
+ if [[ "$seen_load" != true || "$seen_state" != true || ( "$load" != loaded && "$load" != masked ) ]]; then
+ log error "Pulse service is unavailable ($service_name); no service action is safe" >&2
+ return 1
+ fi
+ case "$state" in
+ active|inactive|failed|activating|deactivating|reloading|refreshing|maintenance) printf '%s\n' "$state" ;;
+ *) log error "Unknown Pulse service state ($service_name); no service action is safe" >&2; return 1 ;;
+ esac
+}
+
+# Only a fresh, settled observation permits an explicit start. Return 2 for
+# uncertainty so callers do not turn an unavailable read into rollback consent.
+start_update_service_if_stopped() {
+ local service_name=$1 state=""
+ state=$(read_update_service_state "$service_name") || return 2
+ case "$state" in
+ active) return 0 ;;
+ inactive|failed) timeout -k 1 5 systemctl start "$service_name" 2>/dev/null ;;
+ *) log error "Pulse service state is not settled ($service_name); refusing start"; return 2 ;;
+ esac
+}
+
+stop_update_service_for_rollback() {
+ local service_name=$1 state=""
+ state=$(read_update_service_state "$service_name") || return 1
+ case "$state" in
+ inactive) return 0 ;;
+ active|failed) ;;
+ *) log error "Pulse service state is not settled ($service_name); refusing rollback"; return 1 ;;
+ esac
+ if ! timeout -k 1 5 systemctl stop "$service_name" 2>/dev/null; then
+ return 1
+ fi
+ state=$(read_update_service_state "$service_name") || return 1
+ [[ "$state" == inactive ]]
+}
+
wait_for_service_active() {
local service_name=$1
local timeout_seconds="${2:-20}"
- local elapsed=0
+ local elapsed=0 state=""
while (( elapsed < timeout_seconds )); do
- if systemctl is-active --quiet "$service_name" 2>/dev/null; then
+ state=$(read_update_service_state "$service_name") || return 2
+ if [[ "$state" == active ]]; then
return 0
fi
sleep 1
@@ -398,16 +458,10 @@ ensure_service_restarted() {
if [[ "$service_was_active" != "true" ]]; then
return 0
fi
- if systemctl is-active --quiet "$service_name" 2>/dev/null; then
- return 0
- fi
-
- log warn "Pulse service is not active after update attempt; starting it"
- systemctl start "$service_name" 2>/dev/null || true
- if wait_for_service_active "$service_name" 20; then
+ if start_update_service_if_stopped "$service_name" && wait_for_service_active "$service_name" 20; then
log info "Pulse service is running again"
else
- log error "Pulse service could not be started after update attempt; manual intervention required (systemctl start $service_name)"
+ log error "Pulse service restart not confirmed ($service_name); reconcile service state before manual recovery"
fi
return 0
}
@@ -418,6 +472,11 @@ perform_update() {
# A local declaration hides command-substitution failure. Refuse before
# prior-active observation, backup, installer download or restart traps.
service_name=$(detect_service_name) || return 1
+ # Never cross the stable channel even if the caller selected a preview.
+ if is_prerelease_tag "$new_version"; then
+ log error "Refusing to install prerelease version $new_version via unattended updater"
+ return 1
+ fi
local installer_tmp=""
local signature_tmp=""
local backup_dir=""
@@ -432,10 +491,13 @@ perform_update() {
# Capture whether Pulse was running before the update so we can guarantee it
# comes back up afterwards (#1323: auto-update could leave it stopped on
# unprivileged LXC where the installer's restart silently fails).
- local service_was_active="false"
- if systemctl is-active --quiet "$service_name" 2>/dev/null; then
- service_was_active="true"
- fi
+ local service_was_active="false" service_state="" service_result=0
+ service_state=$(read_update_service_state "$service_name") || return 1
+ case "$service_state" in
+ active) service_was_active="true" ;;
+ inactive|failed) ;;
+ *) log error "Pulse service state is not settled ($service_name); refusing update"; return 1 ;;
+ esac
# Keep the prior-active restart backstop (#1630), except when incomplete
# recovery requires manual intervention. The trap disarms itself on the
@@ -445,14 +507,6 @@ perform_update() {
# that aborted the updater under set -u after a successful update (#2128).
trap 'trap - RETURN; rm -f -- "${installer_tmp:-}" "${signature_tmp:-}" "${restore_files[@]}"; if [[ -n "$backup_dir" && "$retain_backup" == "false" ]]; then rm -rf -- "$backup_dir"; fi; if [[ "$restart_allowed" == "true" ]]; then ensure_service_restarted "$service_name" "$service_was_active"; fi' RETURN
- # Refuse to install a prerelease via the unattended updater. The stable
- # channel must never cross onto a tag like v6.0.0-rc.2, even if every
- # caller above this point thought it was safe.
- if is_prerelease_tag "$new_version"; then
- log error "Refusing to install prerelease version $new_version via unattended updater"
- return 1
- fi
-
log info "Starting update to $new_version"
# main calls this function in an if condition, which disables errexit
@@ -541,18 +595,38 @@ perform_update() {
# again. On unprivileged LXC the installer's restart can silently
# fail, leaving Pulse stopped (#1323).
if [[ "$service_was_active" == "true" ]]; then
- if ! wait_for_service_active "$service_name" 20; then
+ service_result=0
+ wait_for_service_active "$service_name" 20 || service_result=$?
+ if [[ "$service_result" == 1 ]]; then
log warn "Pulse service is not active after update, attempting one explicit start"
- systemctl start "$service_name" || true
+ start_update_service_if_stopped "$service_name" || service_result=$?
fi
-
- if ! wait_for_service_active "$service_name" 20; then
+ if [[ "$service_result" != 2 ]]; then
+ service_result=0
+ wait_for_service_active "$service_name" 20 || service_result=$?
+ fi
+ if [[ "$service_result" == 2 ]]; then
+ retain_backup="true"
+ restart_allowed="false"
+ log error "Update service state is uncertain; backup retained at $backup_dir; reconcile service state before recovery"
+ return 1
+ elif [[ "$service_result" != 0 ]]; then
log error "Pulse service did not come back up after update"
else
update_accepted="true"
fi
else
- update_accepted="true"
+ # Inactive/failed prior state removes the restart obligation,
+ # not the obligation to observe the installed unit safely.
+ service_state=$(read_update_service_state "$service_name") || service_state="unknown"
+ case "$service_state" in
+ active|inactive|failed) update_accepted="true" ;;
+ *)
+ retain_backup="true"
+ restart_allowed="false"
+ log error "Update service state is uncertain; backup retained at $backup_dir; reconcile service state before recovery"
+ return 1 ;;
+ esac
fi
else
log error "Version mismatch after update. Expected: $new_version, Got: $installed_version"
@@ -572,8 +646,8 @@ perform_update() {
retain_backup="true"
restart_allowed="false"
log info "Restoring from backup"
- if ! systemctl stop "$service_name"; then
- log error "Could not stop Pulse for rollback; backup retained at $backup_dir; manual recovery required"
+ if ! stop_update_service_for_rollback "$service_name"; then
+ log error "Could not confirm Pulse stopped for rollback; backup retained at $backup_dir; manual recovery required"
return 1
fi
@@ -606,7 +680,7 @@ perform_update() {
done
if [[ "$service_was_active" == "true" ]]; then
- if ! systemctl start "$service_name" || ! wait_for_service_active "$service_name" 20; then
+ if ! start_update_service_if_stopped "$service_name" || ! wait_for_service_active "$service_name" 20; then
log error "Restored Pulse could not be started; backup retained at $backup_dir; manual recovery required"
return 1
fi
diff --git a/scripts/tests/test-pulse-auto-update.sh b/scripts/tests/test-pulse-auto-update.sh
index c007302886..bd1819f925 100755
--- a/scripts/tests/test-pulse-auto-update.sh
+++ b/scripts/tests/test-pulse-auto-update.sh
@@ -16,6 +16,10 @@ fi
# shellcheck disable=SC1090
source "${AUTO_UPDATE_SCRIPT}"
+# Command doubles run in this shell, not through the host manager. Actual
+# timeout/manager boundaries are exercised by test_auto_update_service_state.py.
+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }
+
failures=0
assert_success() {
@@ -32,33 +36,28 @@ assert_success() {
}
test_wait_for_service_active_succeeds_after_retry() {
- local calls=0
-
+ local counter; counter=$(mktemp)
+ printf '0\n' > "$counter"
systemctl() {
- if [[ "$1" == "is-active" ]]; then
- ((calls += 1))
- if (( calls >= 3 )); then
- return 0
- fi
- return 1
- fi
- return 1
+ [[ "$1" == show ]] || return 1
+ local count; count=$(cat "$counter"); ((count += 1))
+ printf '%s\n' "$count" > "$counter"
+ printf 'LoadState=loaded\n'
+ if (( count >= 3 )); then echo ActiveState=active; else echo ActiveState=activating; fi
}
-
sleep() { :; }
-
- wait_for_service_active pulse 5
+ local result=0
+ wait_for_service_active pulse 5 || result=$?
+ command rm "$counter"
+ return "$result"
}
test_wait_for_service_active_times_out_when_never_active() {
- systemctl() { return 1; }
+ systemctl() { [[ "$1" == show ]] || return 1; printf 'LoadState=loaded\nActiveState=inactive\n'; }
sleep() { :; }
-
- if wait_for_service_active pulse 3; then
- echo "expected wait_for_service_active to fail when service never becomes active" >&2
- return 1
- fi
- return 0
+ local result=0
+ wait_for_service_active pulse 3 || result=$?
+ [[ "$result" == 1 ]] || { echo "expected settled inactive timeout, got $result" >&2; return 1; }
}
test_pick_highest_stable_tag_ignores_list_order_and_prereleases() {
@@ -156,6 +155,7 @@ echo "v5.1.24"
EOF
chmod +x "${INSTALL_DIR}/bin/pulse"
+ printf 'active\n' > "$INSTALL_DIR/service-state"
export INSTALL_DIR
export FAKE_NEW_VERSION="v5.1.25"
@@ -183,6 +183,7 @@ EOF
cat > "$out" <<'INSTALLER'
#!/usr/bin/env bash
printf '%s\n' "${FAKE_NEW_VERSION}" > "${PULSE_INSTALL_DIR}/VERSION"
+echo inactive > "${PULSE_INSTALL_DIR}/service-state"
exit 0
INSTALLER
;;
@@ -191,21 +192,13 @@ INSTALLER
return 0
}
- # Service was running before the update (first is-active call true), then
- # never comes back up; start also fails -> perform_update must restore + fail.
- local is_active_calls=0
+ # The installer records its stop; explicit starts fail for this scenario.
systemctl() {
- # The rollback must stop the service before replacing its executable.
- # Stopping succeeds; only activation/liveness is the injected failure.
- if [[ "$1" == "stop" ]]; then return 0; fi
- if [[ "$1" == "is-active" ]]; then
- ((is_active_calls += 1))
- if (( is_active_calls == 1 )); then
- return 0
- fi
- return 1
- fi
- return 1
+ case "$1" in
+ show) printf 'LoadState=loaded\nActiveState=%s\n' "$(cat "$INSTALL_DIR/service-state")"; return 0 ;;
+ stop) echo inactive > "$INSTALL_DIR/service-state" ;;
+ *) return 1 ;;
+ esac
}
sleep() { :; }
@@ -250,6 +243,11 @@ test_ensure_service_restarted_starts_stopped_service() {
ENSURE_TEST_UP="no"
systemctl() {
case "$1" in
+ show)
+ printf 'LoadState=loaded\n'
+ if [[ "$ENSURE_TEST_UP" == yes ]]; then echo ActiveState=active; else echo ActiveState=inactive; fi
+ return 0
+ ;;
is-active)
[[ "$ENSURE_TEST_UP" == "yes" ]] && return 0 || return 1
;;
@@ -300,6 +298,7 @@ echo "v5.1.24"
EOF
chmod +x "${INSTALL_DIR}/bin/pulse"
+ printf 'active\n' > "$INSTALL_DIR/service-state"
export INSTALL_DIR
is_prerelease_tag() { return 1; }
@@ -324,6 +323,7 @@ EOF
*)
cat > "$out" <<'INSTALLER'
#!/usr/bin/env bash
+echo inactive > "${PULSE_INSTALL_DIR}/service-state"
exit 1
INSTALLER
;;
@@ -339,6 +339,7 @@ INSTALLER
AUTOUPDATE_TEST_STARTS=0
systemctl() {
case "$1" in
+ show) printf 'LoadState=loaded\nActiveState=%s\n' "$(cat "$INSTALL_DIR/service-state")"; return 0 ;;
is-active)
((AUTOUPDATE_TEST_IS_ACTIVE_CALLS += 1))
if (( AUTOUPDATE_TEST_IS_ACTIVE_CALLS == 1 )); then
@@ -349,9 +350,10 @@ INSTALLER
start|restart)
((AUTOUPDATE_TEST_STARTS += 1))
AUTOUPDATE_TEST_UP="yes"
+ echo active > "$INSTALL_DIR/service-state"
return 0
;;
- stop) AUTOUPDATE_TEST_UP="no"; return 0 ;;
+ stop) AUTOUPDATE_TEST_UP="no"; echo inactive > "$INSTALL_DIR/service-state"; return 0 ;;
esac
return 1
}
@@ -436,7 +438,7 @@ INSTALLER
# Pulse was running and stays running, so perform_update succeeds.
systemctl() {
- if [[ "$1" == "is-active" ]]; then return 0; fi
+ if [[ "$1" == show ]]; then printf 'LoadState=loaded\nActiveState=active\n'; fi
return 0
}
sleep() { :; }
diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.py
new file mode 100644
index 0000000000..7950d7d2c5
--- /dev/null
+++ b/scripts/tests/test_auto_update_service_state.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+"""Exercise the real updater with private files and a confined manager double.
+
+No host manager, network, application build or published installer is used.
+The versioned executable and signature verifier are synthetic; existing Go
+transaction tests separately exercise real local SSH signatures and metadata.
+"""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import time
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+UPDATER = Path(os.environ.get("PULSE_AUTO_UPDATER_UNDER_TEST", ROOT / "scripts/pulse-auto-update.sh")).resolve()
+
+MANAGER = r'''#!/usr/bin/env python3
+import json, os, sys, time
+from pathlib import Path
+f = Path(os.environ["FIXTURE"])
+args = sys.argv[1:]
+with (f / "calls").open("a") as out:
+ out.write(json.dumps(args) + "\n")
+mode = os.environ["MODE"]
+after = (f / "installer-ran").exists()
+stopped = (f / "stop-ran").exists()
+state = (f / "state").read_text().strip()
+if args[0] == "show":
+ if not after and mode == "initial-hang": time.sleep(30)
+ if not after and mode == "initial-error":
+ print("LoadState=loaded\nActiveState=inactive")
+ sys.exit(1)
+ if stopped and mode == "readback-error": sys.exit(1)
+ if stopped and mode == "readback-empty": sys.exit(0)
+ if after and not stopped and mode == "after-error": sys.exit(1)
+ if after and mode == "success-error": sys.exit(1)
+ if not after and mode == "initial-empty": sys.exit(0)
+ if not after and mode == "initial-duplicate":
+ print("LoadState=loaded\nActiveState=active\nActiveState=inactive")
+ sys.exit(0)
+ if not after and mode == "initial-extra":
+ print("LoadState=loaded\nActiveState=inactive\nUnknown=active")
+ sys.exit(0)
+ # Reverse property order deliberately: systemd does not promise ordering.
+ print("ActiveState=" + state)
+ print("LoadState=" + os.environ["LOAD_STATE"])
+ sys.exit(0)
+if args[0] == "is-active":
+ sys.exit(0 if state == "active" and not mode.startswith("initial-") else 1)
+if args[0] == "stop":
+ (f / "stop-ran").write_text("attempted\n")
+ if mode == "stop-hang": time.sleep(30)
+ if mode == "stop-error": sys.exit(1)
+ if mode != "stop-noop": (f / "state").write_text("inactive\n")
+ sys.exit(0)
+if args[0] == "start":
+ (f / "start-ran").write_text("attempted\n")
+ (f / "state").write_text("active\n")
+ sys.exit(0)
+sys.exit(97)
+'''
+
+HARNESS = r'''
+source "$UPDATER"
+log() { printf '[%s] %s\n' "$1" "${*:2}"; }
+detect_service_name() { printf '%s\n' "$SERVICE_NAME"; }
+sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.
+verify_release_signature() { :; }
+mktemp() {
+ local -a args=("$@")
+ local last=$((${#args[@]} - 1))
+ if [[ "${args[last]}" == /tmp/* ]]; then
+ args[last]="$FIXTURE/tmp/${args[last]##*/}"
+ fi
+ printf "%s\n" "${args[*]}" >> "$FIXTURE/temp-requests"
+ command mktemp "${args[@]}"
+}
+curl() {
+ local out="" url=""
+ while (( $# )); do
+ case "$1" in -o) out="$2"; shift ;; https://*) url="$1" ;; esac
+ shift
+ done
+ printf '%s\n' "$url" >> "$FIXTURE/downloads"
+ if [[ "$url" == *.sshsig ]]; then printf 'fixture signature\n' > "$out"
+ else command cp "$FIXTURE/installer" "$out"; fi
+}
+if [[ "$FLOW" == backstop ]]; then
+ ensure_service_restarted "$SERVICE_NAME" true
+ status=0
+else
+ # Production main uses a conditional call, disabling Bash errexit.
+ status=0
+ perform_update v6.6.0 || status=$?
+fi
+printf 'STATUS=%s\n' "$status"
+[[ -z "$(trap -p RETURN)" ]] || { echo 'RETURN trap leaked'; exit 98; }
+caller() { :; }; caller
+exit "$status"
+'''
+
+
+class UpdateServiceState(unittest.TestCase):
+ def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update"):
+ with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:
+ f = Path(directory)
+ for name in ("tools", "tmp", "install/bin", "config"):
+ (f / name).mkdir(parents=True)
+ (f / "tools/systemctl").write_text(MANAGER)
+ (f / "tools/systemctl").chmod(0o755)
+ old = b"#!/bin/sh\necho 'Pulse v6.5.0'\n"
+ new = b"#!/bin/sh\necho 'Pulse v6.6.0'\n"
+ (f / "install/bin/pulse").write_bytes(old)
+ (f / "install/bin/pulse").chmod(0o751)
+ (f / "install/VERSION").write_text("v6.5.0\n")
+ (f / "config/system.json").write_text('operator settings\n')
+ (f / "config/agent-id").write_text('original identity\n')
+ (f / "state").write_text(state + "\n")
+ (f / "new-pulse").write_bytes(new)
+ (f / "installer").write_text(
+ 'set -eu\n'
+ 'echo attempted > "$FIXTURE/installer-ran"\n'
+ 'cp "$FIXTURE/new-pulse" "$PULSE_INSTALL_DIR/bin/pulse"\n'
+ 'echo v6.6.0 > "$PULSE_INSTALL_DIR/VERSION"\n'
+ 'echo "$INSTALLER_STATE" > "$FIXTURE/state"\n'
+ 'exit "$INSTALLER_EXIT"\n')
+ env = dict(os.environ)
+ env.update(FIXTURE=str(f), UPDATER=str(UPDATER), MODE=mode, FLOW=flow,
+ PULSE_INSTALL_DIR=str(f / "install"), PULSE_CONFIG_DIR=str(f / "config"),
+ PULSE_SERVICE_NAME="pulse-custom@department", LOAD_STATE=load,
+ INSTALLER_STATE="active" if mode in ("stop-noop", "stop-error", "stop-hang", "after-error", "readback-error", "readback-empty") else "inactive",
+ INSTALLER_EXIT="0" if success else "23",
+ PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])
+ start = time.monotonic()
+ result = subprocess.run(["bash", "-c", HARNESS], env=env, capture_output=True, timeout=15)
+ calls = [json.loads(line) for line in (f / "calls").read_text().splitlines()] if (f / "calls").exists() else []
+ backups = list((f / "tmp").glob("pulse-backup.*"))
+ observation = dict(exit=result.returncode, output=result.stdout.decode(), error=result.stderr.decode(),
+ elapsed=time.monotonic() - start, calls=calls,
+ downloaded=(f / "downloads").exists(), temporary=(f / "temp-requests").exists(),
+ installed=(f / "installer-ran").exists(), stopped=(f / "stop-ran").exists(),
+ started=(f / "start-ran").exists(), state=(f / "state").read_text().strip(),
+ binary=(f / "install/bin/pulse").read_bytes(), version=(f / "install/VERSION").read_text(),
+ backups=[(b / "pulse-bin").read_bytes() for b in backups],
+ settings=(f / "config/system.json").read_text(), identity=(f / "config/agent-id").read_text())
+ self.assertEqual(observation["settings"], "operator settings\n")
+ self.assertEqual(observation["identity"], "original identity\n")
+ self.assertNotIn("trap leaked", observation["output"])
+ for args in calls:
+ # All read/mutation calls target the exact explicit service.
+ self.assertIn("pulse-custom@department", args)
+ return observation
+
+ def test_unknown_initial_state_refuses_before_backup_download_or_installer(self):
+ cases = [(m, "active", "loaded") for m in
+ ("initial-error", "initial-empty", "initial-duplicate", "initial-extra")]
+ cases += [("normal", s, "loaded") for s in
+ ("activating", "deactivating", "reloading", "refreshing", "maintenance", "future-state")]
+ cases += [("normal", "inactive", s) for s in ("not-found", "error", "bad-setting", "future-load")]
+ for mode, state, load in cases:
+ with self.subTest(mode=mode, state=state, load=load):
+ r = self.run_fixture(mode=mode, state=state, load=load, success=True)
+ self.assertNotEqual(r["exit"], 0, r)
+ self.assertFalse(r["installed"] or r["downloaded"] or r["temporary"], r)
+ self.assertFalse(r["started"] or r["stopped"], r)
+ self.assertEqual(r["backups"], [], r)
+ self.assertIn(b"v6.5.0", r["binary"])
+
+ def test_initial_observation_timeout_is_bounded_without_mutation(self):
+ r = self.run_fixture(mode="initial-hang", success=True)
+ self.assertNotEqual(r["exit"], 0, r)
+ self.assertFalse(r["installed"] or r["started"] or r["stopped"], r)
+ self.assertLess(r["elapsed"], 9, r)
+
+ def test_failed_installer_restores_bytes_and_only_prior_active_service(self):
+ for state, load in (("active", "loaded"), ("inactive", "loaded"), ("failed", "loaded"), ("inactive", "masked")):
+ with self.subTest(state=state, load=load):
+ r = self.run_fixture(state=state, load=load)
+ self.assertNotEqual(r["exit"], 0, r)
+ self.assertTrue(r["installed"], r)
+ self.assertIn(b"v6.5.0", r["binary"])
+ self.assertEqual(r["version"], "v6.5.0\n", r)
+ self.assertEqual(r["started"], state == "active", r)
+ self.assertEqual(r["backups"], [], r)
+
+ def test_success_preserves_prior_active_backstop_without_unwanted_start(self):
+ for state in ("active", "inactive"):
+ with self.subTest(state=state):
+ r = self.run_fixture(state=state, success=True)
+ self.assertEqual(r["exit"], 0, r)
+ self.assertIn(b"v6.6.0", r["binary"])
+ self.assertEqual(r["started"], state == "active", r)
+ self.assertFalse(r["stopped"], r)
+ self.assertEqual(r["backups"], [], r)
+
+ def test_uncertain_rollback_stop_retains_backup_without_replacement_or_start(self):
+ for mode in ("stop-noop", "stop-error", "readback-error", "readback-empty", "after-error"):
+ with self.subTest(mode=mode):
+ r = self.run_fixture(mode=mode)
+ self.assertNotEqual(r["exit"], 0, r)
+ self.assertTrue(r["installed"], r)
+ self.assertIn(b"v6.6.0", r["binary"], r)
+ self.assertEqual(r["version"], "v6.6.0\n", r)
+ self.assertEqual(len(r["backups"]), 1, r)
+ self.assertIn(b"v6.5.0", r["backups"][0], r)
+ self.assertFalse(r["started"], r)
+ if mode == "after-error": self.assertFalse(r["stopped"], r)
+
+ def test_rollback_stop_timeout_is_bounded_and_retains_recovery(self):
+ r = self.run_fixture(mode="stop-hang")
+ self.assertNotEqual(r["exit"], 0, r)
+ self.assertIn(b"v6.6.0", r["binary"], r)
+ self.assertEqual(len(r["backups"]), 1, r)
+ self.assertFalse(r["started"], r)
+ self.assertLess(r["elapsed"], 9, r)
+
+ def test_post_install_observation_error_is_not_a_restart_or_rollback_grant(self):
+ for state in ("active", "inactive"):
+ with self.subTest(state=state):
+ r = self.run_fixture(mode="success-error", state=state, success=True)
+ self.assertNotEqual(r["exit"], 0, r)
+ self.assertTrue(r["installed"], r)
+ self.assertFalse(r["started"] or r["stopped"], r)
+ self.assertIn(b"v6.6.0", r["binary"])
+ self.assertEqual(len(r["backups"]), 1, r)
+
+ def test_backstop_never_starts_unknown_or_unsettled_state(self):
+ for mode, state in (("initial-error", "inactive"), ("initial-empty", "inactive"),
+ ("normal", "activating"), ("normal", "deactivating")):
+ with self.subTest(mode=mode, state=state):
+ r = self.run_fixture(mode=mode, state=state, flow="backstop")
+ self.assertEqual(r["exit"], 0, r) # A RETURN trap must not mask its caller's exit.
+ self.assertFalse(r["started"] or r["stopped"] or r["installed"], r)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/tests/test_server_service_discovery.py b/scripts/tests/test_server_service_discovery.py
index e5e0d4bc5b..86371e9fa7 100644
--- a/scripts/tests/test_server_service_discovery.py
+++ b/scripts/tests/test_server_service_discovery.py
@@ -19,8 +19,8 @@ import unittest
ROOT = Path(__file__).resolve().parents[2]
-INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh"))
-UPDATER = Path(os.environ.get("PULSE_AUTO_UPDATER_UNDER_TEST", ROOT / "scripts/pulse-auto-update.sh"))
+INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh")).resolve()
+UPDATER = Path(os.environ.get("PULSE_AUTO_UPDATER_UNDER_TEST", ROOT / "scripts/pulse-auto-update.sh")).resolve()
REAL_SYSTEMCTL = shutil.which("systemctl")
QUERY = ["list-unit-files", "--no-legend", "--no-pager", "--full", "--",
"pulse-backend.service", "pulse.service"]
@@ -55,7 +55,11 @@ if args[0] == "list-unit-files":
os.write(1, os.environ["ROWS"].encode())
sys.exit(int(os.environ["QUERY_EXIT"]))
if args[0] == "show":
- print("inactive" if (f / "stopped").exists() else "active")
+ state = "inactive" if (f / "stopped").exists() else "active"
+ if "--value" in args:
+ print(state)
+ else:
+ print("LoadState=loaded\nActiveState=" + state)
sys.exit(0)
if args[0] == "is-active":
sys.exit(1 if (f / "stopped").exists() else 0)