Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Refuse unattended recovery on uncertain systemd service state
pulse-triage[bot]9060dbd54de3
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 1c4a607d02..d181a2898e 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -52,6 +52,37 @@ execute the actual unattended helper's selection and`perform_update` entry/handoff with a fake installer, including timeout andfailed/partial inventory before any service observation or installer call.+### Unattended restoration requires observed service state++Before backup, download, installer invocation or restart-trap setup, the+standalone helper requires one complete successful observation of the exact+selected unit's LoadState and ActiveState. Loaded and masked units are admitted+only in settled active, inactive or failed states; missing, partial, duplicate,+failed, timed-out, unavailable or transitional observations refuse the update.+A failed is-active call is no longer treated as evidence of prior inactivity.+Discovery, explicit identities, masks, consent and stable/signature admission+retain their existing contracts.++After an attempted installation, rollback replacement requires an observed+inactive unit, or a bounded successful stop followed by a fresh inactive+readback. Successful stop submission alone does not permit file replacement.+Unknown/unsettled state retains the verified private backup, preserves the+current destination bytes, suppresses restart and reports manual reconciliation.+A post-install observation failure is not permission to start or roll back.+The prior-active backstop starts only a freshly observed stopped/failed unit;+it never starts a unit on a failed read. Restoration still stages and compares+all saved files, uses atomic destination replacement and restarts only when+prior activity was established. Incomplete restoration/start leaves the backup+and fails; no new automatic retry, unmasking or config/data rollback is added.++`TestAutoUpdateServiceStateFailsClosed` executes the production helper with+private versioned file fixtures and an external confined manager, including+real query/stop deadlines, successful/no-op/refused stops, failed readback,+prior-state restoration and late unknown state. The existing signed transaction+matrix and smoke controls retain their cryptographic, metadata and return-trap+coverage. These are source/fixture controls, not native systemd/LXC recovery,+published installer trust, #2785's stop cause or installed acceptance.+### Core E2E images are built once and admitted in the same runThe secret-free Core E2E workflow builds its existing `e2e_runtime` server anddiff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.goindex 3401b84de4..484c97bf64 100644--- a/scripts/installtests/pulse_auto_update_test.go+++ b/scripts/installtests/pulse_auto_update_test.go@@ -29,6 +29,28 @@ func TestAutoUpdateServiceDiscoveryAndInstallerHandoff(t *testing.T) {}}+// Service-state admission is independent of the signed/metadata transaction+// matrix below; no native service or published input is used by this fixture.+func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v")+ cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))+ out, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)+ }+}++func extractAutoUpdateServiceFunctions(t *testing.T) string {+ t.Helper()+ var functions []string+ for _, name := range []string{"read_update_service_state", "start_update_service_if_stopped", "stop_update_service_for_rollback", "wait_for_service_active", "ensure_service_restarted"} {+ functions = append(functions, extractAutoUpdateFunction(t, name))+ }+ return strings.Join(functions, "\n")+}+// TestIsPrereleaseTagRecognizesPrereleases asserts that any semver tag with// a hyphen after the patch component is flagged as a prerelease. This is the// pattern used for Pulse RCs (`-rc.N`), betas (`-beta.N`), alphas, nightlies,@@ -142,6 +164,9 @@ INSTALL_DIR="/tmp/pulse-nonexistent-test-install"log() { echo "[$1] $2"; }detect_service_name() { echo pulse; }get_current_version() { echo v5.1.27; }+# Pure-function manager doubles stay in this shell; the Python fixture uses+# real timeout with an external confined manager to test actual deadlines.+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }systemctl() { return 0; }# If any of these get called, the guard has failed.curl() { echo "FAIL: curl invoked during refused update"; exit 99; }@@ -449,7 +474,7 @@ curl() {if [[ -n "$out" ]]; thencase "$out" in*.sig.*) printf 'dummy-signature\n' > "$out" ;;- *) printf '#!/usr/bin/env bash\nexit 1\n' > "$out" ;;+ *) printf '#!/usr/bin/env bash\necho inactive > "${PULSE_INSTALL_DIR}/service-state"\nexit 1\n' > "$out" ;;esacfireturn 0@@ -458,10 +483,15 @@ curl() {# Service is running at the was-active capture, then down (the installer# stopped it before failing) until an explicit start/restart.IS_ACTIVE_CALLS=0+printf 'active\n' > "$INSTALL_DIR/service-state"SERVICE_UP="no"STARTS=0+# Pure-function manager doubles stay in this shell; the Python fixture uses+# real timeout with an external confined manager to test actual deadlines.+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }systemctl() {case "$1" in+ show) printf 'LoadState=loaded\nActiveState=%s\n' "$(cat "$INSTALL_DIR/service-state")"; return 0 ;;is-active)((IS_ACTIVE_CALLS += 1))if (( IS_ACTIVE_CALLS == 1 )); then return 0; fi@@ -470,16 +500,16 @@ systemctl() {start|restart)((STARTS += 1))SERVICE_UP="yes"+ printf 'active\n' > "$INSTALL_DIR/service-state"return 0;;- stop) SERVICE_UP="no"; return 0 ;;+ stop) SERVICE_UP="no"; printf 'inactive\n' > "$INSTALL_DIR/service-state"; return 0 ;;esacreturn 1}` + extractAutoUpdateFunction(t, "is_prerelease_tag") + `` + extractAutoUpdateFunction(t, "resolve_install_script_url") + `-` + extractAutoUpdateFunction(t, "wait_for_service_active") + `-` + extractAutoUpdateFunction(t, "ensure_service_restarted") + `+` + extractAutoUpdateServiceFunctions(t) + `` + extractAutoUpdateFunction(t, "perform_update") + `if perform_update v5.1.25; thenecho "RESULT:succeeded"@@ -514,8 +544,12 @@ func TestEnsureServiceRestartedHonorsPriorServiceState(t *testing.T) {set -uo pipefailSERVICE_UP="no"STARTS=0+# Pure-function manager doubles stay in this shell; the Python fixture uses+# real timeout with an external confined manager to test actual deadlines.+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }systemctl() {case "$1" in+ show) printf 'LoadState=loaded\n'; if [[ "$SERVICE_UP" == yes ]]; then echo ActiveState=active; else echo ActiveState=inactive; fi; return 0 ;;is-active) [[ "$SERVICE_UP" == "yes" ]] && return 0 || return 1 ;;start|restart) ((STARTS += 1)); SERVICE_UP="yes"; return 0 ;;esac@@ -523,8 +557,7 @@ systemctl() {}sleep() { :; }log() { echo "[$1] ${*:2}"; }-` + extractAutoUpdateFunction(t, "wait_for_service_active") + `-` + extractAutoUpdateFunction(t, "ensure_service_restarted") + `+` + extractAutoUpdateServiceFunctions(t) + `ensure_service_restarted pulse false || echo "INACTIVE_PATH_FAILED"echo "STARTS_AFTER_INACTIVE:$STARTS"ensure_service_restarted pulse true || echo "ACTIVE_PATH_FAILED"@@ -601,14 +634,16 @@ INSTALLER}# Pulse was running and stays running, so perform_update succeeds.+# Pure-function manager doubles stay in this shell; the Python fixture uses+# real timeout with an external confined manager to test actual deadlines.+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }systemctl() {- if [[ "$1" == "is-active" ]]; then return 0; fi+ if [[ "$1" == show ]]; then printf 'LoadState=loaded\nActiveState=active\n'; fireturn 0}` + extractAutoUpdateFunction(t, "is_prerelease_tag") + `` + extractAutoUpdateFunction(t, "resolve_install_script_url") + `-` + extractAutoUpdateFunction(t, "wait_for_service_active") + `-` + extractAutoUpdateFunction(t, "ensure_service_restarted") + `+` + extractAutoUpdateServiceFunctions(t) + `` + extractAutoUpdateFunction(t, "perform_update") + `if perform_update v5.1.25; then echo "RESULT:succeeded"; else echo "RESULT:failed"; fiecho "LEAKED:$(trap -p RETURN)"@@ -768,6 +803,7 @@ cat > "$FIXTURE_DIR/installer" <<'INSTALLER'set -euo pipefailprintf 'executed\n' > "$FIXTURE_DIR/installer-executed"printf 'false\n' > "$FIXTURE_DIR/active"+[[ "$FAULT" != stop ]] || printf 'true\n' > "$FIXTURE_DIR/active"mkdir -p "$PULSE_INSTALL_DIR/bin"new_version=v6.5.0[[ "$OUTCOME" != mismatch ]] || new_version=v6.4.6@@ -873,8 +909,12 @@ curl() {command cp "$FIXTURE_DIR/installer" "$out"fi}+# Pure-function manager doubles stay in this shell; the Python fixture uses+# real timeout with an external confined manager to test actual deadlines.+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }systemctl() {case "$1" in+ show) printf 'LoadState=loaded\n'; if [[ "$(cat "$FIXTURE_DIR/active")" == true ]]; then echo ActiveState=active; else echo ActiveState=inactive; fi ;;is-active) [[ "$(cat "$FIXTURE_DIR/active")" == true ]] ;;stop)printf 'stop\n' >> "$FIXTURE_DIR/service-operations"diff --git a/scripts/pulse-auto-update.sh b/scripts/pulse-auto-update.shindex a166d4e6d4..e41683d968 100755--- a/scripts/pulse-auto-update.sh+++ b/scripts/pulse-auto-update.sh@@ -365,14 +365,74 @@ resolve_install_script_url() {printf 'https://github.com/%s/releases/download/%s/install.sh\n' "$GITHUB_REPO" "$target_version"}-# Perform the update+# Read a complete, bounded manager observation. An is-active failure conflates+# an inactive unit with an unavailable manager; neither missing text nor a+# plausible partial result grants an update, restart or rollback.+read_update_service_state() {+ local service_name=$1 observation="" line="" load="" state=""+ local seen_load=false seen_state=false+ if ! observation=$(LC_ALL=C timeout -k 1 5 systemctl show "$service_name" --no-pager --property=LoadState --property=ActiveState 2>/dev/null); then+ log error "Cannot read Pulse service state ($service_name); reconcile service state before retrying" >&2+ return 1+ fi+ while IFS= read -r line; do+ case "$line" in+ LoadState=*)+ [[ "$seen_load" == false ]] || { log error "Duplicate Pulse service load state ($service_name)" >&2; return 1; }+ seen_load=true; load=${line#LoadState=} ;;+ ActiveState=*)+ [[ "$seen_state" == false ]] || { log error "Duplicate Pulse service active state ($service_name)" >&2; return 1; }+ seen_state=true; state=${line#ActiveState=} ;;+ *)+ log error "Incomplete Pulse service observation ($service_name); no service action is safe" >&2+ return 1 ;;+ esac+ done <<< "$observation"+ if [[ "$seen_load" != true || "$seen_state" != true || ( "$load" != loaded && "$load" != masked ) ]]; then+ log error "Pulse service is unavailable ($service_name); no service action is safe" >&2+ return 1+ fi+ case "$state" in+ active|inactive|failed|activating|deactivating|reloading|refreshing|maintenance) printf '%s\n' "$state" ;;+ *) log error "Unknown Pulse service state ($service_name); no service action is safe" >&2; return 1 ;;+ esac+}++# Only a fresh, settled observation permits an explicit start. Return 2 for+# uncertainty so callers do not turn an unavailable read into rollback consent.+start_update_service_if_stopped() {+ local service_name=$1 state=""+ state=$(read_update_service_state "$service_name") || return 2+ case "$state" in+ active) return 0 ;;+ inactive|failed) timeout -k 1 5 systemctl start "$service_name" 2>/dev/null ;;+ *) log error "Pulse service state is not settled ($service_name); refusing start"; return 2 ;;+ esac+}++stop_update_service_for_rollback() {+ local service_name=$1 state=""+ state=$(read_update_service_state "$service_name") || return 1+ case "$state" in+ inactive) return 0 ;;+ active|failed) ;;+ *) log error "Pulse service state is not settled ($service_name); refusing rollback"; return 1 ;;+ esac+ if ! timeout -k 1 5 systemctl stop "$service_name" 2>/dev/null; then+ return 1+ fi+ state=$(read_update_service_state "$service_name") || return 1+ [[ "$state" == inactive ]]+}+wait_for_service_active() {local service_name=$1local timeout_seconds="${2:-20}"- local elapsed=0+ local elapsed=0 state=""while (( elapsed < timeout_seconds )); do- if systemctl is-active --quiet "$service_name" 2>/dev/null; then+ state=$(read_update_service_state "$service_name") || return 2+ if [[ "$state" == active ]]; thenreturn 0fisleep 1@@ -398,16 +458,10 @@ ensure_service_restarted() {if [[ "$service_was_active" != "true" ]]; thenreturn 0fi- if systemctl is-active --quiet "$service_name" 2>/dev/null; then- return 0- fi-- log warn "Pulse service is not active after update attempt; starting it"- systemctl start "$service_name" 2>/dev/null || true- if wait_for_service_active "$service_name" 20; then+ if start_update_service_if_stopped "$service_name" && wait_for_service_active "$service_name" 20; thenlog info "Pulse service is running again"else- log error "Pulse service could not be started after update attempt; manual intervention required (systemctl start $service_name)"+ log error "Pulse service restart not confirmed ($service_name); reconcile service state before manual recovery"fireturn 0}@@ -418,6 +472,11 @@ perform_update() {# A local declaration hides command-substitution failure. Refuse before# prior-active observation, backup, installer download or restart traps.service_name=$(detect_service_name) || return 1+ # Never cross the stable channel even if the caller selected a preview.+ if is_prerelease_tag "$new_version"; then+ log error "Refusing to install prerelease version $new_version via unattended updater"+ return 1+ filocal installer_tmp=""local signature_tmp=""local backup_dir=""@@ -432,10 +491,13 @@ perform_update() {# Capture whether Pulse was running before the update so we can guarantee it# comes back up afterwards (#1323: auto-update could leave it stopped on# unprivileged LXC where the installer's restart silently fails).- local service_was_active="false"- if systemctl is-active --quiet "$service_name" 2>/dev/null; then- service_was_active="true"- fi+ local service_was_active="false" service_state="" service_result=0+ service_state=$(read_update_service_state "$service_name") || return 1+ case "$service_state" in+ active) service_was_active="true" ;;+ inactive|failed) ;;+ *) log error "Pulse service state is not settled ($service_name); refusing update"; return 1 ;;+ esac# Keep the prior-active restart backstop (#1630), except when incomplete# recovery requires manual intervention. The trap disarms itself on the@@ -445,14 +507,6 @@ perform_update() {# that aborted the updater under set -u after a successful update (#2128).trap 'trap - RETURN; rm -f -- "${installer_tmp:-}" "${signature_tmp:-}" "${restore_files[@]}"; if [[ -n "$backup_dir" && "$retain_backup" == "false" ]]; then rm -rf -- "$backup_dir"; fi; if [[ "$restart_allowed" == "true" ]]; then ensure_service_restarted "$service_name" "$service_was_active"; fi' RETURN- # Refuse to install a prerelease via the unattended updater. The stable- # channel must never cross onto a tag like v6.0.0-rc.2, even if every- # caller above this point thought it was safe.- if is_prerelease_tag "$new_version"; then- log error "Refusing to install prerelease version $new_version via unattended updater"- return 1- fi-log info "Starting update to $new_version"# main calls this function in an if condition, which disables errexit@@ -541,18 +595,38 @@ perform_update() {# again. On unprivileged LXC the installer's restart can silently# fail, leaving Pulse stopped (#1323).if [[ "$service_was_active" == "true" ]]; then- if ! wait_for_service_active "$service_name" 20; then+ service_result=0+ wait_for_service_active "$service_name" 20 || service_result=$?+ if [[ "$service_result" == 1 ]]; thenlog warn "Pulse service is not active after update, attempting one explicit start"- systemctl start "$service_name" || true+ start_update_service_if_stopped "$service_name" || service_result=$?fi-- if ! wait_for_service_active "$service_name" 20; then+ if [[ "$service_result" != 2 ]]; then+ service_result=0+ wait_for_service_active "$service_name" 20 || service_result=$?+ fi+ if [[ "$service_result" == 2 ]]; then+ retain_backup="true"+ restart_allowed="false"+ log error "Update service state is uncertain; backup retained at $backup_dir; reconcile service state before recovery"+ return 1+ elif [[ "$service_result" != 0 ]]; thenlog error "Pulse service did not come back up after update"elseupdate_accepted="true"fielse- update_accepted="true"+ # Inactive/failed prior state removes the restart obligation,+ # not the obligation to observe the installed unit safely.+ service_state=$(read_update_service_state "$service_name") || service_state="unknown"+ case "$service_state" in+ active|inactive|failed) update_accepted="true" ;;+ *)+ retain_backup="true"+ restart_allowed="false"+ log error "Update service state is uncertain; backup retained at $backup_dir; reconcile service state before recovery"+ return 1 ;;+ esacfielselog error "Version mismatch after update. Expected: $new_version, Got: $installed_version"@@ -572,8 +646,8 @@ perform_update() {retain_backup="true"restart_allowed="false"log info "Restoring from backup"- if ! systemctl stop "$service_name"; then- log error "Could not stop Pulse for rollback; backup retained at $backup_dir; manual recovery required"+ if ! stop_update_service_for_rollback "$service_name"; then+ log error "Could not confirm Pulse stopped for rollback; backup retained at $backup_dir; manual recovery required"return 1fi@@ -606,7 +680,7 @@ perform_update() {doneif [[ "$service_was_active" == "true" ]]; then- if ! systemctl start "$service_name" || ! wait_for_service_active "$service_name" 20; then+ if ! start_update_service_if_stopped "$service_name" || ! wait_for_service_active "$service_name" 20; thenlog error "Restored Pulse could not be started; backup retained at $backup_dir; manual recovery required"return 1fidiff --git a/scripts/tests/test-pulse-auto-update.sh b/scripts/tests/test-pulse-auto-update.shindex c007302886..bd1819f925 100755--- a/scripts/tests/test-pulse-auto-update.sh+++ b/scripts/tests/test-pulse-auto-update.sh@@ -16,6 +16,10 @@ fi# shellcheck disable=SC1090source "${AUTO_UPDATE_SCRIPT}"+# Command doubles run in this shell, not through the host manager. Actual+# timeout/manager boundaries are exercised by test_auto_update_service_state.py.+timeout() { case "$1" in -k) shift 2 ;; --kill-after=*) shift ;; esac; shift; "$@"; }+failures=0assert_success() {@@ -32,33 +36,28 @@ assert_success() {}test_wait_for_service_active_succeeds_after_retry() {- local calls=0-+ local counter; counter=$(mktemp)+ printf '0\n' > "$counter"systemctl() {- if [[ "$1" == "is-active" ]]; then- ((calls += 1))- if (( calls >= 3 )); then- return 0- fi- return 1- fi- return 1+ [[ "$1" == show ]] || return 1+ local count; count=$(cat "$counter"); ((count += 1))+ printf '%s\n' "$count" > "$counter"+ printf 'LoadState=loaded\n'+ if (( count >= 3 )); then echo ActiveState=active; else echo ActiveState=activating; fi}-sleep() { :; }-- wait_for_service_active pulse 5+ local result=0+ wait_for_service_active pulse 5 || result=$?+ command rm "$counter"+ return "$result"}test_wait_for_service_active_times_out_when_never_active() {- systemctl() { return 1; }+ systemctl() { [[ "$1" == show ]] || return 1; printf 'LoadState=loaded\nActiveState=inactive\n'; }sleep() { :; }-- if wait_for_service_active pulse 3; then- echo "expected wait_for_service_active to fail when service never becomes active" >&2- return 1- fi- return 0+ local result=0+ wait_for_service_active pulse 3 || result=$?+ [[ "$result" == 1 ]] || { echo "expected settled inactive timeout, got $result" >&2; return 1; }}test_pick_highest_stable_tag_ignores_list_order_and_prereleases() {@@ -156,6 +155,7 @@ echo "v5.1.24"EOFchmod +x "${INSTALL_DIR}/bin/pulse"+ printf 'active\n' > "$INSTALL_DIR/service-state"export INSTALL_DIRexport FAKE_NEW_VERSION="v5.1.25"@@ -183,6 +183,7 @@ EOFcat > "$out" <<'INSTALLER'#!/usr/bin/env bashprintf '%s\n' "${FAKE_NEW_VERSION}" > "${PULSE_INSTALL_DIR}/VERSION"+echo inactive > "${PULSE_INSTALL_DIR}/service-state"exit 0INSTALLER;;@@ -191,21 +192,13 @@ INSTALLERreturn 0}- # Service was running before the update (first is-active call true), then- # never comes back up; start also fails -> perform_update must restore + fail.- local is_active_calls=0+ # The installer records its stop; explicit starts fail for this scenario.systemctl() {- # The rollback must stop the service before replacing its executable.- # Stopping succeeds; only activation/liveness is the injected failure.- if [[ "$1" == "stop" ]]; then return 0; fi- if [[ "$1" == "is-active" ]]; then- ((is_active_calls += 1))- if (( is_active_calls == 1 )); then- return 0- fi- return 1- fi- return 1+ case "$1" in+ show) printf 'LoadState=loaded\nActiveState=%s\n' "$(cat "$INSTALL_DIR/service-state")"; return 0 ;;+ stop) echo inactive > "$INSTALL_DIR/service-state" ;;+ *) return 1 ;;+ esac}sleep() { :; }@@ -250,6 +243,11 @@ test_ensure_service_restarted_starts_stopped_service() {ENSURE_TEST_UP="no"systemctl() {case "$1" in+ show)+ printf 'LoadState=loaded\n'+ if [[ "$ENSURE_TEST_UP" == yes ]]; then echo ActiveState=active; else echo ActiveState=inactive; fi+ return 0+ ;;is-active)[[ "$ENSURE_TEST_UP" == "yes" ]] && return 0 || return 1;;@@ -300,6 +298,7 @@ echo "v5.1.24"EOFchmod +x "${INSTALL_DIR}/bin/pulse"+ printf 'active\n' > "$INSTALL_DIR/service-state"export INSTALL_DIRis_prerelease_tag() { return 1; }@@ -324,6 +323,7 @@ EOF*)cat > "$out" <<'INSTALLER'#!/usr/bin/env bash+echo inactive > "${PULSE_INSTALL_DIR}/service-state"exit 1INSTALLER;;@@ -339,6 +339,7 @@ INSTALLERAUTOUPDATE_TEST_STARTS=0systemctl() {case "$1" in+ show) printf 'LoadState=loaded\nActiveState=%s\n' "$(cat "$INSTALL_DIR/service-state")"; return 0 ;;is-active)((AUTOUPDATE_TEST_IS_ACTIVE_CALLS += 1))if (( AUTOUPDATE_TEST_IS_ACTIVE_CALLS == 1 )); then@@ -349,9 +350,10 @@ INSTALLERstart|restart)((AUTOUPDATE_TEST_STARTS += 1))AUTOUPDATE_TEST_UP="yes"+ echo active > "$INSTALL_DIR/service-state"return 0;;- stop) AUTOUPDATE_TEST_UP="no"; return 0 ;;+ stop) AUTOUPDATE_TEST_UP="no"; echo inactive > "$INSTALL_DIR/service-state"; return 0 ;;esacreturn 1}@@ -436,7 +438,7 @@ INSTALLER# Pulse was running and stays running, so perform_update succeeds.systemctl() {- if [[ "$1" == "is-active" ]]; then return 0; fi+ if [[ "$1" == show ]]; then printf 'LoadState=loaded\nActiveState=active\n'; fireturn 0}sleep() { :; }diff --git a/scripts/tests/test_auto_update_service_state.py b/scripts/tests/test_auto_update_service_state.pynew file mode 100644index 0000000000..7950d7d2c5--- /dev/null+++ b/scripts/tests/test_auto_update_service_state.py@@ -0,0 +1,241 @@+#!/usr/bin/env python3+"""Exercise the real updater with private files and a confined manager double.++No host manager, network, application build or published installer is used.+The versioned executable and signature verifier are synthetic; existing Go+transaction tests separately exercise real local SSH signatures and metadata.+"""++import json+import os+from pathlib import Path+import subprocess+import tempfile+import time+import unittest+++ROOT = Path(__file__).resolve().parents[2]+UPDATER = Path(os.environ.get("PULSE_AUTO_UPDATER_UNDER_TEST", ROOT / "scripts/pulse-auto-update.sh")).resolve()++MANAGER = r'''#!/usr/bin/env python3+import json, os, sys, time+from pathlib import Path+f = Path(os.environ["FIXTURE"])+args = sys.argv[1:]+with (f / "calls").open("a") as out:+ out.write(json.dumps(args) + "\n")+mode = os.environ["MODE"]+after = (f / "installer-ran").exists()+stopped = (f / "stop-ran").exists()+state = (f / "state").read_text().strip()+if args[0] == "show":+ if not after and mode == "initial-hang": time.sleep(30)+ if not after and mode == "initial-error":+ print("LoadState=loaded\nActiveState=inactive")+ sys.exit(1)+ if stopped and mode == "readback-error": sys.exit(1)+ if stopped and mode == "readback-empty": sys.exit(0)+ if after and not stopped and mode == "after-error": sys.exit(1)+ if after and mode == "success-error": sys.exit(1)+ if not after and mode == "initial-empty": sys.exit(0)+ if not after and mode == "initial-duplicate":+ print("LoadState=loaded\nActiveState=active\nActiveState=inactive")+ sys.exit(0)+ if not after and mode == "initial-extra":+ print("LoadState=loaded\nActiveState=inactive\nUnknown=active")+ sys.exit(0)+ # Reverse property order deliberately: systemd does not promise ordering.+ print("ActiveState=" + state)+ print("LoadState=" + os.environ["LOAD_STATE"])+ sys.exit(0)+if args[0] == "is-active":+ sys.exit(0 if state == "active" and not mode.startswith("initial-") else 1)+if args[0] == "stop":+ (f / "stop-ran").write_text("attempted\n")+ if mode == "stop-hang": time.sleep(30)+ if mode == "stop-error": sys.exit(1)+ if mode != "stop-noop": (f / "state").write_text("inactive\n")+ sys.exit(0)+if args[0] == "start":+ (f / "start-ran").write_text("attempted\n")+ (f / "state").write_text("active\n")+ sys.exit(0)+sys.exit(97)+'''++HARNESS = r'''+source "$UPDATER"+log() { printf '[%s] %s\n' "$1" "${*:2}"; }+detect_service_name() { printf '%s\n' "$SERVICE_NAME"; }+sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.+verify_release_signature() { :; }+mktemp() {+ local -a args=("$@")+ local last=$((${#args[@]} - 1))+ if [[ "${args[last]}" == /tmp/* ]]; then+ args[last]="$FIXTURE/tmp/${args[last]##*/}"+ fi+ printf "%s\n" "${args[*]}" >> "$FIXTURE/temp-requests"+ command mktemp "${args[@]}"+}+curl() {+ local out="" url=""+ while (( $# )); do+ case "$1" in -o) out="$2"; shift ;; https://*) url="$1" ;; esac+ shift+ done+ printf '%s\n' "$url" >> "$FIXTURE/downloads"+ if [[ "$url" == *.sshsig ]]; then printf 'fixture signature\n' > "$out"+ else command cp "$FIXTURE/installer" "$out"; fi+}+if [[ "$FLOW" == backstop ]]; then+ ensure_service_restarted "$SERVICE_NAME" true+ status=0+else+ # Production main uses a conditional call, disabling Bash errexit.+ status=0+ perform_update v6.6.0 || status=$?+fi+printf 'STATUS=%s\n' "$status"+[[ -z "$(trap -p RETURN)" ]] || { echo 'RETURN trap leaked'; exit 98; }+caller() { :; }; caller+exit "$status"+'''+++class UpdateServiceState(unittest.TestCase):+ def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update"):+ with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:+ f = Path(directory)+ for name in ("tools", "tmp", "install/bin", "config"):+ (f / name).mkdir(parents=True)+ (f / "tools/systemctl").write_text(MANAGER)+ (f / "tools/systemctl").chmod(0o755)+ old = b"#!/bin/sh\necho 'Pulse v6.5.0'\n"+ new = b"#!/bin/sh\necho 'Pulse v6.6.0'\n"+ (f / "install/bin/pulse").write_bytes(old)+ (f / "install/bin/pulse").chmod(0o751)+ (f / "install/VERSION").write_text("v6.5.0\n")+ (f / "config/system.json").write_text('operator settings\n')+ (f / "config/agent-id").write_text('original identity\n')+ (f / "state").write_text(state + "\n")+ (f / "new-pulse").write_bytes(new)+ (f / "installer").write_text(+ 'set -eu\n'+ 'echo attempted > "$FIXTURE/installer-ran"\n'+ 'cp "$FIXTURE/new-pulse" "$PULSE_INSTALL_DIR/bin/pulse"\n'+ 'echo v6.6.0 > "$PULSE_INSTALL_DIR/VERSION"\n'+ 'echo "$INSTALLER_STATE" > "$FIXTURE/state"\n'+ 'exit "$INSTALLER_EXIT"\n')+ env = dict(os.environ)+ env.update(FIXTURE=str(f), UPDATER=str(UPDATER), MODE=mode, FLOW=flow,+ PULSE_INSTALL_DIR=str(f / "install"), PULSE_CONFIG_DIR=str(f / "config"),+ PULSE_SERVICE_NAME="pulse-custom@department", LOAD_STATE=load,+ INSTALLER_STATE="active" if mode in ("stop-noop", "stop-error", "stop-hang", "after-error", "readback-error", "readback-empty") else "inactive",+ INSTALLER_EXIT="0" if success else "23",+ PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])+ start = time.monotonic()+ result = subprocess.run(["bash", "-c", HARNESS], env=env, capture_output=True, timeout=15)+ calls = [json.loads(line) for line in (f / "calls").read_text().splitlines()] if (f / "calls").exists() else []+ backups = list((f / "tmp").glob("pulse-backup.*"))+ observation = dict(exit=result.returncode, output=result.stdout.decode(), error=result.stderr.decode(),+ elapsed=time.monotonic() - start, calls=calls,+ downloaded=(f / "downloads").exists(), temporary=(f / "temp-requests").exists(),+ installed=(f / "installer-ran").exists(), stopped=(f / "stop-ran").exists(),+ started=(f / "start-ran").exists(), state=(f / "state").read_text().strip(),+ binary=(f / "install/bin/pulse").read_bytes(), version=(f / "install/VERSION").read_text(),+ backups=[(b / "pulse-bin").read_bytes() for b in backups],+ settings=(f / "config/system.json").read_text(), identity=(f / "config/agent-id").read_text())+ self.assertEqual(observation["settings"], "operator settings\n")+ self.assertEqual(observation["identity"], "original identity\n")+ self.assertNotIn("trap leaked", observation["output"])+ for args in calls:+ # All read/mutation calls target the exact explicit service.+ self.assertIn("pulse-custom@department", args)+ return observation++ def test_unknown_initial_state_refuses_before_backup_download_or_installer(self):+ cases = [(m, "active", "loaded") for m in+ ("initial-error", "initial-empty", "initial-duplicate", "initial-extra")]+ cases += [("normal", s, "loaded") for s in+ ("activating", "deactivating", "reloading", "refreshing", "maintenance", "future-state")]+ cases += [("normal", "inactive", s) for s in ("not-found", "error", "bad-setting", "future-load")]+ for mode, state, load in cases:+ with self.subTest(mode=mode, state=state, load=load):+ r = self.run_fixture(mode=mode, state=state, load=load, success=True)+ self.assertNotEqual(r["exit"], 0, r)+ self.assertFalse(r["installed"] or r["downloaded"] or r["temporary"], r)+ self.assertFalse(r["started"] or r["stopped"], r)+ self.assertEqual(r["backups"], [], r)+ self.assertIn(b"v6.5.0", r["binary"])++ def test_initial_observation_timeout_is_bounded_without_mutation(self):+ r = self.run_fixture(mode="initial-hang", success=True)+ self.assertNotEqual(r["exit"], 0, r)+ self.assertFalse(r["installed"] or r["started"] or r["stopped"], r)+ self.assertLess(r["elapsed"], 9, r)++ def test_failed_installer_restores_bytes_and_only_prior_active_service(self):+ for state, load in (("active", "loaded"), ("inactive", "loaded"), ("failed", "loaded"), ("inactive", "masked")):+ with self.subTest(state=state, load=load):+ r = self.run_fixture(state=state, load=load)+ self.assertNotEqual(r["exit"], 0, r)+ self.assertTrue(r["installed"], r)+ self.assertIn(b"v6.5.0", r["binary"])+ self.assertEqual(r["version"], "v6.5.0\n", r)+ self.assertEqual(r["started"], state == "active", r)+ self.assertEqual(r["backups"], [], r)++ def test_success_preserves_prior_active_backstop_without_unwanted_start(self):+ for state in ("active", "inactive"):+ with self.subTest(state=state):+ r = self.run_fixture(state=state, success=True)+ self.assertEqual(r["exit"], 0, r)+ self.assertIn(b"v6.6.0", r["binary"])+ self.assertEqual(r["started"], state == "active", r)+ self.assertFalse(r["stopped"], r)+ self.assertEqual(r["backups"], [], r)++ def test_uncertain_rollback_stop_retains_backup_without_replacement_or_start(self):+ for mode in ("stop-noop", "stop-error", "readback-error", "readback-empty", "after-error"):+ with self.subTest(mode=mode):+ r = self.run_fixture(mode=mode)+ self.assertNotEqual(r["exit"], 0, r)+ self.assertTrue(r["installed"], r)+ self.assertIn(b"v6.6.0", r["binary"], r)+ self.assertEqual(r["version"], "v6.6.0\n", r)+ self.assertEqual(len(r["backups"]), 1, r)+ self.assertIn(b"v6.5.0", r["backups"][0], r)+ self.assertFalse(r["started"], r)+ if mode == "after-error": self.assertFalse(r["stopped"], r)++ def test_rollback_stop_timeout_is_bounded_and_retains_recovery(self):+ r = self.run_fixture(mode="stop-hang")+ self.assertNotEqual(r["exit"], 0, r)+ self.assertIn(b"v6.6.0", r["binary"], r)+ self.assertEqual(len(r["backups"]), 1, r)+ self.assertFalse(r["started"], r)+ self.assertLess(r["elapsed"], 9, r)++ def test_post_install_observation_error_is_not_a_restart_or_rollback_grant(self):+ for state in ("active", "inactive"):+ with self.subTest(state=state):+ r = self.run_fixture(mode="success-error", state=state, success=True)+ self.assertNotEqual(r["exit"], 0, r)+ self.assertTrue(r["installed"], r)+ self.assertFalse(r["started"] or r["stopped"], r)+ self.assertIn(b"v6.6.0", r["binary"])+ self.assertEqual(len(r["backups"]), 1, r)++ def test_backstop_never_starts_unknown_or_unsettled_state(self):+ for mode, state in (("initial-error", "inactive"), ("initial-empty", "inactive"),+ ("normal", "activating"), ("normal", "deactivating")):+ with self.subTest(mode=mode, state=state):+ r = self.run_fixture(mode=mode, state=state, flow="backstop")+ self.assertEqual(r["exit"], 0, r) # A RETURN trap must not mask its caller's exit.+ self.assertFalse(r["started"] or r["stopped"] or r["installed"], r)+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/tests/test_server_service_discovery.py b/scripts/tests/test_server_service_discovery.pyindex e5e0d4bc5b..86371e9fa7 100644--- a/scripts/tests/test_server_service_discovery.py+++ b/scripts/tests/test_server_service_discovery.py@@ -19,8 +19,8 @@ import unittestROOT = Path(__file__).resolve().parents[2]-INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh"))-UPDATER = Path(os.environ.get("PULSE_AUTO_UPDATER_UNDER_TEST", ROOT / "scripts/pulse-auto-update.sh"))+INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh")).resolve()+UPDATER = Path(os.environ.get("PULSE_AUTO_UPDATER_UNDER_TEST", ROOT / "scripts/pulse-auto-update.sh")).resolve()REAL_SYSTEMCTL = shutil.which("systemctl")QUERY = ["list-unit-files", "--no-legend", "--no-pager", "--full", "--","pulse-backend.service", "pulse.service"]@@ -55,7 +55,11 @@ if args[0] == "list-unit-files":os.write(1, os.environ["ROWS"].encode())sys.exit(int(os.environ["QUERY_EXIT"]))if args[0] == "show":- print("inactive" if (f / "stopped").exists() else "active")+ state = "inactive" if (f / "stopped").exists() else "active"+ if "--value" in args:+ print(state)+ else:+ print("LoadState=loaded\nActiveState=" + state)sys.exit(0)if args[0] == "is-active":sys.exit(1 if (f / "stopped").exists() else 0)