Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Refuse server removal until affected units are quiescent
pulse-triage[bot]40dcedde858f
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex b68fb58229..17275adafb 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -15,6 +15,35 @@## Purpose+### Server removal requires every affected unit to be quiescent++Both the existing-installation menu and `--uninstall` must stop and disable the+instance's update timer, in-flight update service and server before deleting any+server executable, configuration, persistent data or unit file. Default-instance+removal also reconciles both historical server names; an explicit instance name+must not touch another server's units. An applicable local sensor-proxy footprint+is quiesced before server deletion, and checked again before its own removal.++Bounded systemd reads must establish known load state, explicit inactivity and+non-enabled unit-file state. A genuinely absent inactive unit is a no-op. Failed,+empty, transitional, still-active or still-enabled observations, ignored stop or+disable, and unknown load/enablement state block deletion. All targets are read+back again after the stop sequence. No uncertain or partially stopped sequence+is automatically restarted; diagnostics require service-state reconciliation.+Configuration-removal consent in the menu is unchanged. This does not redesign+`--reset`, grant host execution or establish installed recovery.++`TestRootInstallRemovalPreservesFilesOnUnconfirmedUnits`,+`TestRootInstallRemovalQuiescesUpdaterAndAliases` and+`TestRootInstallRemovalKeepsOtherInstanceUntouched` exercise both actual sourced+entrypoints with stateful, per-test systemd doubles and confined file removals.+These include timer reactivation/re-enablement, the in-flight updater, default+aliases, masked/failed/inactive/absent units and local legacy stop refusal.+A failed legacy-key rename also propagates when invoked conditionally, preserving+the original key file rather than reporting completion after an ignored error.+`TestRootInstallSensorProxyCleanupRejectsFailedKeyReplacement` executes that+boundary. Native systemd/removal acceptance remains separate from source fixtures.+### Legacy installer identity recovery is a whole-file decisionAn installer recovering a pre-descriptor-helper agent-ID file accepts exactlydiff --git a/install.sh b/install.shindex e2678b30ba..622fa446ba 100755--- a/install.sh+++ b/install.sh@@ -628,6 +628,90 @@ stop_pulse_for_replacement() {esac}+# Removal is destructive: neither an is-active error nor a successful stop+# proves the process is gone. Timers and in-flight updaters must be quiescent+# too, before deleting their executable or the server's persistent files.+confirm_pulse_unit_removable() {+ local unit="$1"+ local load_state="" active_state="" unit_file_state=""+ if ! load_state=$(timeout -k 1 5 systemctl show "$unit" --property=LoadState --value 2>/dev/null) ||+ ! active_state=$(timeout -k 1 5 systemctl show "$unit" --property=ActiveState --value 2>/dev/null); then+ print_error "Cannot confirm service state ($unit); refusing removal. Files are preserved; reconcile service state before retrying."+ return 1+ fi+ if [[ "$active_state" != "inactive" ]]; then+ print_error "Unit is not confirmed inactive ($unit); refusing removal. Files are preserved; reconcile service state before retrying."+ return 1+ fi+ [[ "$load_state" == "not-found" ]] && return 0+ case "$load_state" in+ loaded|masked) ;;+ *) print_error "Unknown unit load state ($unit); refusing removal"; return 1 ;;+ esac+ if ! unit_file_state=$(timeout -k 1 5 systemctl show "$unit" --property=UnitFileState --value 2>/dev/null); then+ print_error "Cannot confirm unit disabled ($unit); refusing removal"+ return 1+ fi+ case "$unit_file_state" in+ disabled|static|indirect|masked|masked-runtime) return 0 ;;+ *) print_error "Unit is not confirmed disabled ($unit); refusing removal"; return 1 ;;+ esac+}++stop_pulse_unit_for_removal() {+ local unit="$1"+ local load_state="" active_state=""+ if ! load_state=$(timeout -k 1 5 systemctl show "$unit" --property=LoadState --value 2>/dev/null) ||+ ! active_state=$(timeout -k 1 5 systemctl show "$unit" --property=ActiveState --value 2>/dev/null); then+ print_error "Cannot inspect unit ($unit); refusing removal"+ return 1+ fi+ if [[ "$load_state" == "not-found" && "$active_state" == "inactive" ]]; then+ return 0+ fi+ case "$load_state" in+ loaded|masked) ;;+ *) print_error "Unknown unit load state ($unit); refusing removal"; return 1 ;;+ esac+ case "$active_state" in+ inactive) ;;+ active|failed)+ if ! timeout -k 1 5 systemctl stop "$unit"; then+ print_error "Cannot stop unit ($unit); refusing removal. Reconcile service state before retrying."+ return 1+ fi+ ;;+ *) print_error "Unsettled unit state ($unit); refusing removal"; return 1 ;;+ esac+ if ! timeout -k 1 5 systemctl disable "$unit"; then+ print_error "Cannot disable unit ($unit); refusing removal"+ return 1+ fi+ confirm_pulse_unit_removable "$unit"+}++quiesce_pulse_for_removal() {+ local service_name="$1" unit=""+ local units=("$UPDATE_TIMER_UNIT" "$UPDATE_SERVICE_UNIT" "${service_name%.service}.service")+ if [[ "$SERVICE_NAME_EXPLICIT" != "true" ]]; then+ units+=(pulse.service pulse-backend.service)+ fi+ # A full uninstall also removes the local legacy footprint. Refuse before+ # any server deletion if that footprint cannot be safely quiesced.+ if local_sensor_proxy_present; then+ units+=(pulse-sensor-proxy-selfheal.timer pulse-sensor-cleanup.path+ pulse-sensor-proxy-selfheal.service pulse-sensor-cleanup.service pulse-sensor-proxy.service)+ fi+ for unit in "${units[@]}"; do+ stop_pulse_unit_for_removal "$unit" || return 1+ done+ # Reconcile all targets again, not just the last one stopped. An earlier+ # unit that reactivated/re-enabled during this sequence blocks deletion.+ for unit in "${units[@]}"; do+ confirm_pulse_unit_removable "$unit" || return 1+ done+}+# Called only after a confirmed stop and a failed atomic rename. The previous# executable is still in place. An uncertain stop never reaches this recovery,# and an intentionally inactive/failed service must not be started for the user.@@ -5212,9 +5296,10 @@ main() {exit 0;;remove)- # Stop and disable service- systemctl stop "$SERVICE_NAME" 2>/dev/null || true- systemctl disable "$SERVICE_NAME" 2>/dev/null || true+ if ! quiesce_pulse_for_removal "$SERVICE_NAME"; then+ print_error "Pulse removal is incomplete; no files have been removed. Previously stopped units are not automatically restarted."+ return 1+ fi# Remove service filesrm -f "/etc/systemd/system/$SERVICE_NAME.service"@@ -5373,7 +5458,10 @@ remove_local_sensor_proxy_managed_keys() {fichmod --reference="$auth_file" "$tmp_file" 2>/dev/null || chmod 600 "$tmp_file" 2>/dev/null || truechown --reference="$auth_file" "$tmp_file" 2>/dev/null || true- mv "$tmp_file" "$auth_file"+ if ! mv "$tmp_file" "$auth_file"; then+ rm -f "$tmp_file"+ return 1+ fiecho "Removed legacy pulse-sensor-proxy SSH key entries from $auth_file"}@@ -5389,24 +5477,23 @@ cleanup_local_sensor_proxy() {pulse-sensor-proxy-selfheal.service \pulse-sensor-cleanup.path \pulse-sensor-cleanup.service; do- systemctl stop "$unit" >/dev/null 2>&1 || true- systemctl disable "$unit" >/dev/null 2>&1 || true+ stop_pulse_unit_for_removal "$unit" || return 1done- rm -f "$SENSOR_PROXY_BINARY_PATH"- rm -f /usr/local/bin/pulse-sensor-cleanup.sh- rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-proxy.service"- rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-proxy-selfheal.service"- rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-proxy-selfheal.timer"- rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-cleanup.service"- rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-cleanup.path"- rm -rf "$SENSOR_PROXY_RUNTIME_DIR"- rm -rf "$SENSOR_PROXY_INSTALL_ROOT"- rm -rf "$SENSOR_PROXY_WORK_DIR"- rm -rf "$SENSOR_PROXY_CONFIG_DIR"- rm -rf "$SENSOR_PROXY_LOG_DIR"-- remove_local_sensor_proxy_managed_keys+ rm -f "$SENSOR_PROXY_BINARY_PATH" || return 1+ rm -f /usr/local/bin/pulse-sensor-cleanup.sh || return 1+ rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-proxy.service" || return 1+ rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-proxy-selfheal.service" || return 1+ rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-proxy-selfheal.timer" || return 1+ rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-cleanup.service" || return 1+ rm -f "${SENSOR_PROXY_SYSTEMD_DIR}/pulse-sensor-cleanup.path" || return 1+ rm -rf "$SENSOR_PROXY_RUNTIME_DIR" || return 1+ rm -rf "$SENSOR_PROXY_INSTALL_ROOT" || return 1+ rm -rf "$SENSOR_PROXY_WORK_DIR" || return 1+ rm -rf "$SENSOR_PROXY_CONFIG_DIR" || return 1+ rm -rf "$SENSOR_PROXY_LOG_DIR" || return 1++ remove_local_sensor_proxy_managed_keys || return 1if id -u "$SENSOR_PROXY_SERVICE_USER" >/dev/null 2>&1; thenuserdel --remove "$SENSOR_PROXY_SERVICE_USER" >/dev/null 2>&1 || userdel "$SENSOR_PROXY_SERVICE_USER" >/dev/null 2>&1 || true@@ -5421,6 +5508,7 @@ cleanup_local_sensor_proxy() {echo " curl -fsSL https://raw.githubusercontent.com/rcourtman/Pulse/main/scripts/uninstall-sensor-proxy.sh | bash -s -- --purge --remove-proxmox-access --local-only"}+# Uninstall functionuninstall_pulse() {check_root@@ -5432,23 +5520,11 @@ uninstall_pulse() {local service_nameservice_name=$(detect_service_name)- # Stop and disable service- if systemctl is-active --quiet "$service_name" 2>/dev/null; then- echo "Stopping $service_name..."- systemctl stop "$service_name"- fi-- if systemctl is-enabled --quiet "$service_name" 2>/dev/null; then- echo "Disabling $service_name..."- systemctl disable "$service_name"- fi-- # Stop and disable auto-update timer if it exists- if update_timer_enabled; then- echo "Disabling auto-update timer..."- systemctl disable --now "$UPDATE_TIMER_UNIT"+ if ! quiesce_pulse_for_removal "$service_name"; then+ print_error "Pulse removal is incomplete; no files have been removed. Previously stopped units are not automatically restarted."+ return 1fi-+# Remove filesecho "Removing Pulse files..."rm -rf "$INSTALL_DIR"@@ -5474,7 +5550,7 @@ uninstall_pulse() {# Remove any leftover legacy pulse-sensor-proxy footprint on this host so a# full uninstall on a v5-upgraded Proxmox host leaves nothing behind.- cleanup_local_sensor_proxy+ cleanup_local_sensor_proxy || return 1# Reload systemdsystemctl daemon-reloaddiff --git a/scripts/installtests/root_install_sh_test.go b/scripts/installtests/root_install_sh_test.goindex 3d190208a4..f80d106231 100644--- a/scripts/installtests/root_install_sh_test.go+++ b/scripts/installtests/root_install_sh_test.go@@ -1022,14 +1022,27 @@ func TestRootInstallUninstallCleansLegacySensorProxy(t *testing.T) {SENSOR_PROXY_LOG_DIR="` + filepath.Join(tmp, "log") + `"SENSOR_PROXY_SERVICE_USER="pulse-sensor-proxy-test"SENSOR_PROXY_AUTHORIZED_KEYS_PATH="` + authKeys + `"- systemctl() { return 0; }+ timeout() { shift 3; "$@"; }+ systemctl() {+ if [[ "$1" == "show" ]]; then+ case "$3" in+ --property=LoadState) echo loaded ;;+ --property=ActiveState) echo inactive ;;+ --property=UnitFileState) echo disabled ;;+ *) return 1 ;;+ esac+ fi+ }userdel() { echo "userdel $*" >>"` + marker + `"; return 0; }groupdel() { echo "groupdel $*" >>"` + marker + `"; return 0; }id() { return 0; }getent() { return 0; }`- funcs := extractRootInstallShellFunction(t, "local_sensor_proxy_present") + "\n" ++ funcs := extractRootInstallShellFunction(t, "confirm_pulse_unit_removable") + "\n" ++ extractRootInstallShellFunction(t, "stop_pulse_unit_for_removal") + "\n" ++ `print_error() { echo "$*" >&2; }` + "\n" ++ extractRootInstallShellFunction(t, "local_sensor_proxy_present") + "\n" +extractRootInstallShellFunction(t, "remove_local_sensor_proxy_managed_keys") + "\n" +extractRootInstallShellFunction(t, "cleanup_local_sensor_proxy")@@ -2502,3 +2515,37 @@ func TestRootInstallPctExecCommandsResolveOnPctExecPath(t *testing.T) {t.Fatalf("install.sh:%d runs %q through pct exec by bare name, which is not on PATH=/sbin:/bin:/usr/sbin:/usr/bin: %s", i+1, command, strings.TrimSpace(line))}}++// Failures must propagate even when the cleanup is called from a conditional+// (where Bash disables errexit); a failed key replacement cannot claim success.+func TestRootInstallSensorProxyCleanupRejectsFailedKeyReplacement(t *testing.T) {+ dir := t.TempDir()+ installer, err := filepath.Abs(filepath.Join("..", "..", "install.sh"))+ if err != nil {+ t.Fatal(err)+ }+ keys := filepath.Join(dir, "authorized_keys")+ original := "ssh-ed25519 TEST keep-admin\nssh-ed25519 OLD # pulse-managed-key\n"+ if err := os.WriteFile(keys, []byte(original), 0600); err != nil {+ t.Fatal(err)+ }+ script := `+source "$INSTALLER_UNDER_TEST"+SENSOR_PROXY_AUTHORIZED_KEYS_PATH="$KEYS_UNDER_TEST"+mv() { return 1; }+if remove_local_sensor_proxy_managed_keys; then+ echo UNEXPECTED_COMPLETION+ exit 1+fi+`+ cmd := exec.Command("bash", "-c", script)+ cmd.Env = append(os.Environ(), "INSTALLER_UNDER_TEST="+installer, "KEYS_UNDER_TEST="+keys)+ out, err := cmd.CombinedOutput()+ if err != nil || strings.Contains(string(out), "Removed legacy") {+ t.Fatalf("failed replacement was accepted: %v\n%s", err, out)+ }+ got, err := os.ReadFile(keys)+ if err != nil || string(got) != original {+ t.Fatalf("key replacement failure changed original: %v", err)+ }+}diff --git a/scripts/installtests/server_removal_safety_test.go b/scripts/installtests/server_removal_safety_test.gonew file mode 100644index 0000000000..0f6d0cd957--- /dev/null+++ b/scripts/installtests/server_removal_safety_test.go@@ -0,0 +1,275 @@+package installtests++import (+ "context"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+ "time"+)++// Exercise the actual sourced installer entrypoints. systemctl is an on-disk+// stateful double, so the real timeout wrapper runs it; destructive commands+// can affect only per-test files, never an installed service or host directory.+func TestRootInstallRemovalPreservesFilesOnUnconfirmedUnits(t *testing.T) {+ cases := []struct{ name, mode, target string }{+ {"initial_query_failure", "query-fails", "pulse.service"},+ {"empty_load", "empty-load", "pulse.service"},+ {"unknown_load", "unknown-load", "pulse.service"},+ {"empty_active", "empty-active", "pulse.service"},+ {"unsettled_active", "unsettled", "pulse.service"},+ {"stop_failure", "stop-fails", "pulse.service"},+ {"ignored_stop", "stop-ignored", "pulse.service"},+ {"failed_after_stop", "still-failed", "pulse.service"},+ {"stop_readback_failure", "readback-fails", "pulse.service"},+ {"disable_failure", "disable-fails", "pulse.service"},+ {"ignored_disable", "disable-ignored", "pulse.service"},+ {"empty_enabled_state", "empty-enabled", "pulse.service"},+ {"disable_readback_failure", "enabled-readback-fails", "pulse.service"},+ {"timer_stop_failure", "stop-fails", "pulse-update.timer"},+ {"inflight_update_stop_failure", "stop-fails", "pulse-update.service"},+ {"legacy_alias_stop_failure", "stop-fails", "pulse-backend.service"},+ {"timer_reactivated", "reactivated", "pulse-update.timer"},+ {"timer_reenabled", "reenabled", "pulse-update.timer"},+ {"legacy_proxy_stop_failure", "stop-fails", "pulse-sensor-proxy.service"},+ }+ for _, flow := range []string{"menu", "uninstall"} {+ for _, tc := range cases {+ t.Run(flow+"/"+tc.name, func(t *testing.T) {+ out, calls, err := runServerRemovalFixture(t, flow, tc.mode, tc.target, "active", false, true)+ if err == nil {+ t.Fatalf("expected rejected removal, got success:\n%s\n%s", out, calls)+ }+ if strings.Contains(calls, "DELETE ") || strings.Contains(calls, "start ") || strings.Contains(out, "successfully") || strings.Contains(out, "completely uninstalled") {+ t.Fatalf("unconfirmed unit allowed deletion, restart or completion:\n%s\n%s", out, calls)+ }+ })+ }+ }+}++func TestRootInstallRemovalQuiescesUpdaterAndAliases(t *testing.T) {+ for _, flow := range []string{"menu", "uninstall"} {+ for _, state := range []string{"active", "inactive", "failed", "not-found", "masked"} {+ t.Run(flow+"/"+state, func(t *testing.T) {+ out, calls, err := runServerRemovalFixture(t, flow, "normal", "", state, false, true)+ if err != nil {+ t.Fatalf("safe removal failed: %v\n%s\n%s", err, out, calls)+ }+ deleteAt := strings.Index(calls, "DELETE ")+ if deleteAt < 0 {+ t.Fatalf("removal did not reach file deletion:\n%s", calls)+ }+ for _, unit := range []string{"pulse-update.timer", "pulse-update.service", "pulse.service", "pulse-backend.service", "pulse-sensor-proxy.service"} {+ if !strings.Contains(calls[:deleteAt], "show "+unit+" --property=ActiveState") {+ t.Fatalf("%s was not observed before deletion:\n%s", unit, calls)+ }+ if state != "not-found" && !strings.Contains(calls[:deleteAt], "show "+unit+" --property=UnitFileState") {+ t.Fatalf("%s disablement was not observed:\n%s", unit, calls)+ }+ }+ if state == "active" || state == "failed" {+ timerAt := strings.Index(calls, "stop pulse-update.timer")+ updateAt := strings.Index(calls, "stop pulse-update.service")+ serverAt := strings.Index(calls, "stop pulse.service")+ if !(timerAt >= 0 && timerAt < updateAt && updateAt < serverAt && serverAt < deleteAt) {+ t.Fatalf("unsafe stop ordering:\n%s", calls)+ }+ }+ if strings.Contains(calls, "start ") || (state == "not-found" && strings.Contains(calls, "disable ")) || (state == "inactive" && strings.Contains(calls, "stop ")) {+ t.Fatalf("unnecessary service mutation:\n%s", calls)+ }+ })+ }+ }+}++func TestRootInstallRemovalKeepsOtherInstanceUntouched(t *testing.T) {+ for _, flow := range []string{"menu", "uninstall"} {+ t.Run(flow, func(t *testing.T) {+ out, calls, err := runServerRemovalFixture(t, flow, "normal", "", "active", true, false)+ if err != nil {+ t.Fatalf("custom instance removal: %v\n%s", err, out)+ }+ for _, other := range []string{"pulse.service", "pulse-backend.service", "pulse-update.timer", "pulse-update.service"} {+ if strings.Contains(calls, " "+other) || strings.Contains(calls, "/"+other) {+ t.Fatalf("custom-instance removal touched %s:\n%s", other, calls)+ }+ }+ })+ }+}++func runServerRemovalFixture(t *testing.T, flow, mode, target, state string, custom, legacy bool) (string, string, error) {+ t.Helper()+ dir := t.TempDir()+ binDir := filepath.Join(dir, "tools")+ if err := os.Mkdir(binDir, 0700); err != nil {+ t.Fatal(err)+ }+ for _, d := range []string{"install", "config", "proxy"} {+ if err := os.Mkdir(filepath.Join(dir, d), 0700); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(dir, d, "preserve"), []byte("original-"+d), 0600); err != nil {+ t.Fatal(err)+ }+ }+ service := "pulse"+ if custom {+ service = "pulse-other"+ }+ units := []string{service + ".service", service + "-update.service", service + "-update.timer"}+ if !custom {+ units = append(units, "pulse-backend.service")+ }+ if legacy {+ units = append(units, "pulse-sensor-proxy.service", "pulse-sensor-proxy-selfheal.timer", "pulse-sensor-proxy-selfheal.service", "pulse-sensor-cleanup.path", "pulse-sensor-cleanup.service")+ }+ for _, unit := range units {+ load, active, enabled := "loaded", state, "enabled"+ if state == "not-found" {+ load, active, enabled = "not-found", "inactive", ""+ }+ if state == "masked" {+ load, active, enabled = "masked", "inactive", "masked"+ }+ for suffix, val := range map[string]string{"load": load, "active": active, "enabled": enabled} {+ if err := os.WriteFile(filepath.Join(dir, unit+"."+suffix), []byte(val+"\n"), 0600); err != nil {+ t.Fatal(err)+ }+ }+ }+ mock := `#!/bin/bash+set -eu+printf '%s\n' "$*" >> "$FIXTURE_DIR/calls"+action=$1; unit=${2:-}+if [[ "$action" == show ]]; then+ prop=$3+ if [[ "$unit" == "$TARGET" ]]; then+ case "$MODE:$prop" in+ query-fails:*) exit 1 ;;+ empty-load:--property=LoadState|empty-active:--property=ActiveState|empty-enabled:--property=UnitFileState) exit 0 ;;+ unknown-load:--property=LoadState) echo error; exit 0 ;;+ unsettled:--property=ActiveState) echo activating; exit 0 ;;+ readback-fails:--property=ActiveState) [[ ! -e "$FIXTURE_DIR/$unit.stopped" ]] || exit 1 ;;+ enabled-readback-fails:--property=UnitFileState) exit 1 ;;+ esac+ if [[ -e "$FIXTURE_DIR/$SERVICE.service.disabled" ]]; then+ case "$MODE:$prop" in+ reactivated:--property=ActiveState) echo active; exit 0 ;;+ reenabled:--property=UnitFileState) echo enabled; exit 0 ;;+ esac+ fi+ fi+ case "$prop" in+ --property=LoadState) cat "$FIXTURE_DIR/$unit.load" ;;+ --property=ActiveState) cat "$FIXTURE_DIR/$unit.active" ;;+ --property=UnitFileState) cat "$FIXTURE_DIR/$unit.enabled" ;;+ *) exit 2 ;;+ esac+elif [[ "$action" == stop ]]; then+ [[ "$unit:$MODE" != "$TARGET:stop-fails" ]] || exit 1+ touch "$FIXTURE_DIR/$unit.stopped"+ if [[ "$unit:$MODE" == "$TARGET:still-failed" ]]; then+ echo failed > "$FIXTURE_DIR/$unit.active"+ elif [[ "$unit:$MODE" != "$TARGET:stop-ignored" ]]; then+ echo inactive > "$FIXTURE_DIR/$unit.active"+ fi+elif [[ "$action" == disable ]]; then+ [[ "$unit:$MODE" != "$TARGET:disable-fails" ]] || exit 1+ touch "$FIXTURE_DIR/$unit.disabled"+ if [[ "$unit:$MODE" != "$TARGET:disable-ignored" ]]; then+ case "$(cat "$FIXTURE_DIR/$unit.enabled")" in masked) ;; *) echo disabled > "$FIXTURE_DIR/$unit.enabled" ;; esac+ fi+elif [[ "$action" != daemon-reload ]]; then+ exit 2+fi+`+ if err := os.WriteFile(filepath.Join(binDir, "systemctl"), []byte(mock), 0700); err != nil {+ t.Fatal(err)+ }+ installer := os.Getenv("PULSE_TEST_ROOT_INSTALLER")+ if installer == "" {+ installer = filepath.Join("..", "..", "install.sh")+ }+ installer, err := filepath.Abs(installer)+ if err != nil {+ t.Fatal(err)+ }+ script := `+source "$INSTALLER_UNDER_TEST"+CONFIG_DIR="$FIXTURE_DIR/config"+INSTALL_DIR="$FIXTURE_DIR/install"+BINARY_LINK_PATH="$FIXTURE_DIR/link"+AUTO_UPDATE_DEST="$FIXTURE_DIR/updater"+UPDATE_HELPER_PATH="$FIXTURE_DIR/helper"+UPDATE_SERVICE_PATH="$FIXTURE_DIR/$SERVICE-update.service"+UPDATE_TIMER_PATH="$FIXTURE_DIR/$SERVICE-update.timer"+UPDATE_SERVICE_UNIT="$SERVICE-update.service"+UPDATE_TIMER_UNIT="$SERVICE-update.timer"+SERVICE_NAME="$SERVICE"+SERVICE_NAME_EXPLICIT="$CUSTOM"+SENSOR_PROXY_BINARY_PATH="$FIXTURE_DIR/proxy/binary"+SENSOR_PROXY_INSTALL_ROOT="$FIXTURE_DIR/proxy"+SENSOR_PROXY_RUNTIME_DIR="$FIXTURE_DIR/proxy/run"+SENSOR_PROXY_WORK_DIR="$FIXTURE_DIR/proxy/work"+SENSOR_PROXY_CONFIG_DIR="$FIXTURE_DIR/proxy/config"+SENSOR_PROXY_LOG_DIR="$FIXTURE_DIR/proxy/log"+SENSOR_PROXY_AUTHORIZED_KEYS_PATH="$FIXTURE_DIR/proxy/absent-keys"+SENSOR_PROXY_SYSTEMD_DIR="$FIXTURE_DIR/proxy/units"+IN_CONTAINER=true+IN_DOCKER=false+CURRENT_VERSION=v6.5.0+FORCE_VERSION=""+for fn in print_header check_root detect_os check_docker_environment; do eval "$fn() { :; }"; done+check_existing_installation() { return 0; }+detect_service_name() { echo "$SERVICE"; }+resolve_latest_release_tag_for_channel() { :; }+read_configured_update_channel() { echo stable; }+curl() { :; }+local_sensor_proxy_present() { [[ "$LEGACY" == true ]]; }+id() { return 1; }+getent() { return 1; }+userdel() { echo USER_DELETE >> "$FIXTURE_DIR/calls"; }+rm() {+ echo "DELETE $*" >> "$FIXTURE_DIR/calls"+ # Hard-coded system paths are observed but never mutated. All actual removals+ # must be in the fixture; unexpected options are a fixture error.+ local arg+ for arg in "$@"; do+ case "$arg" in -f|-rf) ;; "$FIXTURE_DIR"/*) /bin/rm -rf -- "$arg" ;; /etc/systemd/system/*|/usr/local/bin/pulse-sensor-cleanup.sh|/var/log/pulse*.log|/opt/pulse.log) ;; *) return 91 ;; esac+ done+}+safe_read() { printf -v "$2" '%s' 2; }+safe_read_with_default() { printf -v "$2" '%s' y; }+if [[ "$FLOW" == menu ]]; then main; else uninstall_pulse; fi+`+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, "bash", "-c", script)+ cmd.Env = append(os.Environ(), "PATH="+binDir+":"+os.Getenv("PATH"), "INSTALLER_UNDER_TEST="+installer, "FIXTURE_DIR="+dir, "FLOW="+flow, "MODE="+mode, "TARGET="+target, "SERVICE="+service, "CUSTOM="+boolString(custom), "LEGACY="+boolString(legacy))+ out, runErr := cmd.CombinedOutput()+ if ctx.Err() != nil {+ t.Fatalf("fixture deadline: %v\n%s", ctx.Err(), out)+ }+ calls, _ := os.ReadFile(filepath.Join(dir, "calls"))+ if mode != "normal" {+ for _, d := range []string{"install", "config", "proxy"} {+ b, e := os.ReadFile(filepath.Join(dir, d, "preserve"))+ if e != nil || string(b) != "original-"+d {+ t.Fatalf("unconfirmed removal changed %s: %v\n%s\n%s", d, e, out, calls)+ }+ }+ }+ return string(out), string(calls), runErr+}++func boolString(v bool) string {+ if v {+ return "true"+ }+ return "false"+}