Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve shared SSH key targets during legacy remote cleanup
pulse-triage[bot]5bc140aa7e3f
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex a55ceb2a5a..c1d504f8d7 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -776,6 +776,16 @@ empty file. Only the two existing Pulse comment suffixes are eligible, andunrelated keys and file metadata remain intact. Conditional helper invocationmust propagate failure rather than print a successful key-removal claim.+Remote cleanup resolves the fixed `authorized_keys` pathname before replacing+its target; it must never replace a cluster-shared symlink with a private copy.+A dangling link, non-file target or failed resolution is not confirmed absence.+Filtering uses a separately opened adjacent staging file, checks grep's no-match+versus error status and preserves target metadata before atomic replacement.+Read, staging, comparison, metadata and rename failures preserve the target and+return failure; an already-clean shared target is not replaced. Staging cleanup+failure is also nonzero, not a successful remote removal. Strict host trust and+the existing fixed remote pathname remain unchanged.+Opted-in API cleanup likewise requires a complete valid JSON token inventory.It does not reinterpret a failed or malformed response as empty or fall back toa guessed table. User deletion follows only successful removal of every listed@@ -794,6 +804,16 @@ fixtures through the repository's Go suite. Existing SSH trust and containerstate tests remain necessary. These are executable source/producer checks, notnative Proxmox cleanup, installed recovery or reporter acceptance.+`TestUninstallSensorProxyRemoteKeyPayloadPreservesSharedTarget` substitutes only+the fixed remote key pathname and executes the actual SSH payload in a separate+POSIX shell. Regular/shared-link paths, peer readback, already-clean targets,+all-managed/absent/broken/non-file inputs and injected operation failures check+key bytes, symlink/inode continuity, modes, staging cleanup and truthful status.+`TestUninstallSensorProxyRemotePayloadFailureReachesMain` exercises complete+inventory through the real cluster/remote helpers into main's incomplete result,+without live SSH, Proxmox or customer operations. These checks do not establish+native cluster-filesystem semantics or any original installed recovery.+### Legacy sensor cleanup requires observed stopped targetsThe legacy sensor uninstaller must confirm each existing unit has stopped anddiff --git a/scripts/installtests/uninstall_sensor_proxy_test.go b/scripts/installtests/uninstall_sensor_proxy_test.goindex c671394dfd..d00ef2565d 100644--- a/scripts/installtests/uninstall_sensor_proxy_test.go+++ b/scripts/installtests/uninstall_sensor_proxy_test.go@@ -588,3 +588,193 @@ if main --remove-proxmox-access; then exit 0; else exit $?; fit.Fatalf("file cleanup reload skipped: %q, %v", got, err)}}++// Confine only the fixed remote pathname. The SSH double below executes the+// actual transmitted command in a separate POSIX shell; it never substitutes+// cleanup_remote_authorized_keys or its filtering/replacement logic.+func sensorProxyRemoteKeyScript(t *testing.T, dir string) string {+ t.Helper()+ source, err := os.ReadFile(repoFile("scripts", "uninstall-sensor-proxy.sh"))+ if err != nil {+ t.Fatal(err)+ }+ const fixed = `auth="/root/.ssh/authorized_keys"`+ if strings.Count(string(source), fixed) != 1 {+ t.Fatal("expected one fixed remote authorized_keys path")+ }+ script := strings.Replace(string(source), fixed, `auth="${FIXTURE_DIR}/remote/authorized_keys"`, 1)+ path := filepath.Join(dir, "uninstall-remote-fixture.sh")+ if err := os.WriteFile(path, []byte(script), 0600); err != nil {+ t.Fatal(err)+ }+ return path+}++const sensorProxyRemoteKeyFixture = `source "$1"+AUTHORIZED_KEYS_PATH="$FIXTURE_DIR/local_authorized_keys"+SSH_KNOWN_HOSTS_PATH=""+ssh() {+ [[ "${@: -2:1}" == [email protected] ]] || return 90+ echo ssh >>"$FIXTURE_DIR/actions"+ local faults='+readlink() { [ "$SCENARIO" != readlink ] || return 9; command readlink "$@"; }+mktemp() {+ [ "$SCENARIO" != mktemp ] || return 9+ if [ "$SCENARIO" = output_open ]; then printf "%s\n" "$FIXTURE_DIR/remote"; return 0; fi+ command mktemp "$@"+}+grep() {+ if [ "$SCENARIO" = grep ]; then printf "incomplete output\n"; return 2; fi+ command grep "$@"+}+cmp() { [ "$SCENARIO" != cmp ] || return 2; command cmp "$@"; }+chmod() { case "$SCENARIO" in chmod|unchanged) return 9;; esac; command chmod "$@"; }+chown() { case "$SCENARIO" in chown|unchanged) return 9;; esac; command chown "$@"; }+mv() { case "$SCENARIO" in mv|unchanged) return 9;; esac; command mv "$@"; }+rm() { [ "$SCENARIO" != cleanup ] || return 9; command rm "$@"; }+'+ sh -c "$faults"$'\n'"${!#}"+}+`++const sensorProxyRecoveryKeys = "ssh-ed25519 AAAAkeep recovery-key\n" ++ "restrict ssh-ed25519 AAAAkeep2 other # pulse-managed-key-not-a-marker\n"++func TestUninstallSensorProxyRemoteKeyPayloadPreservesSharedTarget(t *testing.T) {+ for _, kind := range []string{"regular", "symlink"} {+ for _, phase := range []string{"success", "unchanged", "all_managed", "missing", "directory", "readlink", "mktemp", "output_open", "grep", "cmp", "chmod", "chown", "mv", "cleanup"} {+ t.Run(kind+"/"+phase, func(t *testing.T) {+ dir := t.TempDir()+ auth := filepath.Join(dir, "remote", "authorized_keys")+ target := auth+ if kind == "symlink" {+ target = filepath.Join(dir, "cluster keys", "authorized_keys")+ }+ for _, parent := range []string{filepath.Dir(auth), filepath.Dir(target), filepath.Join(dir, "peer")} {+ if err := os.MkdirAll(parent, 0700); err != nil {+ t.Fatal(err)+ }+ }+ original := "ssh-ed25519 AAAAold pulse # pulse-managed-key\n" ++ sensorProxyRecoveryKeys + "ssh-ed25519 AAAAold2 pulse # pulse-proxy-key\n"+ if phase == "unchanged" || phase == "cleanup" {+ original = sensorProxyRecoveryKeys+ } else if phase == "all_managed" {+ original = "ssh-ed25519 AAAAold pulse # pulse-managed-key\nssh-ed25519 AAAAold2 pulse # pulse-proxy-key\n"+ }+ if phase == "directory" {+ if err := os.Mkdir(target, 0700); err != nil {+ t.Fatal(err)+ }+ } else if phase != "missing" {+ if err := os.WriteFile(target, []byte(original), 0640); err != nil {+ t.Fatal(err)+ }+ }+ const linkTarget = "../cluster keys/authorized_keys"+ peer := filepath.Join(dir, "peer", "authorized_keys")+ if kind == "symlink" {+ for _, link := range []string{auth, peer} {+ if err := os.Symlink(linkTarget, link); err != nil {+ t.Fatal(err)+ }+ }+ }+ before, _ := os.Stat(target)+ linkBefore, _ := os.Lstat(auth)+ out, code := runSensorProxySafetyShell(t, sensorProxyRemoteKeyScript(t, dir), dir, phase,+ sensorProxyRemoteKeyFixture+`if cleanup_remote_authorized_keys 192.0.2.11; then exit 0; else exit $?; fi`)+ success := phase == "success" || phase == "unchanged" || phase == "all_managed" || (phase == "missing" && kind == "regular")+ if (code == 0) != success || strings.Contains(out, "[ OK ]") != success {+ t.Fatalf("remote status mismatch: exit %d: %s", code, out)+ }+ if kind == "symlink" {+ afterLink, err := os.Lstat(auth)+ if err != nil || !os.SameFile(linkBefore, afterLink) {+ t.Fatalf("remote cleanup replaced the shared-key symlink: %v", err)+ }+ for _, link := range []string{auth, peer} {+ if got, err := os.Readlink(link); err != nil || got != linkTarget {+ t.Fatalf("shared-key relationship changed: %q, %v", got, err)+ }+ }+ }+ if phase == "missing" {+ if _, err := os.Stat(target); !os.IsNotExist(err) {+ t.Fatalf("missing target created: %v", err)+ }+ return+ }+ after, err := os.Stat(target)+ if err != nil {+ t.Fatal(err)+ }+ if phase == "directory" {+ if !after.IsDir() || !os.SameFile(before, after) {+ t.Fatal("non-file target was replaced")+ }+ return+ }+ want := original+ if phase == "success" {+ want = sensorProxyRecoveryKeys+ } else if phase == "all_managed" {+ want = ""+ }+ if got, err := os.ReadFile(auth); err != nil || string(got) != want {+ t.Fatalf("remote key content lost or left managed: %q, %v; want %q", got, err, want)+ }+ if kind == "symlink" {+ if got, err := os.ReadFile(peer); err != nil || string(got) != want {+ t.Fatalf("peer still sees old/different shared keys: %q, %v; want %q", got, err, want)+ }+ }+ if after.Mode().Perm() != before.Mode().Perm() {+ t.Fatalf("remote key permissions changed: %v -> %v", before.Mode(), after.Mode())+ }+ if phase != "success" && phase != "all_managed" && !os.SameFile(before, after) {+ t.Fatal("failed or already-clean target was replaced")+ }+ leftovers, err := filepath.Glob(target + ".pulse-cleanup.*")+ if err != nil || (phase != "cleanup" && len(leftovers) != 0) || (phase == "cleanup" && len(leftovers) != 1) {+ t.Fatalf("unexpected staging cleanup result: %v, %v", leftovers, err)+ }+ })+ }+ }+}++func TestUninstallSensorProxyRemotePayloadFailureReachesMain(t *testing.T) {+ dir := t.TempDir()+ auth := filepath.Join(dir, "remote", "authorized_keys")+ if err := os.MkdirAll(filepath.Dir(auth), 0700); err != nil {+ t.Fatal(err)+ }+ const original = "ssh-ed25519 AAAAold pulse # pulse-managed-key\n" + sensorProxyRecoveryKeys+ if err := os.WriteFile(auth, []byte(original), 0600); err != nil {+ t.Fatal(err)+ }+ body := sensorProxyRemoteKeyFixture + `+pvesh() { [[ "$*" == 'get /nodes --output-format json' ]] || return 90; echo '[{"node":"pve-local"},{"node":"pve-remote"}]'; }+getent() {+ [[ "$1" == ahostsv4 ]] || return 90+ case "$2" in pve-local) echo '127.0.0.1 STREAM pve-local';; pve-remote) echo '192.0.2.11 STREAM pve-remote';; *) return 90;; esac+}+disable_legacy_units() { :; }+cleanup_stale_sensor_proxy_mounts() { :; }+remove_legacy_files() { echo files >>"$FIXTURE_DIR/actions"; }+remove_proxmox_access() { :; }+systemctl_if_available() { :; }+if main; then exit 0; else exit $?; fi+`+ out, code := runSensorProxySafetyShell(t, sensorProxyRemoteKeyScript(t, dir), dir, "mv", body)+ if code == 0 || strings.Contains(out, "[ OK ] Removed legacy Pulse SSH key entries from 192.0.2.11") || strings.Contains(out, "cleanup complete") || !strings.Contains(out, "cluster SSH key cleanup is incomplete") {+ t.Fatalf("main concealed the executed remote failure: exit %d: %s", code, out)+ }+ if got, err := os.ReadFile(filepath.Join(dir, "actions")); err != nil || string(got) != "ssh\nfiles\n" {+ t.Fatalf("complete discovery/remote/local-cleanup route not exercised: %q, %v", got, err)+ }+ if got, err := os.ReadFile(auth); err != nil || string(got) != original {+ t.Fatalf("failed remote replacement changed recovery keys: %q, %v", got, err)+ }+}diff --git a/scripts/uninstall-sensor-proxy.sh b/scripts/uninstall-sensor-proxy.shindex 24b2782fe3..6cfdeecfa4 100755--- a/scripts/uninstall-sensor-proxy.sh+++ b/scripts/uninstall-sensor-proxy.sh@@ -264,11 +264,38 @@ cleanup_remote_authorized_keys() {-o BatchMode=yes-o ConnectTimeout=5)- local remote_cmd='set -eu+ local remote_cmd=""+ # Proxmox commonly links this file into its shared cluster filesystem.+ # Replace the resolved target, never the link, and preserve the original+ # keys if filtering, metadata preservation or replacement fails remotely.+ remote_cmd=$(cat <<'SH'+set -euauth="/root/.ssh/authorized_keys"-if [ -f "$auth" ]; then- sed -i -e "/# pulse-managed-key$/d" -e "/# pulse-proxy-key$/d" "$auth"-fi'+if [ -e "$auth" ] || [ -L "$auth" ]; then+ auth=$(readlink -f -- "$auth") || exit 1+ [ -f "$auth" ] || exit 1+ tmp_file=$(mktemp "${auth}.pulse-cleanup.XXXXXX") || exit 1+ trap 'status=$?; rm -f -- "$tmp_file" || status=1; exit "$status"' EXIT+ trap 'exit 1' HUP INT TERM+ exec 3>"$tmp_file" || exit 1+ status=0+ grep -v -E '# pulse-(managed|proxy)-key$' "$auth" >&3 || status=$?+ exec 3>&-+ [ "$status" -le 1 ] || exit 1+ status=0+ cmp -s "$tmp_file" "$auth" || status=$?+ case "$status" in+ 0) ;;+ 1)+ chmod --reference="$auth" "$tmp_file" &&+ chown --reference="$auth" "$tmp_file" &&+ mv -- "$tmp_file" "$auth" || exit 1+ ;;+ *) exit 1 ;;+ esac+fi+SH+ ) || return 1if [[ -n "$SSH_KNOWN_HOSTS_PATH" ]]; thenknown_hosts_path=$(resolve_path "$SSH_KNOWN_HOSTS_PATH")