Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve prior installer snapshots on retry
pulse-triage[bot]b79496219fe6
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex d476901b62..a8d4d6f5a9 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -8142,6 +8142,36 @@ cache. Proof: `scripts/tests/test_browser_os_packages.py`,`scripts/tests/test_e2e_test_images.py`, which also pins that this cache cannever carry the E2E test images.+### Configuration snapshot publication preserves earlier recovery copies++The root installer's pre-update configuration snapshot must not reuse a+second-resolution destination. Each attempt reserves a private, unique hidden+sibling, copies the configuration contents with metadata and nested links+preserved, and publishes it by a same-directory non-overwriting rename. A file,+directory or dangling link at the final path stops the update, even when GNU+`mv -n` returns zero after skipping. Copy/allocation/publication failure removes+only the invocation's incomplete stage and never rotates earlier snapshots.++Incomplete hidden stages are not retention candidates. After publication,+rotation still keeps the configured number of snapshots (five by default),+including the current invocation's completed copy for positive retention counts+after same-second retries or a backwards clock step. Zero still disables+completed-snapshot retention. Legacy timestamp-only snapshots remain eligible for+ordinary rotation, not rewritten or treated as incomplete. The existing+read-only-parent fallback and disk-headroom admission remain. Discard after a+pre-mutation staging failure names only this invocation's completed snapshot.+All three existing-install update/reinstall entries explicitly stop if the+backup fails, including when their caller suppresses implicit shell `errexit`.++`scripts/tests/test_server_config_backups.py` exercises real fixture files,+copy/rename/retention, metadata and links; injected failures cover allocation,+copy and publication, including exit-zero late collisions. It checks rapid+retries, clock reversal, another invocation's retention, fallback, owned discard+and the actual three update entries. The discovered Go installer suite executes+it on the existing repository test routes. These are disposable filesystem+and caller controls, not application-consistent backup, power-loss durability,+installed upgrade/rollback acceptance or the original guest-thaw recovery.+## Release-body updater ownership (30 September 2026)Each executable `/bin/update --version` example in a published body must carrydiff --git a/install.sh b/install.shindex 24f406711c..5e0eba4dd6 100644--- a/install.sh+++ b/install.sh@@ -3060,19 +3060,25 @@ create_user() {prune_config_backups() {# Unattended updates create one snapshot per run; without rotation they# fill small root filesystems (issue #1646). Keep the newest snapshots- # and delete the rest. Snapshot names embed a sortable timestamp, so a- # lexicographic sort orders them oldest-first.+ # and delete the rest. Snapshot names embed a sortable timestamp. Positive+ # retention keeps this invocation's completed snapshot, even after a clock step+ # backwards or when random suffixes share the same timestamp.local backup_parent="$1"local snapshot_prefix="$2"local keep="${CONFIG_BACKUP_KEEP_COUNT:-5}"+ local current_backup="${3:-}" current_count=0local backups=()local entrywhile IFS= read -r entry; do+ if [[ "$entry" == "$current_backup" ]] && (( keep > 0 )); then+ current_count=1+ continue+ fi[[ -n "$entry" ]] && backups+=("$entry")done < <(find "$backup_parent" -maxdepth 1 -mindepth 1 -type d -name "${snapshot_prefix}*" 2>/dev/null | sort)- local excess=$(( ${#backups[@]} - keep ))+ local excess=$(( ${#backups[@]} + current_count - keep ))if (( excess <= 0 )); thenreturn 0fi@@ -3086,9 +3092,10 @@ prune_config_backups() {}backup_existing() {+ LAST_CONFIG_BACKUP_DIR=""if [[ -d "$CONFIG_DIR" ]]; thenprint_info "Backing up existing configuration..."- local backup_parent backup_dir+ local backup_parent backup_dir backup_stage stage_namebackup_parent="$(dirname "$CONFIG_DIR")"if [[ ! -w "$backup_parent" ]]; then# The hardened pulse-update.service unit mounts /etc read-only@@ -3098,13 +3105,31 @@ backup_existing() {mkdir -p "$backup_parent" || return 1fiensure_config_backup_headroom "$CONFIG_DIR" "$backup_parent" || return 1- backup_dir="$backup_parent/$(basename "$CONFIG_DIR").backup.$(date +%Y%m%d-%H%M%S)"- if ! cp -a "$CONFIG_DIR" "$backup_dir"; then- rm -rf "$backup_dir"+ # Reserve private work before copying. A second-resolution destination+ # can already be another run's recovery copy: cp would nest into it,+ # and failed-copy cleanup would then erase that earlier snapshot.+ # Hidden, incomplete copies are not candidates for snapshot rotation.+ if ! backup_stage=$(mktemp -d -- "$backup_parent/.$(basename "$CONFIG_DIR").backup.$(date +%Y%m%d-%H%M%S).XXXXXX"); then+ print_error "Could not create configuration backup; existing snapshots are unchanged"+ return 1+ fi+ stage_name="${backup_stage##*/}"+ backup_dir="$backup_parent/${stage_name#.}"+ if ! cp -a -- "$CONFIG_DIR/." "$backup_stage/"; then+ rm -rf -- "$backup_stage"+ print_error "Configuration backup failed; existing snapshots are unchanged"+ return 1+ fi+ # GNU mv -n can return zero without moving anything. Admit publication+ # only when our stage actually disappeared; never merge or overwrite a+ # file, directory or link that appeared at the final path.+ if ! mv -T -n -- "$backup_stage" "$backup_dir" || [[ -e "$backup_stage" || -L "$backup_stage" ]]; then+ rm -rf -- "$backup_stage"+ print_error "Could not publish configuration backup; existing snapshots are unchanged"return 1fiLAST_CONFIG_BACKUP_DIR="$backup_dir"- prune_config_backups "$backup_parent" "$(basename "$CONFIG_DIR").backup."+ prune_config_backups "$backup_parent" "$(basename "$CONFIG_DIR").backup." "$backup_dir"fi}@@ -5284,7 +5309,7 @@ main() {exit 1fi- backup_existing+ backup_existing || return 1create_userdownload_pulse# A half-removed installation (binary present, /etc/pulse or the@@ -5490,7 +5515,7 @@ main() {exit 1fi- backup_existing+ backup_existing || return 1create_userdownload_pulse# Same repair as the --version path: a half-removed@@ -5522,7 +5547,7 @@ main() {LATEST_RELEASE="$CURRENT_VERSION"offer_existing_auto_updates- backup_existing+ backup_existing || return 1create_userdownload_pulsesetup_directoriesdiff --git a/scripts/installtests/root_install_sh_test.go b/scripts/installtests/root_install_sh_test.goindex b6d051f1f1..b55a32a7ed 100644--- a/scripts/installtests/root_install_sh_test.go+++ b/scripts/installtests/root_install_sh_test.go@@ -365,9 +365,8 @@ func TestRootInstallScriptConfigBackupCleansPartialCopy(t *testing.T) {` + extractRootInstallShellFunction(t, "backup_existing") + `date() { printf '20260704-141422\n'; }cp() {- if [[ "$1" == "-a" && "$2" == "$CONFIG_DIR_UNDER_TEST" ]]; then- mkdir -p "$3"- printf 'partial\n' > "$3/partial"+ if [[ "$1" == "-a" && "$3" == "$CONFIG_DIR_UNDER_TEST/." ]]; then+ printf 'partial\n' > "$4/partial"return 1ficommand cp "$@"@@ -376,7 +375,8 @@ func TestRootInstallScriptConfigBackupCleansPartialCopy(t *testing.T) {echo "backup_existing unexpectedly passed after a failed copy" >&2exit 1fi- if [[ -e "$BACKUP_DIR_UNDER_TEST" ]]; then+ if compgen -G "$CONFIG_DIR_UNDER_TEST.backup.*" >/dev/null ||+ compgen -G "$(dirname "$CONFIG_DIR_UNDER_TEST")/.$(basename "$CONFIG_DIR_UNDER_TEST").backup.*" >/dev/null; thenecho "partial backup was not removed" >&2exit 1fi@@ -465,7 +465,7 @@ func TestRootInstallScriptConfigBackupRotatesOldSnapshots(t *testing.T) {}found := falsefor _, name := range names {- if name == prefix+"20260706-020000" {+ if strings.HasPrefix(name, prefix+"20260706-020000.") {found = true}}@@ -2720,3 +2720,43 @@ func TestRootInstallScriptPulseOwnershipExcludesServerStaging(t *testing.T) {t.Fatalf("ownership calls:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))}}++func TestRootInstallScriptConfigBackupZeroRetention(t *testing.T) {+ configDir := filepath.Join(t.TempDir(), "config")+ if err := os.Mkdir(configDir, 0700); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(configDir, "settings.json"), []byte("retained source\n"), 0600); err != nil {+ t.Fatal(err)+ }+ script := `set -euo pipefail+print_info() { :; }; print_warn() { :; }; print_error() { :; }+ensure_config_backup_headroom() { :; }+CONFIG_BACKUP_KEEP_COUNT=0+` + extractRootInstallShellFunction(t, "prune_config_backups") + "\n" ++ extractRootInstallShellFunction(t, "backup_existing") + `+backup_existing+test -f "$CONFIG_DIR/settings.json"+if compgen -G "$CONFIG_DIR.backup.*" >/dev/null; then+ echo "zero retention kept a completed snapshot" >&2+ exit 1+fi+`+ cmd := exec.Command("bash", "-c", script)+ cmd.Env = append(os.Environ(), "CONFIG_DIR="+configDir)+ out, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("zero snapshot retention: %v\n%s", err, out)+ }+}++func TestRootInstallScriptConfigBackupPublicationPreservesPriorSnapshots(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_server_config_backups.py"), "-v")+ cmd.Env = append(os.Environ(), "PULSE_INSTALLER_UNDER_TEST="+repoFile("install.sh"))+ out, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("configuration backup publication controls: %v\n%s", err, out)+ }+}diff --git a/scripts/tests/test_server_config_backups.py b/scripts/tests/test_server_config_backups.pynew file mode 100644index 0000000000..179e208e23--- /dev/null+++ b/scripts/tests/test_server_config_backups.py@@ -0,0 +1,274 @@+#!/usr/bin/env python3+"""Run real configuration snapshot copy/publication against private fixtures.++Only the clock, disk-capacity observation and injected command failures are+controlled. No service, database, production config or network is touched.+These checks do not establish application-consistent backup or native recovery.+"""++import os+from pathlib import Path+import stat+import subprocess+import tempfile+import unittest+++ROOT = Path(__file__).resolve().parents[2]+INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh")).resolve()+STAMP = "20261023-154500"++PRELUDE = r'''+source "$INSTALLER"+print_info() { :; }+print_warn() { echo "$*" >&2; }+print_error() { echo "$*" >&2; }+ensure_config_backup_headroom() { return "${HEADROOM_EXIT:-0}"; }+date() { printf '%s\n' "$STAMP"; }+'''+++class ConfigBackupPublication(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory(prefix="pulse-config-backup-")+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name)+ self.config = self.root / "source parent" / "config with spaces"+ self.config.mkdir(parents=True, mode=0o700)+ (self.config / "settings.json").write_text("current state\n")+ (self.config / ".env").write_text("private synthetic setting\n")+ (self.config / ".env").chmod(0o600)+ (self.config / "empty").mkdir()+ (self.config / "setting-link").symlink_to("settings.json")+ self.install = self.root / "install"+ self.install.mkdir()++ def old_snapshot(self, stamp=STAMP):+ path = self.config.parent / (self.config.name + ".backup." + stamp)+ path.mkdir()+ (path / "settings.json").write_text("prior recovery state\n")+ return path++ def shell(self, body, **extra):+ env = {k: v for k, v in os.environ.items() if not k.startswith("PULSE_")}+ env.update(INSTALLER=str(INSTALLER), FIXTURE=str(self.root), STAMP=STAMP,+ PULSE_INSTALL_DIR=str(self.install), PULSE_CONFIG_DIR=str(self.config))+ env.update(extra)+ result = subprocess.run(["bash", "-c", PRELUDE + body], cwd=self.root,+ env=env, capture_output=True, text=True, timeout=10)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ return result++ def completed(self):+ return sorted(self.config.parent.glob(self.config.name + ".backup.*"))++ def assert_no_stages(self):+ self.assertEqual(list(self.config.parent.glob("." + self.config.name + ".backup.*")), [])++ def assert_prior(self, old):+ self.assertEqual((old / "settings.json").read_text(), "prior recovery state\n")+ self.assertEqual(sorted(p.name for p in old.iterdir()), ["settings.json"])++ def test_two_successful_same_second_copies_are_separate_complete_snapshots(self):+ old = self.old_snapshot()+ self.shell(r'''+backup_existing+printf '%s\n' "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/first"+printf 'second state\n' > "$CONFIG_DIR/settings.json"+backup_existing+printf '%s\n' "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/second"+''')+ first = Path((self.root / "first").read_text().strip())+ second = Path((self.root / "second").read_text().strip())+ self.assertNotEqual(first, second)+ self.assertEqual((first / "settings.json").read_text(), "current state\n")+ self.assertEqual((second / "settings.json").read_text(), "second state\n")+ self.assertEqual(len(self.completed()), 3)+ for snapshot in (first, second):+ self.assertEqual((snapshot / ".env").read_bytes(), (self.config / ".env").read_bytes())+ self.assertEqual(stat.S_IMODE((snapshot / ".env").stat().st_mode), 0o600)+ self.assertEqual(stat.S_IMODE(snapshot.stat().st_mode), 0o700)+ self.assertTrue((snapshot / "empty").is_dir())+ self.assertTrue((snapshot / "setting-link").is_symlink())+ self.assertEqual(os.readlink(snapshot / "setting-link"), "settings.json")+ self.assert_prior(old)+ self.assert_no_stages()++ def test_failed_partial_copy_does_not_mutate_or_delete_a_previous_snapshot(self):+ old = self.old_snapshot()+ result = self.shell(r'''+LAST_CONFIG_BACKUP_DIR="$CONFIG_DIR.backup.$STAMP"+cp() { command cp "$@"; return 23; }+if backup_existing; then exit 90; fi+test -z "$LAST_CONFIG_BACKUP_DIR"+''')+ self.assertIn("existing snapshots are unchanged", result.stderr)+ self.assert_prior(old)+ self.assertEqual(self.completed(), [old])+ self.assert_no_stages()++ def test_allocation_failure_stops_before_copy_or_rotation(self):+ old = self.old_snapshot()+ self.shell(r'''+mktemp() { return 23; }+cp() { echo unexpected-copy > "$FIXTURE/unexpected"; return 97; }+prune_config_backups() { echo unexpected-prune > "$FIXTURE/unexpected"; }+if backup_existing; then exit 90; fi+test -z "$LAST_CONFIG_BACKUP_DIR"+''')+ self.assertFalse((self.root / "unexpected").exists())+ self.assert_prior(old)+ self.assert_no_stages()++ def test_headroom_failure_creates_no_stage_and_preserves_prior_snapshot(self):+ old = self.old_snapshot()+ self.shell('if backup_existing; then exit 90; fi\ntest -z "$LAST_CONFIG_BACKUP_DIR"\n', HEADROOM_EXIT="23")+ self.assert_prior(old)+ self.assert_no_stages()++ def test_failed_publication_does_not_rotate_or_publish_a_snapshot(self):+ old = self.old_snapshot()+ self.shell(r'''+mv() { return 23; }+prune_config_backups() { echo unexpected-prune > "$FIXTURE/unexpected"; }+if backup_existing; then exit 90; fi+test -z "$LAST_CONFIG_BACKUP_DIR"+''')+ self.assertFalse((self.root / "unexpected").exists())+ self.assertEqual(self.completed(), [old])+ self.assert_prior(old)+ self.assert_no_stages()++ def test_exit_zero_skipped_publication_preserves_file_directory_and_link(self):+ for kind in ("file", "directory", "link"):+ with self.subTest(kind=kind):+ old = self.old_snapshot("collision-" + kind)+ result = self.shell(r'''+mv() {+ local destination="${@: -1}"+ printf '%s\n' "$destination" > "$FIXTURE/collision"+ case "$KIND" in+ file) printf 'do not replace\n' > "$destination" ;;+ directory) mkdir "$destination"; printf 'do not replace\n' > "$destination/marker" ;;+ link) ln -s "$FIXTURE/absent-target" "$destination" ;;+ esac+ command mv "$@"+}+if backup_existing; then exit 90; fi+test -z "$LAST_CONFIG_BACKUP_DIR"+''', KIND=kind)+ collision = Path((self.root / "collision").read_text().strip())+ self.assertIn("Could not publish configuration backup", result.stderr)+ if kind == "file":+ self.assertEqual(collision.read_text(), "do not replace\n")+ collision.unlink()+ elif kind == "directory":+ self.assertEqual([p.name for p in collision.iterdir()], ["marker"])+ self.assertEqual((collision / "marker").read_text(), "do not replace\n")+ (collision / "marker").unlink()+ collision.rmdir()+ else:+ self.assertTrue(collision.is_symlink())+ self.assertEqual(os.readlink(collision), str(self.root / "absent-target"))+ collision.unlink()+ self.assert_prior(old)+ self.assert_no_stages()++ def test_current_snapshot_survives_rotation_even_after_clock_moves_backwards(self):+ for day in range(24, 29):+ self.old_snapshot(f"202610{day}-154500")+ self.shell('backup_existing\nprintf "%s\\n" "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/latest"\n')+ latest = Path((self.root / "latest").read_text().strip())+ self.assertTrue(latest.is_dir())+ self.assertEqual((latest / "settings.json").read_text(), "current state\n")+ self.assertEqual(len(self.completed()), 5)+ self.assertFalse((self.config.parent / (self.config.name + ".backup.20261024-154500")).exists())+ self.assert_no_stages()++ def test_incomplete_copy_is_not_another_invocations_rotation_candidate(self):+ self.shell(r'''+cp() {+ local stage="${@: -1}" peer="$CONFIG_DIR.backup.20261024-154500.peer"+ test -d "$stage" || return 91+ mkdir "$peer"+ CONFIG_BACKUP_KEEP_COUNT=1 prune_config_backups "$(dirname "$CONFIG_DIR")" "$(basename "$CONFIG_DIR").backup." "$peer"+ test -d "$stage" || return 92+ command cp "$@"+}+backup_existing+test -f "$LAST_CONFIG_BACKUP_DIR/settings.json"+''')+ self.assert_no_stages()++ def test_discard_removes_only_this_invocations_new_snapshot(self):+ old = self.old_snapshot()+ self.shell(r'''+backup_existing+current="$LAST_CONFIG_BACKUP_DIR"+discard_last_config_backup+test -z "$LAST_CONFIG_BACKUP_DIR"+test ! -e "$current"+''')+ self.assert_prior(old)+ self.assertEqual(self.completed(), [old])+ self.assert_no_stages()++ def test_missing_config_clears_stale_ownership_without_deleting_prior_snapshot(self):+ old = self.old_snapshot()+ self.shell(r'''+LAST_CONFIG_BACKUP_DIR="$CONFIG_DIR.backup.$STAMP"+CONFIG_DIR="$FIXTURE/absent-config"+backup_existing+test -z "$LAST_CONFIG_BACKUP_DIR"+discard_last_config_backup+''')+ self.assert_prior(old)+ self.assert_no_stages()++ def test_read_only_config_parent_uses_the_existing_hardened_unit_fallback(self):+ if os.geteuid() == 0:+ self.skipTest("root bypasses ordinary write permissions")+ self.config.parent.chmod(0o500)+ try:+ self.shell('backup_existing\nprintf "%s\\n" "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/latest"\n')+ latest = Path((self.root / "latest").read_text().strip())+ self.assertEqual(latest.parent, self.install / "config-backups")+ self.assertEqual((latest / "settings.json").read_text(), "current state\n")+ self.assertEqual(list(latest.parent.glob(".*.backup.*")), [])+ finally:+ self.config.parent.chmod(0o700)++ def test_every_update_entry_stops_on_failed_backup_even_in_an_or_list(self):+ for flow in ("pinned", "menu-update", "menu-reinstall"):+ with self.subTest(flow=flow):+ result = self.shell(r'''+CURRENT_VERSION=v6.5.0+FORCE_VERSION=""+FORCE_CHANNEL=stable+IN_CONTAINER=true+IN_DOCKER=false+[[ "$FLOW" != pinned ]] || FORCE_VERSION=v6.6.0+print_header() { :; }+check_root() { :; }+detect_os() { :; }+check_docker_environment() { :; }+check_existing_installation() { return 0; }+check_proxmox_host() { return 1; }+resolve_latest_release_tag_for_channel() { printf 'v6.6.0\n'; }+detect_service_name() { printf 'pulse\n'; }+safe_read() { local selection=1; [[ "$FLOW" != menu-reinstall ]] || selection=2; printf -v "$2" '%s' "$selection"; }+offer_existing_auto_updates() { :; }+run_upgrade_readiness_preflight() { :; }+backup_existing() { printf 'backup refused\n' >&2; return 23; }+create_user() { echo user > "$FIXTURE/unexpected"; }+download_pulse() { echo download > "$FIXTURE/unexpected"; exit 0; }+status=0+main || status=$?+test "$status" = 1+''', FLOW=flow)+ self.assertIn("backup refused", result.stderr)+ self.assertFalse((self.root / "unexpected").exists())+++if __name__ == "__main__":+ unittest.main()