Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve prior installer snapshots on retry

pulse-triage[bot]b79496219fe6
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index d476901b62..a8d4d6f5a9 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -8142,6 +8142,36 @@ cache. Proof: `scripts/tests/test_browser_os_packages.py`,
`scripts/tests/test_e2e_test_images.py`, which also pins that this cache can
never carry the E2E test images.
+### Configuration snapshot publication preserves earlier recovery copies
+
+The root installer's pre-update configuration snapshot must not reuse a
+second-resolution destination. Each attempt reserves a private, unique hidden
+sibling, copies the configuration contents with metadata and nested links
+preserved, and publishes it by a same-directory non-overwriting rename. A file,
+directory or dangling link at the final path stops the update, even when GNU
+`mv -n` returns zero after skipping. Copy/allocation/publication failure removes
+only the invocation's incomplete stage and never rotates earlier snapshots.
+
+Incomplete hidden stages are not retention candidates. After publication,
+rotation still keeps the configured number of snapshots (five by default),
+including the current invocation's completed copy for positive retention counts
+after same-second retries or a backwards clock step. Zero still disables
+completed-snapshot retention. Legacy timestamp-only snapshots remain eligible for
+ordinary rotation, not rewritten or treated as incomplete. The existing
+read-only-parent fallback and disk-headroom admission remain. Discard after a
+pre-mutation staging failure names only this invocation's completed snapshot.
+All three existing-install update/reinstall entries explicitly stop if the
+backup fails, including when their caller suppresses implicit shell `errexit`.
+
+`scripts/tests/test_server_config_backups.py` exercises real fixture files,
+copy/rename/retention, metadata and links; injected failures cover allocation,
+copy and publication, including exit-zero late collisions. It checks rapid
+retries, clock reversal, another invocation's retention, fallback, owned discard
+and the actual three update entries. The discovered Go installer suite executes
+it on the existing repository test routes. These are disposable filesystem
+and caller controls, not application-consistent backup, power-loss durability,
+installed upgrade/rollback acceptance or the original guest-thaw recovery.
+
## Release-body updater ownership (30 September 2026)
Each executable `/bin/update --version` example in a published body must carry
diff --git a/install.sh b/install.sh
index 24f406711c..5e0eba4dd6 100644
--- a/install.sh
+++ b/install.sh
@@ -3060,19 +3060,25 @@ create_user() {
prune_config_backups() {
# Unattended updates create one snapshot per run; without rotation they
# fill small root filesystems (issue #1646). Keep the newest snapshots
- # and delete the rest. Snapshot names embed a sortable timestamp, so a
- # lexicographic sort orders them oldest-first.
+ # and delete the rest. Snapshot names embed a sortable timestamp. Positive
+ # retention keeps this invocation's completed snapshot, even after a clock step
+ # backwards or when random suffixes share the same timestamp.
local backup_parent="$1"
local snapshot_prefix="$2"
local keep="${CONFIG_BACKUP_KEEP_COUNT:-5}"
+ local current_backup="${3:-}" current_count=0
local backups=()
local entry
while IFS= read -r entry; do
+ if [[ "$entry" == "$current_backup" ]] && (( keep > 0 )); then
+ current_count=1
+ continue
+ fi
[[ -n "$entry" ]] && backups+=("$entry")
done < <(find "$backup_parent" -maxdepth 1 -mindepth 1 -type d -name "${snapshot_prefix}*" 2>/dev/null | sort)
- local excess=$(( ${#backups[@]} - keep ))
+ local excess=$(( ${#backups[@]} + current_count - keep ))
if (( excess <= 0 )); then
return 0
fi
@@ -3086,9 +3092,10 @@ prune_config_backups() {
}
backup_existing() {
+ LAST_CONFIG_BACKUP_DIR=""
if [[ -d "$CONFIG_DIR" ]]; then
print_info "Backing up existing configuration..."
- local backup_parent backup_dir
+ local backup_parent backup_dir backup_stage stage_name
backup_parent="$(dirname "$CONFIG_DIR")"
if [[ ! -w "$backup_parent" ]]; then
# The hardened pulse-update.service unit mounts /etc read-only
@@ -3098,13 +3105,31 @@ backup_existing() {
mkdir -p "$backup_parent" || return 1
fi
ensure_config_backup_headroom "$CONFIG_DIR" "$backup_parent" || return 1
- backup_dir="$backup_parent/$(basename "$CONFIG_DIR").backup.$(date +%Y%m%d-%H%M%S)"
- if ! cp -a "$CONFIG_DIR" "$backup_dir"; then
- rm -rf "$backup_dir"
+ # Reserve private work before copying. A second-resolution destination
+ # can already be another run's recovery copy: cp would nest into it,
+ # and failed-copy cleanup would then erase that earlier snapshot.
+ # Hidden, incomplete copies are not candidates for snapshot rotation.
+ if ! backup_stage=$(mktemp -d -- "$backup_parent/.$(basename "$CONFIG_DIR").backup.$(date +%Y%m%d-%H%M%S).XXXXXX"); then
+ print_error "Could not create configuration backup; existing snapshots are unchanged"
+ return 1
+ fi
+ stage_name="${backup_stage##*/}"
+ backup_dir="$backup_parent/${stage_name#.}"
+ if ! cp -a -- "$CONFIG_DIR/." "$backup_stage/"; then
+ rm -rf -- "$backup_stage"
+ print_error "Configuration backup failed; existing snapshots are unchanged"
+ return 1
+ fi
+ # GNU mv -n can return zero without moving anything. Admit publication
+ # only when our stage actually disappeared; never merge or overwrite a
+ # file, directory or link that appeared at the final path.
+ if ! mv -T -n -- "$backup_stage" "$backup_dir" || [[ -e "$backup_stage" || -L "$backup_stage" ]]; then
+ rm -rf -- "$backup_stage"
+ print_error "Could not publish configuration backup; existing snapshots are unchanged"
return 1
fi
LAST_CONFIG_BACKUP_DIR="$backup_dir"
- prune_config_backups "$backup_parent" "$(basename "$CONFIG_DIR").backup."
+ prune_config_backups "$backup_parent" "$(basename "$CONFIG_DIR").backup." "$backup_dir"
fi
}
@@ -5284,7 +5309,7 @@ main() {
exit 1
fi
- backup_existing
+ backup_existing || return 1
create_user
download_pulse
# A half-removed installation (binary present, /etc/pulse or the
@@ -5490,7 +5515,7 @@ main() {
exit 1
fi
- backup_existing
+ backup_existing || return 1
create_user
download_pulse
# Same repair as the --version path: a half-removed
@@ -5522,7 +5547,7 @@ main() {
LATEST_RELEASE="$CURRENT_VERSION"
offer_existing_auto_updates
- backup_existing
+ backup_existing || return 1
create_user
download_pulse
setup_directories
diff --git a/scripts/installtests/root_install_sh_test.go b/scripts/installtests/root_install_sh_test.go
index b6d051f1f1..b55a32a7ed 100644
--- a/scripts/installtests/root_install_sh_test.go
+++ b/scripts/installtests/root_install_sh_test.go
@@ -365,9 +365,8 @@ func TestRootInstallScriptConfigBackupCleansPartialCopy(t *testing.T) {
` + extractRootInstallShellFunction(t, "backup_existing") + `
date() { printf '20260704-141422\n'; }
cp() {
- if [[ "$1" == "-a" && "$2" == "$CONFIG_DIR_UNDER_TEST" ]]; then
- mkdir -p "$3"
- printf 'partial\n' > "$3/partial"
+ if [[ "$1" == "-a" && "$3" == "$CONFIG_DIR_UNDER_TEST/." ]]; then
+ printf 'partial\n' > "$4/partial"
return 1
fi
command cp "$@"
@@ -376,7 +375,8 @@ func TestRootInstallScriptConfigBackupCleansPartialCopy(t *testing.T) {
echo "backup_existing unexpectedly passed after a failed copy" >&2
exit 1
fi
- if [[ -e "$BACKUP_DIR_UNDER_TEST" ]]; then
+ if compgen -G "$CONFIG_DIR_UNDER_TEST.backup.*" >/dev/null ||
+ compgen -G "$(dirname "$CONFIG_DIR_UNDER_TEST")/.$(basename "$CONFIG_DIR_UNDER_TEST").backup.*" >/dev/null; then
echo "partial backup was not removed" >&2
exit 1
fi
@@ -465,7 +465,7 @@ func TestRootInstallScriptConfigBackupRotatesOldSnapshots(t *testing.T) {
}
found := false
for _, name := range names {
- if name == prefix+"20260706-020000" {
+ if strings.HasPrefix(name, prefix+"20260706-020000.") {
found = true
}
}
@@ -2720,3 +2720,43 @@ func TestRootInstallScriptPulseOwnershipExcludesServerStaging(t *testing.T) {
t.Fatalf("ownership calls:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
}
+
+func TestRootInstallScriptConfigBackupZeroRetention(t *testing.T) {
+ configDir := filepath.Join(t.TempDir(), "config")
+ if err := os.Mkdir(configDir, 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(configDir, "settings.json"), []byte("retained source\n"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ script := `set -euo pipefail
+print_info() { :; }; print_warn() { :; }; print_error() { :; }
+ensure_config_backup_headroom() { :; }
+CONFIG_BACKUP_KEEP_COUNT=0
+` + extractRootInstallShellFunction(t, "prune_config_backups") + "\n" +
+ extractRootInstallShellFunction(t, "backup_existing") + `
+backup_existing
+test -f "$CONFIG_DIR/settings.json"
+if compgen -G "$CONFIG_DIR.backup.*" >/dev/null; then
+ echo "zero retention kept a completed snapshot" >&2
+ exit 1
+fi
+`
+ cmd := exec.Command("bash", "-c", script)
+ cmd.Env = append(os.Environ(), "CONFIG_DIR="+configDir)
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("zero snapshot retention: %v\n%s", err, out)
+ }
+}
+
+func TestRootInstallScriptConfigBackupPublicationPreservesPriorSnapshots(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_server_config_backups.py"), "-v")
+ cmd.Env = append(os.Environ(), "PULSE_INSTALLER_UNDER_TEST="+repoFile("install.sh"))
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("configuration backup publication controls: %v\n%s", err, out)
+ }
+}
diff --git a/scripts/tests/test_server_config_backups.py b/scripts/tests/test_server_config_backups.py
new file mode 100644
index 0000000000..179e208e23
--- /dev/null
+++ b/scripts/tests/test_server_config_backups.py
@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""Run real configuration snapshot copy/publication against private fixtures.
+
+Only the clock, disk-capacity observation and injected command failures are
+controlled. No service, database, production config or network is touched.
+These checks do not establish application-consistent backup or native recovery.
+"""
+
+import os
+from pathlib import Path
+import stat
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh")).resolve()
+STAMP = "20261023-154500"
+
+PRELUDE = r'''
+source "$INSTALLER"
+print_info() { :; }
+print_warn() { echo "$*" >&2; }
+print_error() { echo "$*" >&2; }
+ensure_config_backup_headroom() { return "${HEADROOM_EXIT:-0}"; }
+date() { printf '%s\n' "$STAMP"; }
+'''
+
+
+class ConfigBackupPublication(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory(prefix="pulse-config-backup-")
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name)
+ self.config = self.root / "source parent" / "config with spaces"
+ self.config.mkdir(parents=True, mode=0o700)
+ (self.config / "settings.json").write_text("current state\n")
+ (self.config / ".env").write_text("private synthetic setting\n")
+ (self.config / ".env").chmod(0o600)
+ (self.config / "empty").mkdir()
+ (self.config / "setting-link").symlink_to("settings.json")
+ self.install = self.root / "install"
+ self.install.mkdir()
+
+ def old_snapshot(self, stamp=STAMP):
+ path = self.config.parent / (self.config.name + ".backup." + stamp)
+ path.mkdir()
+ (path / "settings.json").write_text("prior recovery state\n")
+ return path
+
+ def shell(self, body, **extra):
+ env = {k: v for k, v in os.environ.items() if not k.startswith("PULSE_")}
+ env.update(INSTALLER=str(INSTALLER), FIXTURE=str(self.root), STAMP=STAMP,
+ PULSE_INSTALL_DIR=str(self.install), PULSE_CONFIG_DIR=str(self.config))
+ env.update(extra)
+ result = subprocess.run(["bash", "-c", PRELUDE + body], cwd=self.root,
+ env=env, capture_output=True, text=True, timeout=10)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ return result
+
+ def completed(self):
+ return sorted(self.config.parent.glob(self.config.name + ".backup.*"))
+
+ def assert_no_stages(self):
+ self.assertEqual(list(self.config.parent.glob("." + self.config.name + ".backup.*")), [])
+
+ def assert_prior(self, old):
+ self.assertEqual((old / "settings.json").read_text(), "prior recovery state\n")
+ self.assertEqual(sorted(p.name for p in old.iterdir()), ["settings.json"])
+
+ def test_two_successful_same_second_copies_are_separate_complete_snapshots(self):
+ old = self.old_snapshot()
+ self.shell(r'''
+backup_existing
+printf '%s\n' "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/first"
+printf 'second state\n' > "$CONFIG_DIR/settings.json"
+backup_existing
+printf '%s\n' "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/second"
+''')
+ first = Path((self.root / "first").read_text().strip())
+ second = Path((self.root / "second").read_text().strip())
+ self.assertNotEqual(first, second)
+ self.assertEqual((first / "settings.json").read_text(), "current state\n")
+ self.assertEqual((second / "settings.json").read_text(), "second state\n")
+ self.assertEqual(len(self.completed()), 3)
+ for snapshot in (first, second):
+ self.assertEqual((snapshot / ".env").read_bytes(), (self.config / ".env").read_bytes())
+ self.assertEqual(stat.S_IMODE((snapshot / ".env").stat().st_mode), 0o600)
+ self.assertEqual(stat.S_IMODE(snapshot.stat().st_mode), 0o700)
+ self.assertTrue((snapshot / "empty").is_dir())
+ self.assertTrue((snapshot / "setting-link").is_symlink())
+ self.assertEqual(os.readlink(snapshot / "setting-link"), "settings.json")
+ self.assert_prior(old)
+ self.assert_no_stages()
+
+ def test_failed_partial_copy_does_not_mutate_or_delete_a_previous_snapshot(self):
+ old = self.old_snapshot()
+ result = self.shell(r'''
+LAST_CONFIG_BACKUP_DIR="$CONFIG_DIR.backup.$STAMP"
+cp() { command cp "$@"; return 23; }
+if backup_existing; then exit 90; fi
+test -z "$LAST_CONFIG_BACKUP_DIR"
+''')
+ self.assertIn("existing snapshots are unchanged", result.stderr)
+ self.assert_prior(old)
+ self.assertEqual(self.completed(), [old])
+ self.assert_no_stages()
+
+ def test_allocation_failure_stops_before_copy_or_rotation(self):
+ old = self.old_snapshot()
+ self.shell(r'''
+mktemp() { return 23; }
+cp() { echo unexpected-copy > "$FIXTURE/unexpected"; return 97; }
+prune_config_backups() { echo unexpected-prune > "$FIXTURE/unexpected"; }
+if backup_existing; then exit 90; fi
+test -z "$LAST_CONFIG_BACKUP_DIR"
+''')
+ self.assertFalse((self.root / "unexpected").exists())
+ self.assert_prior(old)
+ self.assert_no_stages()
+
+ def test_headroom_failure_creates_no_stage_and_preserves_prior_snapshot(self):
+ old = self.old_snapshot()
+ self.shell('if backup_existing; then exit 90; fi\ntest -z "$LAST_CONFIG_BACKUP_DIR"\n', HEADROOM_EXIT="23")
+ self.assert_prior(old)
+ self.assert_no_stages()
+
+ def test_failed_publication_does_not_rotate_or_publish_a_snapshot(self):
+ old = self.old_snapshot()
+ self.shell(r'''
+mv() { return 23; }
+prune_config_backups() { echo unexpected-prune > "$FIXTURE/unexpected"; }
+if backup_existing; then exit 90; fi
+test -z "$LAST_CONFIG_BACKUP_DIR"
+''')
+ self.assertFalse((self.root / "unexpected").exists())
+ self.assertEqual(self.completed(), [old])
+ self.assert_prior(old)
+ self.assert_no_stages()
+
+ def test_exit_zero_skipped_publication_preserves_file_directory_and_link(self):
+ for kind in ("file", "directory", "link"):
+ with self.subTest(kind=kind):
+ old = self.old_snapshot("collision-" + kind)
+ result = self.shell(r'''
+mv() {
+ local destination="${@: -1}"
+ printf '%s\n' "$destination" > "$FIXTURE/collision"
+ case "$KIND" in
+ file) printf 'do not replace\n' > "$destination" ;;
+ directory) mkdir "$destination"; printf 'do not replace\n' > "$destination/marker" ;;
+ link) ln -s "$FIXTURE/absent-target" "$destination" ;;
+ esac
+ command mv "$@"
+}
+if backup_existing; then exit 90; fi
+test -z "$LAST_CONFIG_BACKUP_DIR"
+''', KIND=kind)
+ collision = Path((self.root / "collision").read_text().strip())
+ self.assertIn("Could not publish configuration backup", result.stderr)
+ if kind == "file":
+ self.assertEqual(collision.read_text(), "do not replace\n")
+ collision.unlink()
+ elif kind == "directory":
+ self.assertEqual([p.name for p in collision.iterdir()], ["marker"])
+ self.assertEqual((collision / "marker").read_text(), "do not replace\n")
+ (collision / "marker").unlink()
+ collision.rmdir()
+ else:
+ self.assertTrue(collision.is_symlink())
+ self.assertEqual(os.readlink(collision), str(self.root / "absent-target"))
+ collision.unlink()
+ self.assert_prior(old)
+ self.assert_no_stages()
+
+ def test_current_snapshot_survives_rotation_even_after_clock_moves_backwards(self):
+ for day in range(24, 29):
+ self.old_snapshot(f"202610{day}-154500")
+ self.shell('backup_existing\nprintf "%s\\n" "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/latest"\n')
+ latest = Path((self.root / "latest").read_text().strip())
+ self.assertTrue(latest.is_dir())
+ self.assertEqual((latest / "settings.json").read_text(), "current state\n")
+ self.assertEqual(len(self.completed()), 5)
+ self.assertFalse((self.config.parent / (self.config.name + ".backup.20261024-154500")).exists())
+ self.assert_no_stages()
+
+ def test_incomplete_copy_is_not_another_invocations_rotation_candidate(self):
+ self.shell(r'''
+cp() {
+ local stage="${@: -1}" peer="$CONFIG_DIR.backup.20261024-154500.peer"
+ test -d "$stage" || return 91
+ mkdir "$peer"
+ CONFIG_BACKUP_KEEP_COUNT=1 prune_config_backups "$(dirname "$CONFIG_DIR")" "$(basename "$CONFIG_DIR").backup." "$peer"
+ test -d "$stage" || return 92
+ command cp "$@"
+}
+backup_existing
+test -f "$LAST_CONFIG_BACKUP_DIR/settings.json"
+''')
+ self.assert_no_stages()
+
+ def test_discard_removes_only_this_invocations_new_snapshot(self):
+ old = self.old_snapshot()
+ self.shell(r'''
+backup_existing
+current="$LAST_CONFIG_BACKUP_DIR"
+discard_last_config_backup
+test -z "$LAST_CONFIG_BACKUP_DIR"
+test ! -e "$current"
+''')
+ self.assert_prior(old)
+ self.assertEqual(self.completed(), [old])
+ self.assert_no_stages()
+
+ def test_missing_config_clears_stale_ownership_without_deleting_prior_snapshot(self):
+ old = self.old_snapshot()
+ self.shell(r'''
+LAST_CONFIG_BACKUP_DIR="$CONFIG_DIR.backup.$STAMP"
+CONFIG_DIR="$FIXTURE/absent-config"
+backup_existing
+test -z "$LAST_CONFIG_BACKUP_DIR"
+discard_last_config_backup
+''')
+ self.assert_prior(old)
+ self.assert_no_stages()
+
+ def test_read_only_config_parent_uses_the_existing_hardened_unit_fallback(self):
+ if os.geteuid() == 0:
+ self.skipTest("root bypasses ordinary write permissions")
+ self.config.parent.chmod(0o500)
+ try:
+ self.shell('backup_existing\nprintf "%s\\n" "$LAST_CONFIG_BACKUP_DIR" > "$FIXTURE/latest"\n')
+ latest = Path((self.root / "latest").read_text().strip())
+ self.assertEqual(latest.parent, self.install / "config-backups")
+ self.assertEqual((latest / "settings.json").read_text(), "current state\n")
+ self.assertEqual(list(latest.parent.glob(".*.backup.*")), [])
+ finally:
+ self.config.parent.chmod(0o700)
+
+ def test_every_update_entry_stops_on_failed_backup_even_in_an_or_list(self):
+ for flow in ("pinned", "menu-update", "menu-reinstall"):
+ with self.subTest(flow=flow):
+ result = self.shell(r'''
+CURRENT_VERSION=v6.5.0
+FORCE_VERSION=""
+FORCE_CHANNEL=stable
+IN_CONTAINER=true
+IN_DOCKER=false
+[[ "$FLOW" != pinned ]] || FORCE_VERSION=v6.6.0
+print_header() { :; }
+check_root() { :; }
+detect_os() { :; }
+check_docker_environment() { :; }
+check_existing_installation() { return 0; }
+check_proxmox_host() { return 1; }
+resolve_latest_release_tag_for_channel() { printf 'v6.6.0\n'; }
+detect_service_name() { printf 'pulse\n'; }
+safe_read() { local selection=1; [[ "$FLOW" != menu-reinstall ]] || selection=2; printf -v "$2" '%s' "$selection"; }
+offer_existing_auto_updates() { :; }
+run_upgrade_readiness_preflight() { :; }
+backup_existing() { printf 'backup refused\n' >&2; return 23; }
+create_user() { echo user > "$FIXTURE/unexpected"; }
+download_pulse() { echo download > "$FIXTURE/unexpected"; exit 0; }
+status=0
+main || status=$?
+test "$status" = 1
+''', FLOW=flow)
+ self.assertIn("backup refused", result.stderr)
+ self.assertFalse((self.root / "unexpected").exists())
+
+
+if __name__ == "__main__":
+ unittest.main()