Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve effective systemd masks before update asset replacement
pulse-triage[bot]8512d38dd548
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex d0d2c34bc3..4395c4d9ad 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -25,7 +25,16 @@ inventory's output is not proof of an installed timer.All five existing/new-install refresh consumers can therefore replace staleauto-update assets when a timer exists, without enabling/starting it or changing`system.json`. A configured timer or updater masked by a `/dev/null` symlink is-an explicit operator stop: refresh retains the mask and all existing assets.+an explicit operator stop. Before any asset staging, the shared writer also+reads the effective state of both exact units using a bounded successful+`systemctl` inventory. Persistent and runtime masks anywhere in systemd's unit+search path retain the mask and every existing helper/unit/configuration asset,+even when an `/etc` destination would shadow a `/run` mask or the caller's+configured paths differ from the effective mask location. Failed, unavailable,+malformed, duplicate or unknown inventory stops asset replacement; a successful+empty inventory remains valid for first-time setup. This guard applies to normal+refresh, explicit setup and the sandbox-repair asset writer, without changing+service state or suggesting that enabling updates repairs a mask.Discovery is separate from enablement; version changes still require affirmativeconsent to set up automatic updates.@@ -33,8 +42,12 @@ consent to set up automatic updates.discovery, installer `main` and asset refresh together for version pins, sourcebuilds, menu updates/reinstalls and leftover fresh-install timers. Its controlscover enabled/disabled rows, exact-name impostors, long instance names, missing/-failed/hung inventory, byte-preserved preferences and both unit masks. Where-available, real `systemctl --root` reads only a temporary fixture filesystem.+failed/hung inventory, byte-preserved preferences and both unit masks. Real+`systemctl --root` inventories with vendor units and effective `/run` or `/etc`+masks compare the exact before/after states and complete fixture asset hashes+through all five consumers, asset repair and setup. They also retain the normal+enabled/disabled refresh controls. Where available, this real inventory reads+only a temporary fixture filesystem.Manager operations and server installation remain doubles: this is source/flowproof, not native systemd activation, published installation or customer recovery.diff --git a/install.sh b/install.shindex 06db4797ef..52497d8dd4 100755--- a/install.sh+++ b/install.sh@@ -4467,6 +4467,54 @@ migrate_auto_update_assets_outside_sandbox() {return 0}+# A runtime mask may live in /run while we write /etc, and a persistent mask+# may live outside a configured destination. Checking only those destination+# symlinks can therefore override an effective operator stop. Inspect both+# exact unit names before even staging assets, including sandbox repair/setup.+auto_update_units_refreshable() {+ local timer_path="$1" service_path="$2" asset=""+ for asset in "$timer_path" "$service_path"; do+ if [[ -L "$asset" && "$asset" -ef /dev/null ]]; then+ print_info "Auto-update unit is deliberately masked ($asset); preserving the existing helper and units."+ return 1+ fi+ done++ local timer_unit service_unit unit_files=""+ timer_unit="$(basename "$timer_path")"+ service_unit="$(basename "$service_path")"+ if ! command -v systemctl >/dev/null 2>&1 ||+ ! unit_files=$(LC_ALL=C timeout -k 1 5 systemctl list-unit-files --no-legend --no-pager --full -- "$timer_unit" "$service_unit" 2>/dev/null); then+ print_warn "Cannot confirm auto-update unit masks; preserving the existing helper and units."+ return 1+ fi++ local unit="" state="" remainder="" seen_timer=false seen_service=false+ while read -r unit state remainder; do+ [[ -n "$unit" ]] || continue+ if [[ "$unit" == "$timer_unit" && "$seen_timer" == false ]]; then+ seen_timer=true+ elif [[ "$unit" == "$service_unit" && "$seen_service" == false ]]; then+ seen_service=true+ else+ print_warn "Unexpected auto-update unit inventory; preserving the existing helper and units."+ return 1+ fi+ case "$state" in+ masked|masked-runtime)+ print_info "Auto-update unit is deliberately $state ($unit); preserving the existing helper and units."+ return 1 ;;+ enabled|enabled-runtime|disabled|static|indirect|linked|linked-runtime|alias|generated|transient) ;;+ *)+ print_warn "Cannot confirm auto-update unit state ($unit); preserving the existing helper and units."+ return 1 ;;+ esac+ done <<< "$unit_files"+ # A successful empty inventory is valid for first-time setup; failed or+ # malformed stdout above is never treated as absence or consent.+ return 0+}+# Installs the auto-update helper script and rewrites the systemd# service/timer units. Shared by setup_auto_updates (first-time enable) and# refresh_auto_updates (updates/reinstalls where the timer already exists).@@ -4480,6 +4528,7 @@ install_auto_update_assets() {local update_timer_path="${UPDATE_TIMER_PATH:-${PULSE_UPDATE_TIMER_PATH:-/etc/systemd/system/${service_name}-update.timer}}"local update_timer_unitupdate_timer_unit="$(basename "$update_timer_path")"+ auto_update_units_refreshable "$update_timer_path" "$update_service_path" || return 1local auto_update_bin_dir update_unit_dir update_timer_dirauto_update_bin_dir="$(dirname "$auto_update_dest")"update_unit_dir="$(dirname "$update_service_path")"@@ -4689,7 +4738,7 @@ setup_auto_updates() {fiif ! install_auto_update_assets; then- print_warn "Continuing without automatic updates. Re-run install.sh with --enable-auto-updates once the issue above is resolved."+ print_warn "Continuing without automatic updates. Review the reported unit state or asset error before retrying."ENABLE_AUTO_UPDATES=falsereturn 0fi@@ -4739,19 +4788,10 @@ setup_auto_updates() {# Refresh the installed assets unconditionally instead, without changing the# user's choices: system.json and the timer's enabled state stay untouched.refresh_auto_updates() {- # Discovering an existing masked unit is not consent to unmask it. The- # staged unit rename would otherwise replace its /dev/null symlink.- local asset- for asset in "${UPDATE_TIMER_PATH:-${PULSE_UPDATE_TIMER_PATH:-}}" "${UPDATE_SERVICE_PATH:-${PULSE_UPDATE_SERVICE_PATH:-}}"; do- if [[ -L "$asset" && "$asset" -ef /dev/null ]]; then- print_info "Auto-update unit is deliberately masked ($asset); preserving the existing helper and units."- return 0- fi- doneprint_info "Refreshing the installed auto-update helper..."if ! install_auto_update_assets; thenprint_warn "Could not refresh the auto-update helper; the previously installed one may be stale."- print_warn "Re-run install.sh with --enable-auto-updates to repair it."+ print_warn "Review the reported unit state or asset error before retrying; enabling updates is not a mask repair."fireturn 0}diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.goindex d3b05e5bab..77d5c2e096 100644--- a/scripts/installtests/install_sh_test.go+++ b/scripts/installtests/install_sh_test.go@@ -3191,11 +3191,14 @@ func extractSetupAutoUpdatesShellFunctions(t *testing.T) string {// install_auto_update_assets delegates its writability probe, its staged-helper// sanity check and the sandbox-escape migration to sibling functions, so every-// harness that executes it has to pull those in alongside it.+// harness that executes it has to pull those in alongside it. These isolated+// asset-rendering tests admit an unmasked inventory; the whole-installer Python+// controls below exercise the actual effective-mask guard and all its callers.func extractInstallAutoUpdateAssetsShellFunctions(t *testing.T) string {t.Helper()- return extractRootInstallShellFunction(t, "auto_update_dir_writable") + "\n" ++ return "auto_update_units_refreshable() { return 0; }\n" ++ extractRootInstallShellFunction(t, "auto_update_dir_writable") + "\n" +extractRootInstallShellFunction(t, "auto_update_helper_is_sane") + "\n" +extractRootInstallShellFunction(t, "migrate_auto_update_assets_outside_sandbox") + "\n" +extractRootInstallShellFunction(t, "install_auto_update_assets")@@ -4543,7 +4546,7 @@ func TestRootInstallTemporaryInputsRemainPrivateAndAuthenticated(t *testing.T) {// Execute discovery and every installer refresh consumer together. systemd// manager operations are confined doubles; real inventory uses a fixture root.func TestRootInstallTimerDiscoveryReachesConsentPreservingRefresh(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)defer cancel()cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_server_update_timer.py"), "-v")cmd.Env = append(os.Environ(), "PULSE_INSTALLER_UNDER_TEST="+repoFile("install.sh"))diff --git a/scripts/tests/test_server_update_timer.py b/scripts/tests/test_server_update_timer.pyindex 2629e9093b..66bbed054d 100644--- a/scripts/tests/test_server_update_timer.py+++ b/scripts/tests/test_server_update_timer.py@@ -5,6 +5,7 @@ The server/install producers and systemd manager operations are doubles. Theoptional real systemctl inventory uses --root, never the host's unit manager."""+import hashlibimport jsonimport osfrom pathlib import Path@@ -23,6 +24,16 @@ set -euprintf '%s\n' "$*" >> "$FIXTURE/calls"case "$1" inlist-unit-files)+ # The one-unit read discovers the timer; the two-unit read admits+ # replacement only after checking the effective masks of both units.+ if [[ "$#" == 7 ]]; then+ if [[ "$GUARD_INVENTORY" == real ]]; then+ exec "$REAL_SYSTEMCTL" --root="$FIXTURE/root" "$@"+ fi+ if [[ "$GUARD_INVENTORY" == hang ]]; then exec sleep 30; fi+ printf '%s' "$GUARD_ROWS"+ exit "$GUARD_EXIT"+ fiif [[ "$INVENTORY" == real ]]; thenexec "$REAL_SYSTEMCTL" --root="$FIXTURE/root" "$@"fi@@ -87,6 +98,19 @@ case "$FLOW" in}if update_timer_exists; then echo PRESENT; else echo ABSENT; fiexit 0 ;;+ refresh-unavailable)+ command() {+ [[ "$*" != '-v systemctl' ]] || return 1+ builtin command "$@"+ }+ refresh_auto_updates+ exit 0 ;;+ refresh) refresh_auto_updates; exit 0 ;;+ setup) ENABLE_AUTO_UPDATES=true; setup_auto_updates; exit 0 ;;+ assets)+ result=0+ install_auto_update_assets || result=$?+ exit "$result" ;;version) FORCE_VERSION=v6.6.0 ;;source) BUILD_FROM_SOURCE=true ;;update|reinstall|fresh) : ;;@@ -99,7 +123,8 @@ mainclass ServerUpdateTimerTest(unittest.TestCase):def exercise(self, *, flow="query", state="enabled", service="pulse",rows=None, query_exit=0, inventory="double", mask="",- config_enabled=True):+ config_enabled=True, mask_location="configured", guard_rows=None,+ guard_exit=0, guard_inventory=None):with tempfile.TemporaryDirectory() as temporary:fixture = Path(temporary)config = fixture / "config"@@ -115,41 +140,112 @@ class ServerUpdateTimerTest(unittest.TestCase):settings = json.dumps({"autoUpdateEnabled": config_enabled, "updateChannel": "stable"})(config / "system.json").write_text(settings)timer_name = f"{service}-update.timer"+ service_name = f"{service}-update.service"timer = units / timer_name- updater = units / f"{service}-update.service"- timer.write_text("[Timer]\nOnCalendar=daily\n[Install]\nWantedBy=timers.target\n")- updater.write_text("[Service]\nExecStart=/bin/true\n")+ updater = units / service_name+ installed_units = units+ if mask and mask_location != "configured":+ installed_units = fixture / "root/usr/lib/systemd/system"+ installed_units.mkdir(parents=True)+ if mask_location == "persistent-other":+ # Caller-selected destinations are not the directories in+ # systemd's effective search path. Neither is the mask.+ destination = fixture / "root/opt/selected-units"+ destination.mkdir(parents=True)+ timer = destination / timer_name+ updater = destination / service_name+ (installed_units / timer_name).write_text(+ "[Timer]\nOnCalendar=daily\n[Install]\nWantedBy=timers.target\n")+ (installed_units / service_name).write_text("[Service]\nExecStart=/bin/true\n")if state == "enabled":wants = units / "timers.target.wants"wants.mkdir()- (wants / timer_name).symlink_to(f"../{timer_name}")- masked = timer if mask == "timer" else updater if mask == "service" else None+ (wants / timer_name).symlink_to(+ "/" + str((installed_units / timer_name).relative_to(fixture / "root")))+ masked = None+ if mask:+ mask_name = timer_name if mask == "timer" else service_name+ if mask_location == "configured":+ masked = timer if mask == "timer" else updater+ else:+ mask_dir = (fixture / "root/run/systemd/system"+ if mask_location == "runtime" else units)+ mask_dir.mkdir(parents=True, exist_ok=True)+ masked = mask_dir / mask_nameif masked:- masked.unlink()+ if masked.exists() or masked.is_symlink():+ masked.unlink()masked.symlink_to("/dev/null")systemctl = tools / "systemctl"systemctl.write_text(SYSTEMCTL)systemctl.chmod(0o755)if rows is None:rows = f"{timer_name} {state} enabled\n"+ if guard_rows is None:+ timer_state = "masked" if mask == "timer" else state+ service_state = "masked" if mask == "service" else "disabled"+ guard_rows = f"{timer_name} {timer_state} enabled\n{service_name} {service_state} enabled\n"env = {**os.environ, "INSTALLER": str(INSTALLER), "FIXTURE": str(fixture),"FLOW": flow, "ROWS": rows, "QUERY_EXIT": str(query_exit),+ "GUARD_ROWS": guard_rows, "GUARD_EXIT": str(guard_exit),+ "GUARD_INVENTORY": guard_inventory or inventory,"INVENTORY": inventory, "REAL_SYSTEMCTL": REAL_SYSTEMCTL or "","PATH": f"{tools}:{os.environ['PATH']}", "PULSE_SERVICE_NAME": service,"PULSE_INSTALL_DIR": str(install), "PULSE_CONFIG_DIR": str(config),"PULSE_UPDATE_SERVICE_PATH": str(updater), "PULSE_UPDATE_TIMER_PATH": str(timer),"PULSE_AUTO_UPDATE_DEST": str(helper)}++ def assets():+ result = {}+ for path in sorted(fixture.rglob("*")):+ if path.name in ("calls", "prompts"):+ continue+ mode = path.lstat().st_mode+ if path.is_symlink():+ value = {"mode": mode, "link": os.readlink(path)}+ elif path.is_file():+ value = {"mode": mode, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}+ else:+ value = {"mode": mode}+ result[str(path.relative_to(fixture))] = value+ return result++ def effective_states():+ if inventory != "real":+ return None+ observed = subprocess.run(+ [REAL_SYSTEMCTL, f"--root={fixture / 'root'}", "list-unit-files",+ "--no-legend", "--no-pager", "--full", "--", timer_name, service_name],+ capture_output=True, text=True, timeout=5)+ self.assertEqual(observed.returncode, 0, observed.stdout + observed.stderr)+ return dict(row.split()[:2] for row in observed.stdout.splitlines())++ before_assets = assets()+ before_states = effective_states()result = subprocess.run(["bash", "-c", HARNESS], env=env,capture_output=True, text=True, timeout=12)def lines(name):path = fixture / namereturn path.read_text().splitlines() if path.exists() else []- return {"result": result, "helper": helper.read_text(),+ evidence = {"result": result, "helper": helper.read_text(),"settings": (config / "system.json").read_text(), "original_settings": settings,"calls": lines("calls"), "prompts": lines("prompts"),+ "before_assets": before_assets, "after_assets": assets(),+ "before_states": before_states, "after_states": effective_states(),"timer_masked": timer.is_symlink() and os.readlink(timer) == "/dev/null","service_masked": updater.is_symlink() and os.readlink(updater) == "/dev/null","timer_enabled": (units / "timers.target.wants" / timer_name).is_symlink()}+ if os.environ.get("PULSE_TIMER_TRACE") == "1":+ def digest(value):+ return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()+ print(json.dumps({"flow": flow, "state": state, "service": service,+ "mask": mask, "mask_location": mask_location, "inventory": inventory,+ "guard_inventory": guard_inventory or inventory, "guard_exit": guard_exit,+ "exit": result.returncode, "calls": evidence["calls"],+ "before_states": before_states, "after_states": evidence["after_states"],+ "before_assets_sha256": digest(before_assets),+ "after_assets_sha256": digest(evidence["after_assets"])}, indent=2))+ return evidencedef assert_discovery(self, evidence, present):r = evidence["result"]@@ -166,6 +262,14 @@ class ServerUpdateTimerTest(unittest.TestCase):mutations = [call for call in evidence["calls"] if not call.startswith("list-unit-files")]self.assertEqual(mutations, ["daemon-reload"])+ def assert_preserved(self, evidence, *, status=0):+ r = evidence["result"]+ self.assertEqual(r.returncode, status, r.stdout + r.stderr)+ self.assertIn("stale-helper", evidence["helper"])+ self.assertEqual(evidence["before_assets"], evidence["after_assets"])+ self.assertEqual(evidence["before_states"], evidence["after_states"])+ self.assertFalse(any(not call.startswith("list-unit-files") for call in evidence["calls"]))+def test_discovers_full_systemctl_rows_regardless_of_enablement(self):for state in ("enabled", "enabled-runtime", "disabled", "static", "indirect","linked", "linked-runtime", "masked", "masked-runtime"):@@ -221,13 +325,66 @@ class ServerUpdateTimerTest(unittest.TestCase):for mask in ("timer", "service"):with self.subTest(mask=mask):e = self.exercise(flow="version", mask=mask)- self.assertEqual(e["result"].returncode, 0)+ self.assert_preserved(e)self.assertTrue(e[f"{mask}_masked"])self.assertIn("stale-helper", e["helper"])self.assertIn("masked", e["result"].stdout)self.assertEqual(e["settings"], e["original_settings"])self.assertFalse(any(call == "daemon-reload" for call in e["calls"]))+ @unittest.skipUnless(REAL_SYSTEMCTL, "systemctl filesystem inventory is unavailable")+ def test_all_consumers_preserve_effective_masks_outside_destinations(self):+ for flow in ("version", "source", "update", "reinstall", "fresh"):+ for location, expected in (("runtime", "masked-runtime"), ("persistent-other", "masked")):+ for mask in ("timer", "service"):+ with self.subTest(flow=flow, location=location, mask=mask):+ e = self.exercise(flow=flow, state="disabled", mask=mask,+ mask_location=location, inventory="real", config_enabled=False)+ self.assertEqual(e["before_states"][f"pulse-update.{mask}"], expected)+ self.assert_preserved(e)+ self.assertIn(expected, e["result"].stdout)++ @unittest.skipUnless(REAL_SYSTEMCTL, "systemctl filesystem inventory is unavailable")+ def test_asset_repair_and_explicit_setup_cannot_replace_effectively_masked_units(self):+ for flow in ("assets", "setup"):+ for mask in ("timer", "service"):+ with self.subTest(flow=flow, mask=mask):+ e = self.exercise(flow=flow, state="disabled", mask=mask,+ mask_location="runtime", inventory="real", config_enabled=False)+ self.assert_preserved(e, status=1 if flow == "assets" else 0)+ self.assertIn("masked-runtime", e["result"].stdout)++ def test_failed_unknown_and_duplicate_effective_inventory_cannot_admit_writes(self):+ for rows, code in (("pulse-update.timer disabled enabled\n", 1),+ ("pulse-update.timer not-understood enabled\n", 0),+ ("pulse-update.timer\n", 0),+ ("pulse-updateXtimer disabled enabled\n", 0),+ ("pulse-update.timer disabled enabled\npulse-update.timer disabled enabled\n", 0),+ ("", 1)):+ with self.subTest(rows=rows, code=code):+ self.assert_preserved(self.exercise(flow="version", guard_rows=rows, guard_exit=code))++ def test_effective_inventory_is_bounded_and_missing_systemctl_preserves_assets(self):+ self.assert_preserved(self.exercise(flow="version", guard_inventory="hang"))+ e = self.exercise(flow="refresh-unavailable")+ self.assert_preserved(e)+ self.assertEqual(e["calls"], [])++ def test_successful_absent_or_unmasked_inventory_keeps_asset_repair_working(self):+ for rows in ("", "pulse-update.timer disabled enabled\npulse-update.service disabled enabled\n"):+ with self.subTest(rows=rows):+ self.assert_refreshed(self.exercise(flow="assets", state="disabled", guard_rows=rows), False)++ def test_effective_inventory_names_are_exact_full_custom_instance_names(self):+ service = "pulse-" + "department-" * 12+ e = self.exercise(flow="version", state="disabled", service=service)+ self.assert_refreshed(e, False)+ self.assertEqual(e["calls"][-2],+ f"list-unit-files --no-legend --no-pager --full -- {service}-update.timer {service}-update.service")+ self.assertTrue(e["calls"][:-2])+ self.assertTrue(all(call == f"list-unit-files --no-legend --no-pager --full -- {service}-update.timer"+ for call in e["calls"][:-2]))+@unittest.skipUnless(REAL_SYSTEMCTL, "systemctl filesystem inventory is unavailable")def test_real_systemctl_inventory_and_complete_flow_use_only_fixture_root(self):for state, enabled in (("enabled", True), ("disabled", False)):