Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Make native NAS cleanup proof portable and observable
pulse-triage[bot]81d614e937b4
diff --git a/scripts/installtests/agent_nas_teardown.py b/scripts/installtests/agent_nas_teardown.pyindex 7f7ffcc7ae..53bca0b3c7 100644--- a/scripts/installtests/agent_nas_teardown.py+++ b/scripts/installtests/agent_nas_teardown.py@@ -11,8 +11,11 @@ import shleximport shutilimport statimport subprocess+import sysimport tempfile+import timeimport unittest+from unittest import mockROOT = Path(__file__).resolve().parents[2]SOURCE = Path(os.environ.get("PULSE_AGENT_NAS_SOURCE", ROOT / "scripts/install.sh"))@@ -80,11 +83,11 @@ exec ''' + shlex.quote(str(executable)) + ' -ww "$@"\n')return target-MODEL = r'''#!/usr/bin/python3+MODEL = r'''import json, os, pathlib, sysroot = pathlib.Path(os.environ['FIXTURE'])path = root / 'model.json'-v = json.loads(path.read_text()); role, args = pathlib.Path(sys.argv[0]).name, sys.argv[1:]+v = json.loads(path.read_text()); role, args = sys.argv[1], sys.argv[2:]fault = v['fault']def event(s):with (root / 'events').open('a') as f: f.write(s + '\n')@@ -135,6 +138,65 @@ sys.exit(0)'''+def write_model_tools(root, tools):+ # Use the interpreter selected by the native runner, not Darwin's unrelated+ # /usr/bin/python3. These models use only the standard library: repeated+ # site startup is unnecessary, and site hooks must not alter fault controls.+ model = root / 'model.py'+ model.write_text(MODEL)+ for name in ('ps', 'kill', 'flash'):+ tool = tools / name+ tool.write_text('#!/bin/sh\nexec ' + shlex.quote(sys.executable)+ + ' -I -S ' + shlex.quote(str(model)) + ' ' + name + ' "$@"\n')+ tool.chmod(0o700)+++def observed_process_absence(pid_file):+ # /proc does not exist on Darwin. Missing process evidence is not proof+ # of cleanup; only a kernel ESRCH for the recorded positive PID is absence.+ try:+ pid = int(pid_file.read_text())+ except (OSError, ValueError):+ return False+ if pid <= 0:+ return False+ try:+ os.kill(pid, 0)+ except ProcessLookupError:+ return True+ except OSError:+ return False+ return False+++class NASTimingResult(unittest.TextTestResult):+ """Retain progress before the outer deadline, without changing any budget."""++ def startTest(self, test):+ self.started = time.monotonic()+ if not hasattr(self, 'suite_started'):+ self.suite_started = self.started+ self.subcases = 0+ self.progress('start', test)+ super().startTest(test)++ def addSubTest(self, test, subtest, err):+ self.subcases += 1+ super().addSubTest(test, subtest, err)++ def stopTest(self, test):+ super().stopTest(test)+ self.progress('complete', test)++ def progress(self, phase, test):+ now = time.monotonic()+ self.stream.writeln('NAS fixture timing: ' + json.dumps(dict(+ phase=phase, control=test._testMethodName, completed_subcases=self.subcases,+ suite_seconds=round(now - self.suite_started, 6),+ control_seconds=round(now - self.started, 6)), sort_keys=True))+ self.stream.flush()++class AgentNASTeardown(unittest.TestCase):source = SOURCE.read_text()@@ -143,8 +205,7 @@ class AgentNASTeardown(unittest.TestCase):with tempfile.TemporaryDirectory() as directory:root = Path(directory)tools = root / "tools"; tools.mkdir()- for name in ("ps", "kill", "flash"):- p = tools / name; p.write_text(MODEL); p.chmod(0o700)+ write_model_tools(root, tools)for name in ("state", "helper", "profile", "bin", "flash", "legacy", "foreign"):d = root / name; d.mkdir(mode=0o700)if name in ("state", "helper", "profile", "foreign"):@@ -285,7 +346,7 @@ remove_safe_profile_state_dir() { rm -rf -- "$SAFE_PROFILE_STATE_DIR"; }after = self.snapshot(root)content = bootfile.read_bytes() if bootfile.is_file() and not bootfile.is_symlink() else Nonerecovered = {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("retired")}- timed_stopped = not (root / "timed-pid").exists() or not Path('/proc/' + (root / "timed-pid").read_text()).exists()+ timed_stopped = observed_process_absence(root / "timed-pid")return result, second, events, before, after, json.loads((root / "model.json").read_text()), content, original_boot, recovered, timed_stopped@staticmethod@@ -410,6 +471,90 @@ remove_safe_profile_state_dir() { rm -rf -- "$SAFE_PROFILE_STATE_DIR"; }self.assertNotIn('/synthetic/private-only', r[0].stdout); self.assertIn('200 1 other\n', r[0].stdout)r = self.exercise('QNAP', 'timeout-process'); self.retained(r); self.assertTrue(r[9])+ def test_private_model_uses_selected_isolated_interpreter(self):+ with tempfile.TemporaryDirectory(prefix='nas model ') as directory:+ root = Path(directory)+ tools = root / 'tools'; tools.mkdir()+ invocations = root / 'interpreter-arguments'+ interpreter = root / 'selected python'+ interpreter.write_text('#!/bin/sh\nprintf "%s\\n" "$@" >> '+ + shlex.quote(str(invocations)) + '\nexec '+ + shlex.quote(sys.executable) + ' "$@"\n')+ interpreter.chmod(0o700)+ with mock.patch.object(sys, 'executable', str(interpreter)):+ write_model_tools(root, tools)+ # A site hook would contaminate model state if startup isolation+ # disappeared. No external package, service or process is involved.+ (root / 'sitecustomize.py').write_text(+ 'from pathlib import Path\nPath(' + repr(str(root / 'site-ran')) + ').touch()\n')+ (root / 'model.json').write_text(json.dumps(dict(+ platform='QNAP', fault='', reads=0, watchdog=True, collector=True)))+ (root / 'flash').mkdir(); (root / 'flash/autorun.sh').write_text('#!/bin/sh\n')+ env = dict(os.environ, FIXTURE=str(root), PYTHONPATH=str(root))+ commands = (+ ('ps', '-ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'comm=', '-o', 'args='),+ ('kill', '-TERM', '100'),+ ('flash', 'mount_flash_config'),+ )+ for name, *args in commands:+ result = subprocess.run([str(tools / name), *args], env=env,+ capture_output=True, text=True, timeout=5)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(result.stderr, '')+ recorded = invocations.read_text().splitlines()+ self.assertEqual(recorded.count('-I'), 3)+ self.assertEqual(recorded.count('-S'), 3)+ self.assertEqual(recorded.count(str(root / 'model.py')), 3)+ self.assertFalse((root / 'site-ran').exists())+ self.assertEqual((root / 'events').read_text().splitlines(), [+ 'process:read', 'signal:100', 'boot:absent-at-signal', 'flash:mount_flash_config'])+ self.assertFalse(json.loads((root / 'model.json').read_text())['watchdog'])++ def test_timeout_child_absence_requires_observed_exit(self):+ with tempfile.TemporaryDirectory() as directory:+ pid_file = Path(directory) / 'child-pid'+ with mock.patch.object(os, 'kill') as probe:+ for value in (None, '', 'invalid', '0', '-1'):+ if value is not None:+ pid_file.write_text(value)+ self.assertFalse(observed_process_absence(pid_file))+ probe.assert_not_called()+ process = subprocess.Popen(['sleep', '30'])+ try:+ pid_file.write_text(str(process.pid))+ self.assertFalse(observed_process_absence(pid_file), 'live child reported absent')+ with mock.patch.object(os, 'kill', side_effect=PermissionError):+ self.assertFalse(observed_process_absence(pid_file), 'refusal reported absence')+ finally:+ process.terminate()+ process.wait(timeout=3)+ self.assertTrue(observed_process_absence(pid_file), 'reaped child still reported live')++ def test_control_timing_reports_cumulative_and_per_control_time(self):+ import io++ class InertControl(unittest.TestCase):+ def test_inert(self):+ for _ in range(2):+ with self.subTest():+ self.assertTrue(True)++ stream = io.StringIO()+ result = unittest.TextTestRunner(stream=stream, verbosity=2,+ resultclass=NASTimingResult).run(+ unittest.defaultTestLoader.loadTestsFromTestCase(InertControl))+ self.assertTrue(result.wasSuccessful())+ records = [json.loads(line.removeprefix('NAS fixture timing: '))+ for line in stream.getvalue().splitlines() if line.startswith('NAS fixture timing: ')]+ self.assertEqual([r['phase'] for r in records], ['start', 'complete'])+ self.assertEqual([r['completed_subcases'] for r in records], [0, 2])+ for record in records:+ self.assertEqual(set(record), {'phase', 'control', 'completed_subcases',+ 'suite_seconds', 'control_seconds'})+ self.assertEqual(record['control'], 'test_inert')+ self.assertGreaterEqual(record['suite_seconds'], record['control_seconds'])+ self.assertGreaterEqual(records[1]['control_seconds'], records[0]['control_seconds'])+def test_live_unraid_boot_shell_is_unknown_not_a_signal_target(self):r = self.exercise('Unraid', 'live-go'); events = self.retained(r)self.assertEqual(r[6], r[7]); self.assertFalse(any(e.startswith('signal:') for e in events))@@ -561,4 +706,5 @@ remove_safe_profile_state_dir() { rm -rf -- "$SAFE_PROFILE_STATE_DIR"; }self.assertIn('[[ "$uninstall_upstart" != true && -z "$uninstall_nas_platform" ]]', caller)-if __name__ == '__main__': unittest.main(verbosity=2)+if __name__ == '__main__':+ unittest.main(verbosity=2, testRunner=unittest.TextTestRunner(verbosity=2, resultclass=NASTimingResult))diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.goindex d1071241f4..0ed61bf65b 100644--- a/scripts/installtests/install_sh_test.go+++ b/scripts/installtests/install_sh_test.go@@ -8576,6 +8576,9 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {"test_recovered_ordinary_linux_state_is_not_qnap_platform_authority","test_live_unraid_boot_shell_is_unknown_not_a_signal_target","test_actual_generated_watchdogs_and_children_become_quiescent",+ "test_private_model_uses_selected_isolated_interpreter",+ "test_timeout_child_absence_requires_observed_exit",+ "test_control_timing_reports_cumulative_and_per_control_time","test_private_process_reader_requests_native_basename_and_full_argv","test_generated_collector_observation_keeps_exact_identity_on_both_unix_formats","test_caller_has_no_blanket_nas_or_legacy_tree_deletion",@@ -8584,7 +8587,7 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {t.Fatalf("NAS removal control %s did not pass:\n%s", name, output)}}- if !strings.Contains(string(output), "\nRan 20 tests in ") || !strings.Contains(string(output), "\nOK\n") {+ if !strings.Contains(string(output), "\nRan 23 tests in ") || !strings.Contains(string(output), "\nOK\n") {t.Fatalf("incomplete NAS removal verdict:\n%s", output)}t.Logf("%s", output)