Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Overlap isolated memory-reader deadline controls
pulse-triage[bot]117a5c47bb83
diff --git a/scripts/tests/test_memory_troubleshooting_docs.py b/scripts/tests/test_memory_troubleshooting_docs.pyindex 00a5f73a38..d571258d6b 100644--- a/scripts/tests/test_memory_troubleshooting_docs.py+++ b/scripts/tests/test_memory_troubleshooting_docs.py@@ -8,6 +8,7 @@ inspect a host service, database, heap or container daemon.from __future__ import annotations+from concurrent.futures import ThreadPoolExecutorimport jsonimport osfrom pathlib import Path@@ -18,8 +19,10 @@ import signalimport subprocessimport sysimport tempfile+import threadingimport timeimport unittest+from unittest.mock import patchROOT = Path(__file__).resolve().parents[2]@@ -157,6 +160,11 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):self.assertNotIn("Pulse process (PID and UTC start)", result.stdout)self.assertNotRegex(result.stdout, r"VmRSS:|RssAnon:|RssFile:|RssShmem:|VmSwap:")+ def timed_exercise(self, **settings):+ started = time.monotonic()+ result = self.exercise(**settings)+ return time.monotonic() - started, result+def test_shipped_guide_is_the_exercised_source_and_shell_is_valid(self):self.assertEqual(DOC.read_bytes(),(ROOT / "frontend-modern/public/docs/TROUBLESHOOTING.md").read_bytes())@@ -233,23 +241,26 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):self.assertEqual(result.returncode, 0, result.stderr)self.assertEqual(result.stdout.count("0 kB"), 5)- def test_hung_service_reader_is_unavailable_before_the_test_deadline(self):- started = time.monotonic()- result, calls = self.exercise(HANG_TOOL="systemctl")- self.assertLess(time.monotonic() - started, 10)- self.assertEqual(result.returncode, 137, result.stderr)- self.assert_no_memory_claim(result)- self.assertIn("sample unavailable", result.stderr)- self.assertEqual([call["name"] for call in calls], ["date", "systemctl"])-- def test_hung_counter_reader_never_publishes_identity_or_partial_memory(self):- started = time.monotonic()- result, calls = self.exercise(HANG_TOOL="awk")- self.assertLess(time.monotonic() - started, 10)- self.assertEqual(result.returncode, 137, result.stderr)- self.assert_no_memory_claim(result)- self.assertIn("sample unavailable", result.stderr)- self.assertEqual([call["name"] for call in calls], ["date", "systemctl", "ps", "awk"])+ def test_hung_readers_are_unavailable_without_identity_or_partial_memory(self):+ # Each exercise owns its directory, environment, process group and+ # reader pidfd. Overlap only this fixed pair; keep the real 8s deadline+ # and per-reader timing/termination oracles. Assertions and exception+ # collection stay on the test thread, and the executor drains both+ # submissions even when an observation fails.+ with ThreadPoolExecutor(max_workers=2) as pool:+ pending = [(tool, pool.submit(self.timed_exercise, HANG_TOOL=tool))+ for tool in ("systemctl", "awk")]+ for tool, future in pending:+ with self.subTest(tool=tool):+ elapsed, (result, calls) = future.result()+ self.assertLess(elapsed, 10)+ self.assertEqual(result.returncode, 137, result.stderr)+ self.assert_no_memory_claim(result)+ self.assertIn("sample unavailable", result.stderr)+ expected = ["date", "systemctl"]+ if tool == "awk":+ expected += ["ps", "awk"]+ self.assertEqual([call["name"] for call in calls], expected)def test_missing_deadline_utility_is_not_a_sample_or_an_unbounded_fallback(self):copied = recipe().replace("timeout", "pulse-missing-memory-deadline")@@ -261,14 +272,17 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):self.assertEqual(calls, [])def test_term_resistant_readers_are_stopped_even_when_their_pipes_close(self):- for tool in ("systemctl", "awk"):- with self.subTest(tool=tool):- started = time.monotonic()- result, calls = self.exercise(HANG_TOOL=tool, IGNORE_TERM="1", CLOSE_PIPES="1")- self.assertLess(time.monotonic() - started, 10)- self.assertEqual(result.returncode, 137, result.stderr)- self.assert_no_memory_claim(result)- self.assertEqual(calls[-1]["name"], tool)+ with ThreadPoolExecutor(max_workers=2) as pool:+ pending = [(tool, pool.submit(self.timed_exercise, HANG_TOOL=tool,+ IGNORE_TERM="1", CLOSE_PIPES="1"))+ for tool in ("systemctl", "awk")]+ for tool, future in pending:+ with self.subTest(tool=tool):+ elapsed, (result, calls) = future.result()+ self.assertLess(elapsed, 10)+ self.assertEqual(result.returncode, 137, result.stderr)+ self.assert_no_memory_claim(result)+ self.assertEqual(calls[-1]["name"], tool)def test_shell_exit_alone_does_not_pass_the_reader_termination_oracle(self):copied = recipe().replace("--signal=KILL 8s", "--kill-after=1s 8s")@@ -344,6 +358,69 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase):(ROOT / "frontend-modern/public/docs/FAQ.md").read_bytes())+class MemoryDeadlineSchedulingTest(unittest.TestCase):+ """Require bounded overlap and every adverse observation, not a fast sample."""++ controls = (+ ("test_hung_readers_are_unavailable_without_identity_or_partial_memory", {}),+ ("test_term_resistant_readers_are_stopped_even_when_their_pipes_close",+ {"IGNORE_TERM": "1", "CLOSE_PIPES": "1"}),+ )++ def run_control(self, method, settings, *, failed=None, exceptional=None):+ control = MemoryTroubleshootingDocsTest(method)+ barrier = threading.Barrier(2)+ readers = []++ def exercise(**actual):+ tool = actual.pop("HANG_TOOL")+ self.assertIn(tool, ("systemctl", "awk"))+ self.assertEqual(actual, settings)+ readers.append(tool)+ # Serial work cannot reach this rendezvous. This never shortens+ # the copied recipe or signals a real reader, service or guest.+ barrier.wait(timeout=2)+ if tool == exceptional:+ raise RuntimeError("synthetic memory observation failure")+ result = subprocess.CompletedProcess([], 0 if tool == failed else 137,+ "", "Resident-memory sample unavailable\n")+ names = ["date", "systemctl"] + (["ps", "awk"] if tool == "awk" else [])+ return result, [{"name": name} for name in names]++ result = unittest.TestResult()+ with patch.object(control, "exercise", side_effect=exercise), \+ patch(__name__ + ".ThreadPoolExecutor", wraps=ThreadPoolExecutor) as executor:+ control.run(result)+ executor.assert_called_once_with(max_workers=2)+ self.assertCountEqual(readers, ("systemctl", "awk"))+ self.assertEqual(result.testsRun, 1)+ self.assertEqual(result.skipped, [])+ return result++ def test_each_reader_bad_result_still_fails_the_paired_control(self):+ for method, settings in self.controls:+ for failed in (None, "systemctl", "awk"):+ with self.subTest(method=method, failed=failed):+ result = self.run_control(method, settings, failed=failed)+ self.assertEqual(result.errors, [])+ self.assertEqual(len(result.failures), int(failed is not None))+ if failed is not None:+ self.assertIn(f"tool={failed!r}", result.failures[0][0].id())++ def test_exception_does_not_drop_the_other_reader_bad_result(self):+ for method, settings in self.controls:+ for exceptional in ("systemctl", "awk"):+ with self.subTest(method=method, exceptional=exceptional):+ failed = "awk" if exceptional == "systemctl" else "systemctl"+ result = self.run_control(method, settings, failed=failed,+ exceptional=exceptional)+ self.assertEqual(len(result.errors), 1)+ self.assertIn(f"tool={exceptional!r}", result.errors[0][0].id())+ self.assertIn("synthetic memory observation failure", result.errors[0][1])+ self.assertEqual(len(result.failures), 1)+ self.assertIn(f"tool={failed!r}", result.failures[0][0].id())++class GuestMemoryInterpretationDocsTest(unittest.TestCase):HEADING = "#### Guest memory is high but available memory is plentiful\n"ANCHOR = "#guest-memory-is-high-but-available-memory-is-plentiful"