Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Overlap isolated passive-preflight deadline controls
pulse-triage[bot]b827aca349c5
diff --git a/scripts/tests/test_vm_disk_diagnostics.py b/scripts/tests/test_vm_disk_diagnostics.pyindex 52c24de134..fbd7a23e08 100644--- a/scripts/tests/test_vm_disk_diagnostics.py+++ b/scripts/tests/test_vm_disk_diagnostics.py@@ -7,6 +7,7 @@ command selection, failure handling and private output, not native thaw.from __future__ import annotations+from concurrent.futures import ThreadPoolExecutorimport jsonimport osfrom pathlib import Path@@ -17,8 +18,10 @@ import signalimport subprocessimport tempfileimport textwrap+import threadingimport timeimport unittest+from unittest import mockROOT = Path(__file__).resolve().parents[2]@@ -31,6 +34,13 @@ PRIVATE = "synthetic-private-infrastructure-marker"READS = [["status", "100"], ["config", "100", "--current"]]STATUS_LIMIT = 256CONFIG_LIMIT = 64 * 1024+DEADLINE_CASES = (("status", "producer"),+ ("status", "inherited-pipe"),+ ("status", "reader"),+ ("status", "closed-pipe-reader"),+ ("config", "inherited-pipe"),+ ("config", "reader"),+ ("config", "closed-pipe-reader"))def guide_section(guide, heading):@@ -492,31 +502,36 @@ class VMDiskDiagnosticsTest(unittest.TestCase):self.assertIn(f"unavailable: {missing}", result.stderr)self.assertEqual(calls, [])- def test_real_deadline_stops_a_blocked_synthetic_local_read(self):+ def test_real_deadlines_stop_every_owned_local_reader(self):self.assertIsNotNone(REAL_TIMEOUT)- start = time.monotonic()- result, calls = self.exercise(settings={"real_timeout": True, "sleep_operation": "status"})- elapsed = time.monotonic() - start- self.assertEqual(result.returncode, 1)- self.assertIn("status read timed out", result.stderr)- self.assertLess(elapsed, 14)- self.assert_passive(calls, operations=READS[:1])- def test_complete_read_deadline_covers_inherited_stdout_and_the_reader(self):- self.assertIsNotNone(REAL_TIMEOUT)- for operation in ("status", "config"):+ def exercise_case(case):+ operation, mode = caselimit = str(STATUS_LIMIT if operation == "status" else CONFIG_LIMIT)- for mode in ("inherited-pipe", "reader", "closed-pipe-reader"):+ settings = {"real_timeout": True}+ if mode == "producer":+ settings["sleep_operation"] = operation+ elif mode == "inherited-pipe":+ settings["pipe_holder_operation"] = operation+ else:+ settings.update(blocked_response_reader=True, blocked_limit=limit,+ close_reader_pipes=mode == "closed-pipe-reader")+ start = time.monotonic()+ result, calls = self.exercise(settings=settings)+ return result, calls, time.monotonic() - start++ # Only these seven fixed, wait-bound synthetic controls overlap.+ # exercise() owns a distinct directory/environment and+ # run_owned_preflight() owns a session and exact reader pidfd per case.+ # No process-global mocks or arbitrary suites run concurrently. Keep+ # each real 10s collection deadline, 14s assertion and cleanup unchanged.+ # Collect every verdict on the test thread, even after a case raises;+ # exiting the executor drains all submitted work.+ with ThreadPoolExecutor(max_workers=3) as pool:+ pending = [(case, pool.submit(exercise_case, case)) for case in DEADLINE_CASES]+ for (operation, mode), future in pending:with self.subTest(operation=operation, mode=mode):- settings = {"real_timeout": True}- if mode == "inherited-pipe":- settings["pipe_holder_operation"] = operation- else:- settings.update(blocked_response_reader=True, blocked_limit=limit,- close_reader_pipes=mode == "closed-pipe-reader")- start = time.monotonic()- result, calls = self.exercise(settings=settings)- elapsed = time.monotonic() - start+ result, calls, elapsed = future.result()self.assertEqual(result.returncode, 1, result.stdout)label = "status" if operation == "status" else "configuration"self.assertIn(f"{label} read timed out", result.stderr)@@ -529,6 +544,75 @@ class VMDiskDiagnosticsTest(unittest.TestCase):self.assertLess(len(result.stdout + result.stderr), 600)self.assert_passive(calls, operations=READS[:1] if operation == "status" else READS)+ def deadline_adapter(self, barrier, calls, *, fail, error=None):+ """Independent command/result witness for scheduling controls only."""+ def exercise(*, settings):+ self.assertTrue(settings["real_timeout"])+ if "sleep_operation" in settings:+ case = settings["sleep_operation"], "producer"+ elif "pipe_holder_operation" in settings:+ case = settings["pipe_holder_operation"], "inherited-pipe"+ else:+ self.assertTrue(settings["blocked_response_reader"])+ self.assertIn(settings["blocked_limit"], ("256", "65536"))+ operation = "status" if settings["blocked_limit"] == "256" else "config"+ case = operation, "closed-pipe-reader" if settings["close_reader_pipes"] else "reader"+ calls.append(case)+ # The first wave must coexist. Later queued cases need no sleep.+ if case in DEADLINE_CASES[:3]:+ barrier.wait(timeout=2)+ if case == error:+ raise RuntimeError("synthetic owned preflight observation failure")+ operation = case[0]+ operations = READS[:1] if operation == "status" else READS+ observed = []+ for command in operations:+ limit = "256" if command[0] == "status" else "65536"+ observed.extend(({"name": "qm", "args": command},+ {"name": "python3", "args": ["-I", "-c", "fixture", limit]}))+ label = "status" if operation == "status" else "configuration"+ result = subprocess.CompletedProcess([], 0 if fail(case) else 1, "",+ f"{label} read timed out")+ return result, observed+ return exercise++ def test_deadline_controls_overlap_boundedly_and_keep_every_failure_gating(self):+ expected = (("status", "producer"), ("status", "inherited-pipe"),+ ("status", "reader"), ("status", "closed-pipe-reader"),+ ("config", "inherited-pipe"), ("config", "reader"),+ ("config", "closed-pipe-reader"))+ self.assertEqual(DEADLINE_CASES, expected)+ for failed in (None, *expected):+ with self.subTest(failed=failed):+ calls = []+ adapter = self.deadline_adapter(threading.Barrier(3), calls,+ fail=lambda case: case == failed)+ control = type(self)("test_real_deadlines_stop_every_owned_local_reader")+ result = unittest.TestResult()+ with mock.patch.object(control, "exercise", side_effect=adapter), \+ mock.patch(__name__ + ".ThreadPoolExecutor", wraps=ThreadPoolExecutor) as executor:+ control.run(result)+ executor.assert_called_once_with(max_workers=3)+ self.assertEqual(sorted(calls), sorted(expected))+ self.assertEqual(result.errors, [])+ self.assertEqual(len(result.failures), int(failed is not None))+ if failed is not None:+ self.assertIn(f"operation={failed[0]!r}", result.failures[0][0].id())+ self.assertIn(f"mode={failed[1]!r}", result.failures[0][0].id())++ def test_deadline_exception_does_not_lose_later_queued_verdicts(self):+ calls = []+ adapter = self.deadline_adapter(threading.Barrier(3), calls,+ fail=lambda case: True, error=DEADLINE_CASES[0])+ control = type(self)("test_real_deadlines_stop_every_owned_local_reader")+ result = unittest.TestResult()+ with mock.patch.object(control, "exercise", side_effect=adapter):+ control.run(result)+ self.assertEqual(sorted(calls), sorted(DEADLINE_CASES))+ self.assertEqual(len(result.errors), 1)+ self.assertIn("synthetic owned preflight observation failure", result.errors[0][1])+ self.assertEqual(len(result.failures), 6)+def test_exact_documented_command_uses_the_passive_helper(self):recipes = bash_examples(guide_section(DOC.read_text(), "Passive host preflight"))self.assertEqual(len(recipes), 1)