Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Overlap isolated incident-log deadline controls
pulse-triage[bot]0d8d4a59b96f
diff --git a/scripts/tests/test_troubleshooting_logs.py b/scripts/tests/test_troubleshooting_logs.pyindex 85235ab54c..81e5f643f3 100644--- a/scripts/tests/test_troubleshooting_logs.py+++ b/scripts/tests/test_troubleshooting_logs.py@@ -6,6 +6,7 @@ This checks shell behaviour, not an installed Pulse service or Docker daemon.from __future__ import annotations+from concurrent.futures import ThreadPoolExecutorimport jsonimport osfrom pathlib import Path@@ -16,6 +17,7 @@ import signalimport subprocessimport sysimport tempfile+import threadingimport unittestfrom unittest.mock import Mock, call, patch@@ -305,12 +307,20 @@ class TroubleshootingLogRecipesTest(unittest.TestCase):self.assertIn("Log read unavailable (exit 2)", result.stderr)def test_hung_readers_withhold_partial_matches_and_stop_by_deadline(self):- for reader in recipes():- with self.subTest(reader=reader):- result = self.exercise(reader, stdout=JSON_LOG, hang="term")- self.assertEqual(result.returncode, 137, result.stderr)- self.assertEqual(result.stdout, "")- self.assertIn("Log read unavailable", result.stderr)+ # Each exercise owns its directory, environment, process group and+ # reader pidfd. Overlap only this fixed pair, keeping the real 8s+ # deadline and 500ms survival/cleanup observations unchanged. Collect+ # both verdicts on the test thread even if a future raises; the+ # executor drains all submitted work before this method returns.+ with ThreadPoolExecutor(max_workers=2) as pool:+ pending = [(reader, pool.submit(self.exercise, reader, stdout=JSON_LOG, hang="term"))+ for reader in recipes()]+ for reader, future in pending:+ with self.subTest(reader=reader):+ result = future.result()+ self.assertEqual(result.returncode, 137, result.stderr)+ self.assertEqual(result.stdout, "")+ self.assertIn("Log read unavailable", result.stderr)def test_missing_timeout_stops_before_either_reader(self):for reader in recipes():@@ -387,12 +397,17 @@ class GeneralLogEvidenceTest(unittest.TestCase):self.assertNotIn("private access error", result.stderr)def test_general_readers_stop_and_withhold_partial_errors_at_real_deadline(self):- for reader in recipes("### Inspect Notification Logs"):- with self.subTest(reader=reader):- result = self.exercise(reader, stdout="partial error\n", hang="term")- self.assertEqual(result.returncode, 137, result.stderr)- self.assertEqual(result.stdout, "")- self.assertIn("Log read unavailable (exit 137)", result.stderr)+ # Only the two private wait-bound fixtures overlap, not other tests+ # or their process-global mocks. Keep every real deadline and oracle.+ with ThreadPoolExecutor(max_workers=2) as pool:+ pending = [(reader, pool.submit(self.exercise, reader, stdout="partial error\n", hang="term"))+ for reader in recipes("### Inspect Notification Logs")]+ for reader, future in pending:+ with self.subTest(reader=reader):+ result = future.result()+ self.assertEqual(result.returncode, 137, result.stderr)+ self.assertEqual(result.stdout, "")+ self.assertIn("Log read unavailable (exit 137)", result.stderr)def test_general_readers_do_not_fall_back_when_timeout_is_missing(self):for reader in recipes("### Inspect Notification Logs"):@@ -553,14 +568,18 @@ class LogByteBoundsTest(unittest.TestCase):def test_hard_deadline_terminates_closed_pipe_term_resistant_readers_and_cleans_buffer(self):selected = [("docker", recipes("### Inspect Notification Logs")["docker"]),("journalctl", self.copied_recipes()[-1][2])]- for reader, recipe in selected:- with self.subTest(reader=reader):- result, _ = self.exercise(reader, recipe, stdout="partial private log\n",- hang="ignore-term-closed-pipes")- self.assertEqual(result.returncode, 137, result.stderr)- self.assertEqual(result.stdout, "")- self.assertNotIn("partial private log", result.stderr)- self.assertEqual(len(result.log_cleanup), 1)+ with ThreadPoolExecutor(max_workers=2) as pool:+ pending = [(reader, pool.submit(self.exercise, reader, recipe,+ stdout="partial private log\n",+ hang="ignore-term-closed-pipes"))+ for reader, recipe in selected]+ for reader, future in pending:+ with self.subTest(reader=reader):+ result, _ = future.result()+ self.assertEqual(result.returncode, 137, result.stderr)+ self.assertEqual(result.stdout, "")+ self.assertNotIn("partial private log", result.stderr)+ self.assertEqual(len(result.log_cleanup), 1)def test_each_entry_keeps_collection_deadline_pipe_failures_and_byte_limiter_together(self):for heading, reader, recipe in self.copied_recipes():@@ -572,5 +591,71 @@ class LogByteBoundsTest(unittest.TestCase):self.assertNotIn("--foreground", recipe)+class LogDeadlineSchedulingTest(unittest.TestCase):+ """Prove bounded overlap without a timing threshold or real reader replay."""++ controls = (+ (TroubleshootingLogRecipesTest,+ "test_hung_readers_withhold_partial_matches_and_stop_by_deadline", "term"),+ (GeneralLogEvidenceTest,+ "test_general_readers_stop_and_withhold_partial_errors_at_real_deadline", "term"),+ (LogByteBoundsTest,+ "test_hard_deadline_terminates_closed_pipe_term_resistant_readers_and_cleans_buffer",+ "ignore-term-closed-pipes"),+ )++ def run_control(self, case, *, failed=None, exceptional=None):+ test_class, method, hang = case+ control = test_class(method)+ barrier = threading.Barrier(2)+ readers = []++ def exercise(reader, *args, **settings):+ self.assertIn(reader, ("docker", "journalctl"))+ self.assertEqual(settings["hang"], hang)+ readers.append(reader)+ # A serial implementation cannot satisfy this rendezvous. Neither+ # a shorter safeguard nor a favourable wall-clock sample can pass.+ barrier.wait(timeout=2)+ if reader == exceptional:+ raise RuntimeError("synthetic log observation failure")+ result = subprocess.CompletedProcess([], 0 if reader == failed else 137,+ "", "Log read unavailable (exit 137)\n")+ result.log_cleanup = [{"mode": 0o600, "bytes": 0}]+ return (result, None) if args else result++ result = unittest.TestResult()+ with patch.object(control, "exercise", side_effect=exercise), \+ patch(__name__ + ".ThreadPoolExecutor", wraps=ThreadPoolExecutor) as executor:+ control.run(result)+ executor.assert_called_once_with(max_workers=2)+ self.assertCountEqual(readers, ("docker", "journalctl"))+ self.assertEqual(result.testsRun, 1)+ self.assertEqual(result.skipped, [])+ return result++ def test_two_reader_overlap_keeps_each_failure_gating(self):+ for case in self.controls:+ for failed in (None, "docker", "journalctl"):+ with self.subTest(method=case[1], failed=failed):+ result = self.run_control(case, failed=failed)+ self.assertEqual(result.errors, [])+ self.assertEqual(len(result.failures), int(failed is not None))+ if failed is not None:+ self.assertIn(f"reader={failed!r}", result.failures[0][0].id())++ def test_exception_still_collects_the_other_readers_failure(self):+ for case in self.controls:+ for exceptional in ("docker", "journalctl"):+ with self.subTest(method=case[1], exceptional=exceptional):+ failed = "journalctl" if exceptional == "docker" else "docker"+ result = self.run_control(case, failed=failed, exceptional=exceptional)+ self.assertEqual(len(result.errors), 1)+ self.assertIn(f"reader={exceptional!r}", result.errors[0][0].id())+ self.assertIn("synthetic log observation failure", result.errors[0][1])+ self.assertEqual(len(result.failures), 1)+ self.assertIn(f"reader={failed!r}", result.failures[0][0].id())++if __name__ == "__main__":unittest.main()