Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep noisy CI test diagnostics out of an output-sized heap
pulse-triage[bot]a370a6f8de08
diff --git a/.github/scripts/record-internal-api-test-seconds.py b/.github/scripts/record-internal-api-test-seconds.pyindex a640e9b934..d8f87c279e 100755--- a/.github/scripts/record-internal-api-test-seconds.py+++ b/.github/scripts/record-internal-api-test-seconds.py@@ -44,13 +44,19 @@ from __future__ import annotationsimport jsonfrom pathlib import Pathimport re+import shutilimport sys+import tempfilefrom collections import defaultdictMAX_FAILURE_IDENTIFIERS = 256MAX_IDENTIFIER_LENGTH = 256SOURCE_TEST_NAME = re.compile(r"(?:Test|Fuzz|Example)[A-Za-z0-9_]*")+# Retain diagnostics, not an output-sized Python heap. Short roots stay in+# memory; noisy roots spill to an anonymous, private temporary file. This is+# a storage threshold, never an output limit or a different coverage verdict.+HELD_OUTPUT_MEMORY_BYTES = 64 * 1024def bounded_identifiers(names: set[str]) -> dict[str, object]:@@ -88,7 +94,17 @@ def main() -> int:expected = set(names)expected_package = sys.argv[3]- held: dict[str, list[str]] = defaultdict(list)+ held = defaultdict(lambda: tempfile.SpooledTemporaryFile(+ max_size=HELD_OUTPUT_MEMORY_BYTES, mode="w+", encoding="utf-8", newline=""))++ def finish_output(top: str, *, show: bool = False) -> None:+ buffer = held.pop(top, None)+ if buffer is not None:+ with buffer:+ if show:+ buffer.seek(0)+ shutil.copyfileobj(buffer, sys.stdout, length=HELD_OUTPUT_MEMORY_BYTES)+running: set[str] = set()seconds: dict[str, float] = {}package_seconds = None@@ -187,7 +203,7 @@ def main() -> int:started.add(top)running.add(top)if text:- held[top].append(text)+ held[top].write(text)if test != top or action not in ("pass", "fail", "skip"):continue@@ -195,11 +211,10 @@ def main() -> int:invalid_events += 1running.discard(top)completed.add(top)- lines = held.pop(top, [])+ finish_output(top, show=action == "fail")if action == "fail":failed = Truefailed_tests.add(top)- out.write("".join(lines))out.flush()elapsed = event.get("Elapsed")if isinstance(elapsed, (int, float)):@@ -208,7 +223,11 @@ def main() -> int:for top in sorted(running):failed = Trueout.write(f"--- {top} did not finish; its output follows\n")- out.write("".join(held.get(top, [])))+ finish_output(top, show=True)+ # Malformed streams can contain output without a root start. They still+ # fail below; do not leave their temporary buffers open until GC.+ for top in list(held):+ finish_output(top)out.flush()missing = expected - completed if expected is not None else set()diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex d9114e9568..c8d6673bb5 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -1390,6 +1390,17 @@ change test selection, race/vet coverage or required check names.`test_api_failure_index.py` exercises failure/unfinished/package-only/unknown,privacy and bounds controls; the existing Go shard contract verifies the indexbeside unchanged timing output.+Held test output uses a 64 KiB in-memory spool per active root, then secure+temporary storage, rather than an output-sized Python heap. Passing/skipped+roots discard their buffers; failed roots print all their output in arrival+order, and unfinished roots print all their output in sorted root order at EOF.+The threshold changes storage, not log length, timing/index schemas, source+selection or the verdict. Overflow files use anonymous/private temporary files,+not test or subtest labels as filenames, and are closed when each root ends or+the stream is drained. Registered recorder controls stream 96 MiB through four+interleaved roots under a fixed 64 MiB Linux address-space budget, requiring+exact failure/unfinished bytes, quiet pass/skip output and unchanged index+privacy. Smaller Unicode/interleaving controls also check the spill path.The recorder itself must now fail incomplete or inconsistent execution evidence,even when the producer exits zero: it requires exactly one passing packagecompletion, no malformed events, and non-repeated complete top-level executions.diff --git a/scripts/tests/test_api_failure_index.py b/scripts/tests/test_api_failure_index.pyindex e3db3b3263..b68c6e18b4 100644--- a/scripts/tests/test_api_failure_index.py+++ b/scripts/tests/test_api_failure_index.py@@ -1,6 +1,7 @@#!/usr/bin/env python3"""Shard artifacts expose bounded source identities, never assertion payloads."""+import hashlibimport jsonimport osfrom pathlib import Path@@ -41,6 +42,99 @@ class APIFailureIndexTest(unittest.TestCase):raw = Path(str(seconds) + '.failures.json').read_text()return result, seconds.read_text(), json.loads(raw), raw+ def test_spilled_output_keeps_interleaved_unicode_and_unfinished_order(self):+ names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestZPending', 'TestAPending']+ output = {name: ''.join(f'{i}: λ雪 {name}/private-case\n' for i in range(4096))+ for name in names}+ stream = ''.join(event('run', name) for name in names)+ for i in range(0, max(map(len, output.values())), 1024):+ for name in names:+ stream += event('output', name + '/private-case', output[name][i:i + 1024])+ stream += (event('fail', 'TestBroken', elapsed=.1) ++ event('pass', 'TestQuiet', elapsed=.2) ++ event('skip', 'TestSkipped', elapsed=.3) + event('fail', elapsed=.4))+ result, seconds, index, raw = self.record(stream, names)+ self.assertEqual(result.returncode, 1, result.stderr)+ expected = output['TestBroken']+ for name in ['TestAPending', 'TestZPending']:+ expected += f'--- {name} did not finish; its output follows\n' + output[name]+ self.assertEqual(result.stdout, expected)+ self.assertEqual(seconds, '# package-seconds 0.40\nTestBroken 0.10\n'+ 'TestQuiet 0.20\nTestSkipped 0.30\n')+ self.assertEqual(index['failed_top_level_tests']['names'], ['TestBroken'])+ self.assertEqual(index['unfinished_top_level_tests']['names'],+ ['TestAPending', 'TestZPending'])+ self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])+ self.assertNotIn('private-case', raw)++ def noisy_record(self, *, interrupted):+ # Stream 96 MiB from disk in small events: the recorder, not the test+ # driver, must retain all four active roots. No application/Go test or+ # excluded crash scenario is involved. A fixed address-space budget+ # catches output-sized retention without a timing/RSS threshold.+ import resource+ names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestPending']+ chunks = {name: (name + '/private-case λ\r\n').ljust(8192, '.') for name in names}+ repeats = 3072+ expected = hashlib.sha256()+ if interrupted:+ for _ in range(repeats):+ expected.update(chunks['TestBroken'].encode())+ expected.update(b'--- TestPending did not finish; its output follows\n')+ for _ in range(repeats):+ expected.update(chunks['TestPending'].encode())+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ source, seconds, log = (root / name for name in ['events.jsonl', 'seconds.txt', 'log'])+ spool = root / 'spool'+ spool.mkdir()+ selection = root / 'selected'+ selection.write_text('\n'.join(names) + '\n')+ with source.open('w', encoding='utf-8', newline='') as stream:+ stream.write(''.join(event('run', name) for name in names))+ rows = ''.join(event('output', name + '/private-case', chunks[name]) for name in names)+ for _ in range(repeats):+ stream.write(rows)+ stream.write(event('fail' if interrupted else 'pass', names[0], elapsed=.1))+ stream.write(event('pass', names[1], elapsed=.2))+ stream.write(event('skip', names[2], elapsed=.3))+ if not interrupted:+ stream.write(event('pass', names[3], elapsed=.4))+ stream.write(event('fail' if interrupted else 'pass', elapsed=.5))++ def memory_budget():+ resource.setrlimit(resource.RLIMIT_AS, (64 * 1024**2, 64 * 1024**2))++ with source.open('rb') as stream, log.open('wb') as output:+ result = subprocess.run(+ [sys.executable, str(RECORDER), str(seconds), str(selection), 'example/internal/api'],+ stdin=stream, stdout=output, stderr=subprocess.PIPE, timeout=60,+ env={**os.environ, 'TMPDIR': str(spool)}, preexec_fn=memory_budget)+ self.assertEqual(result.returncode, int(interrupted), result.stderr.decode())+ self.assertNotIn(b'Traceback', result.stderr)+ actual = hashlib.sha256()+ with log.open('rb') as output:+ for block in iter(lambda: output.read(64 * 1024), b''):+ actual.update(block)+ self.assertEqual(actual.hexdigest(), expected.hexdigest())+ raw = Path(str(seconds) + '.failures.json').read_text()+ index = json.loads(raw)+ self.assertEqual(index['recorder_exit_code'], int(interrupted))+ self.assertEqual(index['completed_top_level_test_count'], 3 if interrupted else 4)+ self.assertEqual(index['unfinished_top_level_tests']['names'],+ ['TestPending'] if interrupted else [])+ self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])+ self.assertNotIn('private-case', raw)+ self.assertEqual(list(spool.iterdir()), [])++ @unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')+ def test_noisy_passing_and_skipped_roots_fit_fixed_memory(self):+ self.noisy_record(interrupted=False)++ @unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')+ def test_noisy_failed_and_unfinished_roots_fit_fixed_memory_without_log_loss(self):+ self.noisy_record(interrupted=True)+def test_pass_preserves_timing_and_quiet_output(self):result, seconds, index, _ = self.record(event('run', 'TestGood') + event('output', 'TestGood', 'private output\n') +