Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep noisy CI test diagnostics out of an output-sized heap

pulse-triage[bot]a370a6f8de08
diff --git a/.github/scripts/record-internal-api-test-seconds.py b/.github/scripts/record-internal-api-test-seconds.py
index a640e9b934..d8f87c279e 100755
--- a/.github/scripts/record-internal-api-test-seconds.py
+++ b/.github/scripts/record-internal-api-test-seconds.py
@@ -44,13 +44,19 @@ from __future__ import annotations
import json
from pathlib import Path
import re
+import shutil
import sys
+import tempfile
from collections import defaultdict
MAX_FAILURE_IDENTIFIERS = 256
MAX_IDENTIFIER_LENGTH = 256
SOURCE_TEST_NAME = re.compile(r"(?:Test|Fuzz|Example)[A-Za-z0-9_]*")
+# Retain diagnostics, not an output-sized Python heap. Short roots stay in
+# memory; noisy roots spill to an anonymous, private temporary file. This is
+# a storage threshold, never an output limit or a different coverage verdict.
+HELD_OUTPUT_MEMORY_BYTES = 64 * 1024
def bounded_identifiers(names: set[str]) -> dict[str, object]:
@@ -88,7 +94,17 @@ def main() -> int:
expected = set(names)
expected_package = sys.argv[3]
- held: dict[str, list[str]] = defaultdict(list)
+ held = defaultdict(lambda: tempfile.SpooledTemporaryFile(
+ max_size=HELD_OUTPUT_MEMORY_BYTES, mode="w+", encoding="utf-8", newline=""))
+
+ def finish_output(top: str, *, show: bool = False) -> None:
+ buffer = held.pop(top, None)
+ if buffer is not None:
+ with buffer:
+ if show:
+ buffer.seek(0)
+ shutil.copyfileobj(buffer, sys.stdout, length=HELD_OUTPUT_MEMORY_BYTES)
+
running: set[str] = set()
seconds: dict[str, float] = {}
package_seconds = None
@@ -187,7 +203,7 @@ def main() -> int:
started.add(top)
running.add(top)
if text:
- held[top].append(text)
+ held[top].write(text)
if test != top or action not in ("pass", "fail", "skip"):
continue
@@ -195,11 +211,10 @@ def main() -> int:
invalid_events += 1
running.discard(top)
completed.add(top)
- lines = held.pop(top, [])
+ finish_output(top, show=action == "fail")
if action == "fail":
failed = True
failed_tests.add(top)
- out.write("".join(lines))
out.flush()
elapsed = event.get("Elapsed")
if isinstance(elapsed, (int, float)):
@@ -208,7 +223,11 @@ def main() -> int:
for top in sorted(running):
failed = True
out.write(f"--- {top} did not finish; its output follows\n")
- out.write("".join(held.get(top, [])))
+ finish_output(top, show=True)
+ # Malformed streams can contain output without a root start. They still
+ # fail below; do not leave their temporary buffers open until GC.
+ for top in list(held):
+ finish_output(top)
out.flush()
missing = expected - completed if expected is not None else set()
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index d9114e9568..c8d6673bb5 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -1390,6 +1390,17 @@ change test selection, race/vet coverage or required check names.
`test_api_failure_index.py` exercises failure/unfinished/package-only/unknown,
privacy and bounds controls; the existing Go shard contract verifies the index
beside unchanged timing output.
+Held test output uses a 64 KiB in-memory spool per active root, then secure
+temporary storage, rather than an output-sized Python heap. Passing/skipped
+roots discard their buffers; failed roots print all their output in arrival
+order, and unfinished roots print all their output in sorted root order at EOF.
+The threshold changes storage, not log length, timing/index schemas, source
+selection or the verdict. Overflow files use anonymous/private temporary files,
+not test or subtest labels as filenames, and are closed when each root ends or
+the stream is drained. Registered recorder controls stream 96 MiB through four
+interleaved roots under a fixed 64 MiB Linux address-space budget, requiring
+exact failure/unfinished bytes, quiet pass/skip output and unchanged index
+privacy. Smaller Unicode/interleaving controls also check the spill path.
The recorder itself must now fail incomplete or inconsistent execution evidence,
even when the producer exits zero: it requires exactly one passing package
completion, no malformed events, and non-repeated complete top-level executions.
diff --git a/scripts/tests/test_api_failure_index.py b/scripts/tests/test_api_failure_index.py
index e3db3b3263..b68c6e18b4 100644
--- a/scripts/tests/test_api_failure_index.py
+++ b/scripts/tests/test_api_failure_index.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
"""Shard artifacts expose bounded source identities, never assertion payloads."""
+import hashlib
import json
import os
from pathlib import Path
@@ -41,6 +42,99 @@ class APIFailureIndexTest(unittest.TestCase):
raw = Path(str(seconds) + '.failures.json').read_text()
return result, seconds.read_text(), json.loads(raw), raw
+ def test_spilled_output_keeps_interleaved_unicode_and_unfinished_order(self):
+ names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestZPending', 'TestAPending']
+ output = {name: ''.join(f'{i}: λ雪 {name}/private-case\n' for i in range(4096))
+ for name in names}
+ stream = ''.join(event('run', name) for name in names)
+ for i in range(0, max(map(len, output.values())), 1024):
+ for name in names:
+ stream += event('output', name + '/private-case', output[name][i:i + 1024])
+ stream += (event('fail', 'TestBroken', elapsed=.1) +
+ event('pass', 'TestQuiet', elapsed=.2) +
+ event('skip', 'TestSkipped', elapsed=.3) + event('fail', elapsed=.4))
+ result, seconds, index, raw = self.record(stream, names)
+ self.assertEqual(result.returncode, 1, result.stderr)
+ expected = output['TestBroken']
+ for name in ['TestAPending', 'TestZPending']:
+ expected += f'--- {name} did not finish; its output follows\n' + output[name]
+ self.assertEqual(result.stdout, expected)
+ self.assertEqual(seconds, '# package-seconds 0.40\nTestBroken 0.10\n'
+ 'TestQuiet 0.20\nTestSkipped 0.30\n')
+ self.assertEqual(index['failed_top_level_tests']['names'], ['TestBroken'])
+ self.assertEqual(index['unfinished_top_level_tests']['names'],
+ ['TestAPending', 'TestZPending'])
+ self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])
+ self.assertNotIn('private-case', raw)
+
+ def noisy_record(self, *, interrupted):
+ # Stream 96 MiB from disk in small events: the recorder, not the test
+ # driver, must retain all four active roots. No application/Go test or
+ # excluded crash scenario is involved. A fixed address-space budget
+ # catches output-sized retention without a timing/RSS threshold.
+ import resource
+ names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestPending']
+ chunks = {name: (name + '/private-case λ\r\n').ljust(8192, '.') for name in names}
+ repeats = 3072
+ expected = hashlib.sha256()
+ if interrupted:
+ for _ in range(repeats):
+ expected.update(chunks['TestBroken'].encode())
+ expected.update(b'--- TestPending did not finish; its output follows\n')
+ for _ in range(repeats):
+ expected.update(chunks['TestPending'].encode())
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ source, seconds, log = (root / name for name in ['events.jsonl', 'seconds.txt', 'log'])
+ spool = root / 'spool'
+ spool.mkdir()
+ selection = root / 'selected'
+ selection.write_text('\n'.join(names) + '\n')
+ with source.open('w', encoding='utf-8', newline='') as stream:
+ stream.write(''.join(event('run', name) for name in names))
+ rows = ''.join(event('output', name + '/private-case', chunks[name]) for name in names)
+ for _ in range(repeats):
+ stream.write(rows)
+ stream.write(event('fail' if interrupted else 'pass', names[0], elapsed=.1))
+ stream.write(event('pass', names[1], elapsed=.2))
+ stream.write(event('skip', names[2], elapsed=.3))
+ if not interrupted:
+ stream.write(event('pass', names[3], elapsed=.4))
+ stream.write(event('fail' if interrupted else 'pass', elapsed=.5))
+
+ def memory_budget():
+ resource.setrlimit(resource.RLIMIT_AS, (64 * 1024**2, 64 * 1024**2))
+
+ with source.open('rb') as stream, log.open('wb') as output:
+ result = subprocess.run(
+ [sys.executable, str(RECORDER), str(seconds), str(selection), 'example/internal/api'],
+ stdin=stream, stdout=output, stderr=subprocess.PIPE, timeout=60,
+ env={**os.environ, 'TMPDIR': str(spool)}, preexec_fn=memory_budget)
+ self.assertEqual(result.returncode, int(interrupted), result.stderr.decode())
+ self.assertNotIn(b'Traceback', result.stderr)
+ actual = hashlib.sha256()
+ with log.open('rb') as output:
+ for block in iter(lambda: output.read(64 * 1024), b''):
+ actual.update(block)
+ self.assertEqual(actual.hexdigest(), expected.hexdigest())
+ raw = Path(str(seconds) + '.failures.json').read_text()
+ index = json.loads(raw)
+ self.assertEqual(index['recorder_exit_code'], int(interrupted))
+ self.assertEqual(index['completed_top_level_test_count'], 3 if interrupted else 4)
+ self.assertEqual(index['unfinished_top_level_tests']['names'],
+ ['TestPending'] if interrupted else [])
+ self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])
+ self.assertNotIn('private-case', raw)
+ self.assertEqual(list(spool.iterdir()), [])
+
+ @unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')
+ def test_noisy_passing_and_skipped_roots_fit_fixed_memory(self):
+ self.noisy_record(interrupted=False)
+
+ @unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')
+ def test_noisy_failed_and_unfinished_roots_fit_fixed_memory_without_log_loss(self):
+ self.noisy_record(interrupted=True)
+
def test_pass_preserves_timing_and_quiet_output(self):
result, seconds, index, _ = self.record(
event('run', 'TestGood') + event('output', 'TestGood', 'private output\n') +