Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Retain E2E container diagnostics without flooding Actions logs
pulse-triage[bot]e2a3ce4bc2e3
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.ymlindex c3d29e5f58..dc865f1665 100644--- a/.github/workflows/test-e2e.yml+++ b/.github/workflows/test-e2e.yml@@ -16,6 +16,8 @@ on:- 'scripts/tests/test_playwright_acquisition.py'- 'scripts/e2e_test_images.py'- 'scripts/tests/test_e2e_test_images.py'+ - 'scripts/collect_e2e_container_logs.py'+ - 'scripts/tests/test_e2e_container_logs.py'push:branches:- main@@ -32,6 +34,8 @@ on:- 'scripts/tests/test_playwright_acquisition.py'- 'scripts/e2e_test_images.py'- 'scripts/tests/test_e2e_test_images.py'+ - 'scripts/collect_e2e_container_logs.py'+ - 'scripts/tests/test_e2e_container_logs.py'workflow_dispatch:# Let branch runs finish so busy main and release branches retain verdicts.@@ -62,6 +66,9 @@ jobs:- name: Validate shared E2E image admissionrun: python3 scripts/tests/test_e2e_test_images.py+ - name: Validate bounded container diagnostic retention+ run: python3 scripts/tests/test_e2e_container_logs.py+- name: Validate browser dependency acquisition policyrun: python3 scripts/tests/test_playwright_acquisition.py@@ -370,14 +377,19 @@ jobs:- name: Collect container logsif: always()- working-directory: tests/integration- run: |- echo "=== Docker containers ==="- docker ps -a- echo "=== Pulse test server logs ==="- docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"- echo "=== Mock GitHub server logs ==="- docker logs pulse-mock-github 2>&1 || echo "No pulse-mock-github container"+ timeout-minutes: 3+ run: python3 scripts/collect_e2e_container_logs.py "$RUNNER_TEMP/pulse-e2e-container-logs"++ # Preserve all diagnostic bytes, including partial/failed reads, while+ # the combined Actions log stays small enough to inspect its verdicts.+ - name: Upload complete container diagnostics+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: e2e-container-logs-shard-${{ matrix.shard }}-${{ github.sha }}-${{ github.run_attempt }}+ path: ${{ runner.temp }}/pulse-e2e-container-logs/+ retention-days: 3+ if-no-files-found: error# continue-on-error keeps the job green on a probation failure, so# failure() never fires for these; key off the step outcome instead.@@ -473,8 +485,17 @@ jobs:- name: Collect container logsif: always()- working-directory: tests/integration- run: docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"+ timeout-minutes: 3+ run: python3 scripts/collect_e2e_container_logs.py "$RUNNER_TEMP/pulse-e2e-container-logs" --registration++ - name: Upload complete container diagnostics+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: e2e-container-logs-agent-registration-${{ github.sha }}-${{ github.run_attempt }}+ path: ${{ runner.temp }}/pulse-e2e-container-logs/+ retention-days: 3+ if-no-files-found: error- name: Upload Playwright reportif: failure()diff --git a/scripts/collect_e2e_container_logs.py b/scripts/collect_e2e_container_logs.pynew file mode 100644index 0000000000..78f8075083--- /dev/null+++ b/scripts/collect_e2e_container_logs.py@@ -0,0 +1,104 @@+#!/usr/bin/env python3+"""Retain CI container diagnostics without flooding the combined Actions log.++Full stdout/stderr belongs in this run's short-lived artifact. The console gets+only a bounded, escaped tail, byte/hash identity and an honest collection result.+This observes fixed disposable E2E containers; it never starts or changes one.+"""++import argparse+import hashlib+import json+import os+from pathlib import Path+import re+import subprocess+import sys+++TAIL_BYTES = 4096+COMMAND_TIMEOUT_SECONDS = 30+COMMANDS = (+ ("containers.txt", ("docker", "ps", "-a")),+ ("pulse-test-server.log", ("docker", "logs", "pulse-test-server")),+ ("pulse-mock-github.log", ("docker", "logs", "pulse-mock-github")),+)+++def checkout_identity():+ identity = {+ "source_sha": os.environ["GITHUB_SHA"],+ "run_id": os.environ["GITHUB_RUN_ID"],+ "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"],+ }+ if not re.fullmatch(r"[0-9a-f]{40}", identity["source_sha"]):+ raise ValueError("invalid E2E source identity")+ if any(not re.fullmatch(r"[1-9][0-9]*", identity[key])+ for key in ("run_id", "run_attempt")):+ raise ValueError("invalid E2E run identity")+ source = subprocess.run(+ ["git", "rev-parse", "HEAD"], check=True, capture_output=True,+ text=True, timeout=COMMAND_TIMEOUT_SECONDS,+ ).stdout.strip()+ if source != identity["source_sha"]:+ raise ValueError("E2E checkout differs from this run")+ return identity+++def file_observation(path):+ digest = hashlib.sha256()+ with path.open("rb") as stream:+ for block in iter(lambda: stream.read(65536), b""):+ digest.update(block)+ size = stream.tell()+ stream.seek(max(0, size - TAIL_BYTES))+ tail = stream.read(TAIL_BYTES)+ return size, digest.hexdigest(), tail+++def collect(directory, identity, *, registration=False, run=subprocess.run, console=None):+ console = sys.stdout if console is None else console+ # Never overwrite another attempt's evidence, including through a symlink.+ directory.mkdir(parents=True, exist_ok=False)+ records = []+ for filename, command in COMMANDS[:2] if registration else COMMANDS:+ path = directory / filename+ record = {"file": filename, "exit_code": None, "collection_complete": False}+ with path.open("xb") as stream:+ try:+ result = run(command, stdout=stream, stderr=subprocess.STDOUT,+ timeout=COMMAND_TIMEOUT_SECONDS, check=False)+ record.update(exit_code=result.returncode,+ collection_complete=result.returncode == 0)+ except subprocess.TimeoutExpired:+ # subprocess.run stops and waits for its Docker CLI on timeout;+ # TimeoutExpired exposes no terminal exit. Do not invent one.+ record["failure"] = "timeout"+ except OSError:+ record["failure"] = "spawn"+ size, digest, tail = file_observation(path)+ record.update(bytes=size, sha256=digest)+ records.append(record)+ # JSON escaping prevents log text becoming Actions commands. Never+ # mistake this bounded excerpt for the complete artifact bytes.+ print(json.dumps({**record, "console_tail_bytes": len(tail),+ "console_tail_truncated": size > len(tail),+ "console_tail": tail.decode("utf-8", errors="replace")}),+ file=console, flush=True)+ complete = all(record["collection_complete"] for record in records)+ manifest = {"schema_version": 1, **identity, "collection_complete": complete,+ "records": records}+ (directory / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")+ return 0 if complete else 1+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("output", type=Path)+ parser.add_argument("--registration", action="store_true")+ arguments = parser.parse_args()+ return collect(arguments.output, checkout_identity(), registration=arguments.registration)+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/tests/test_e2e_container_logs.py b/scripts/tests/test_e2e_container_logs.pynew file mode 100644index 0000000000..67c3029aaf--- /dev/null+++ b/scripts/tests/test_e2e_container_logs.py@@ -0,0 +1,228 @@+#!/usr/bin/env python3+"""Exercise actual log-file retention, bounded output and CI wiring offline."""++import hashlib+import importlib.util+import io+import json+import os+from pathlib import Path+import re+import subprocess+import sys+import tempfile+import unittest+from unittest.mock import patch+++ROOT = Path(__file__).resolve().parents[2]+WORKFLOW = ROOT / ".github/workflows/test-e2e.yml"+SPEC = importlib.util.spec_from_file_location(+ "e2e_container_logs", ROOT / "scripts/collect_e2e_container_logs.py")+LOGS = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(LOGS)+IDENTITY = {"source_sha": "a" * 40, "run_id": "123", "run_attempt": "2"}+++def job_block(workflow, job):+ return re.split(r"(?m)^ \S", workflow.split(f"\n {job}:\n", 1)[1], maxsplit=1)[0]+++def assert_workflow_policy(test, workflow):+ for event in ("pull_request", "push"):+ block = re.split(r"(?m)^ \S", workflow.split(f" {event}:\n", 1)[1], maxsplit=1)[0]+ test.assertIn("'scripts/collect_e2e_container_logs.py'", block)+ test.assertIn("'scripts/tests/test_e2e_container_logs.py'", block)+ selection = job_block(workflow, "tier-selection")+ test.assertLess(selection.index("python3 scripts/tests/test_e2e_container_logs.py"),+ selection.index("npm ci"))+ for job, suffix, selector in (("e2e", "shard-${{ matrix.shard }}", ""),+ ("agent-registration", "agent-registration", " --registration")):+ steps = re.split(r"(?m)^ - name: ", job_block(workflow, job))[1:]+ collector = next(step for step in steps if step.startswith("Collect container logs\n"))+ upload = next(step for step in steps if step.startswith("Upload complete container diagnostics\n"))+ test.assertIn("if: always()", collector)+ test.assertIn("timeout-minutes: 3", collector)+ test.assertIn('python3 scripts/collect_e2e_container_logs.py "$RUNNER_TEMP/pulse-e2e-container-logs"' + selector,+ collector)+ test.assertNotIn("continue-on-error", collector)+ test.assertNotIn("docker logs", collector)+ test.assertNotIn("||", collector)+ test.assertIn("if: always()", upload)+ test.assertIn("actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", upload)+ test.assertIn("e2e-container-logs-" + suffix + "-${{ github.sha }}-${{ github.run_attempt }}", upload)+ test.assertIn("path: ${{ runner.temp }}/pulse-e2e-container-logs/", upload)+ test.assertIn("retention-days: 3", upload)+ test.assertIn("if-no-files-found: error", upload)+ test.assertLess(steps.index(collector), steps.index(upload))+++class ContainerLogTest(unittest.TestCase):+ def test_large_real_subprocess_stdout_and_stderr_are_retained_not_printed(self):+ # Real child/pipe/file I/O, but no Docker or localhost listener. This+ # exceeds the observed gateway ceiling without sending it to stdout.+ size = 8_001_774+ calls = []++ def run(command, **options):+ calls.append(command)+ return subprocess.run(+ [sys.executable, "-c",+ "import sys; sys.stdout.buffer.write(b'x'*" + str(size) + "); "+ "sys.stderr.buffer.write(b'\\n::error::untrusted diagnostic\\n')"],+ **options,+ )++ payload = b"x" * size + b"\n::error::untrusted diagnostic\n"+ with tempfile.TemporaryDirectory() as temp:+ output = Path(temp) / "logs"+ console = io.StringIO()+ self.assertEqual(0, LOGS.collect(output, IDENTITY, registration=True,+ run=run, console=console))+ self.assertEqual(calls, [command for _, command in LOGS.COMMANDS[:2]])+ manifest = json.loads((output / "manifest.json").read_text())+ self.assertTrue(manifest["collection_complete"])+ self.assertEqual(manifest["source_sha"], IDENTITY["source_sha"])+ for record in manifest["records"]:+ self.assertEqual((output / record["file"]).read_bytes(), payload)+ self.assertEqual(record["bytes"], len(payload))+ self.assertEqual(record["sha256"], hashlib.sha256(payload).hexdigest())+ summaries = [json.loads(line) for line in console.getvalue().splitlines()]+ self.assertEqual(len(summaries), 2)+ for summary in summaries:+ self.assertTrue(summary["console_tail_truncated"])+ self.assertEqual(summary["console_tail_bytes"], LOGS.TAIL_BYTES)+ self.assertLess(len(console.getvalue().encode()), 12_000)+ self.assertFalse(any(line.startswith("::") for line in console.getvalue().splitlines()))++ def test_nonzero_timeout_and_spawn_failures_keep_partial_bytes_and_other_records(self):+ for failure in ("exit", "signal", "timeout", "spawn"):+ with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temp:+ calls = []++ def run(command, **options):+ calls.append(command)+ options["stdout"].write(b"partial diagnostic\n")+ self.assertEqual(options["stderr"], subprocess.STDOUT)+ self.assertEqual(options["timeout"], 30)+ self.assertFalse(options["check"])+ if len(calls) != 1:+ return subprocess.CompletedProcess(command, 0)+ if failure == "timeout":+ raise subprocess.TimeoutExpired(command, 30)+ if failure == "spawn":+ raise OSError("private path must not be printed")+ return subprocess.CompletedProcess(command, -15 if failure == "signal" else 7)++ output = Path(temp) / "logs"+ console = io.StringIO()+ self.assertEqual(1, LOGS.collect(output, IDENTITY, run=run, console=console))+ manifest = json.loads((output / "manifest.json").read_text())+ self.assertEqual(len(calls), 3)+ self.assertFalse(manifest["collection_complete"])+ first = manifest["records"][0]+ expected = {"exit": 7, "signal": -15}.get(failure)+ self.assertEqual(first["exit_code"], expected)+ self.assertFalse(first["collection_complete"])+ if failure in ("timeout", "spawn"):+ self.assertEqual(first["failure"], failure)+ self.assertTrue(all(record["collection_complete"] for record in manifest["records"][1:]))+ self.assertNotIn("private path", console.getvalue())+ self.assertEqual((output / first["file"]).read_bytes(), b"partial diagnostic\n")++ def test_real_child_deadline_retains_partial_output_and_reaps_owned_cli_double(self):+ observed = []+ real_popen = subprocess.Popen++ def observe_popen(*args, **options):+ child = real_popen(*args, **options)+ observed.append(child)+ return child++ def run(command, **options):+ if command != LOGS.COMMANDS[0][1]:+ return subprocess.CompletedProcess(command, 0)+ # Only this owned CLI double has a short deadline. The production+ # helper's fixed 30-second deadline is checked separately above.+ return subprocess.run(+ [sys.executable, "-c", "import time; print('partial', flush=True); time.sleep(10)"],+ **{**options, "timeout": 0.2},+ )++ with tempfile.TemporaryDirectory() as temp, patch.object(subprocess, "Popen", observe_popen):+ output = Path(temp) / "logs"+ self.assertEqual(1, LOGS.collect(output, IDENTITY, registration=True,+ run=run, console=io.StringIO()))+ manifest = json.loads((output / "manifest.json").read_text())+ first = manifest["records"][0]+ self.assertEqual(first["failure"], "timeout")+ self.assertIsNone(first["exit_code"])+ self.assertEqual((output / first["file"]).read_bytes(), b"partial\n")+ self.assertEqual(len(observed), 1)+ self.assertIsNotNone(observed[0].returncode)+ self.assertIsNotNone(observed[0].poll())++ def test_empty_binary_and_short_logs_have_exact_byte_identity_and_bounded_safe_console(self):+ for payload in (b"", b"short\n", b"\xff\x00\r\n::warning::\x1b[31m" * 1000):+ with self.subTest(size=len(payload)), tempfile.TemporaryDirectory() as temp:+ def run(command, **options):+ options["stdout"].write(payload)+ return subprocess.CompletedProcess(command, 0)+ output = Path(temp) / "logs"+ console = io.StringIO()+ self.assertEqual(0, LOGS.collect(output, IDENTITY, run=run, console=console))+ for summary in map(json.loads, console.getvalue().splitlines()):+ self.assertEqual(summary["bytes"], len(payload))+ self.assertEqual(summary["sha256"], hashlib.sha256(payload).hexdigest())+ self.assertEqual(summary["console_tail_truncated"], len(payload) > LOGS.TAIL_BYTES)+ self.assertLess(len(console.getvalue()), 80_000)+ self.assertEqual(len(console.getvalue().splitlines()), 3)++ def test_existing_directory_or_symlink_is_never_overwritten(self):+ with tempfile.TemporaryDirectory() as temp:+ existing = Path(temp) / "existing"+ existing.mkdir()+ (existing / "keep").write_bytes(b"unchanged")+ link = Path(temp) / "link"+ link.symlink_to(existing, target_is_directory=True)+ for output in (existing, link):+ with self.assertRaises(FileExistsError):+ LOGS.collect(output, IDENTITY, run=lambda *args, **kwargs: self.fail("unexpected command"))+ self.assertEqual((existing / "keep").read_bytes(), b"unchanged")++ def test_source_and_run_identity_are_checked_before_collection(self):+ environment = {"GITHUB_SHA": "a" * 40, "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "2"}+ with patch.dict(os.environ, environment), patch.object(LOGS.subprocess, "run") as run:+ run.return_value.stdout = "a" * 40 + "\n"+ self.assertEqual(LOGS.checkout_identity(), IDENTITY)+ run.assert_called_once_with(["git", "rev-parse", "HEAD"], check=True,+ capture_output=True, text=True, timeout=30)+ run.return_value.stdout = "b" * 40 + "\n"+ with self.assertRaisesRegex(ValueError, "checkout differs"):+ LOGS.checkout_identity()+ for key, value in (("GITHUB_SHA", "not-a-commit"), ("GITHUB_RUN_ID", "0"),+ ("GITHUB_RUN_ATTEMPT", "1\n2")):+ with patch.dict(os.environ, {**environment, key: value}), patch.object(LOGS.subprocess, "run") as run:+ with self.assertRaises(ValueError):+ LOGS.checkout_identity()+ run.assert_not_called()++ def test_workflow_retains_full_diagnostics_on_success_and_failure(self):+ assert_workflow_policy(self, WORKFLOW.read_text())++ def test_old_dump_missing_upload_and_failure_suppression_are_rejected(self):+ workflow = WORKFLOW.read_text()+ mutants = (+ workflow.replace('python3 scripts/collect_e2e_container_logs.py "$RUNNER_TEMP/pulse-e2e-container-logs"',+ 'docker logs pulse-test-server'),+ workflow.replace("if-no-files-found: error", "if-no-files-found: ignore"),+ workflow.replace("if: always()\n timeout-minutes: 3", "if: failure()\n timeout-minutes: 3"),+ workflow.replace("Collect container logs\n", "Collect container logs\n continue-on-error: true\n"),+ )+ for workflow in mutants:+ with self.subTest(mutant=workflow[:30]), self.assertRaises(AssertionError):+ assert_workflow_policy(self, workflow)+++if __name__ == "__main__":+ unittest.main()