Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reject incomplete smoke-test discovery before execution

pulse-triage[bot]51db712d85e8
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
index 832ca573f1..b066faa26c 100755
--- a/scripts/tests/run.sh
+++ b/scripts/tests/run.sh
@@ -18,10 +18,20 @@ EOF
discover_tests() {
local -n ref=$1
- mapfile -t ref < <(
+ local inventory
+ # Process substitution hides producer failures from mapfile. Admit the
+ # complete discovery pipeline before running any tests, never a partial list.
+ if ! inventory="$(
find "${TEST_DIR}" -maxdepth 1 -type f \
\( -name 'test-*.sh' -o -name 'test_*.py' \) | sort
- )
+ )"; then
+ echo "Failed to discover the complete smoke test inventory under ${TEST_DIR}" >&2
+ return 1
+ fi
+ ref=()
+ if [[ -n "${inventory}" ]]; then
+ mapfile -t ref <<< "${inventory}"
+ fi
}
resolve_test_path() {
@@ -99,7 +109,9 @@ main() {
tests+=("${resolved}")
done
else
- discover_tests tests
+ if ! discover_tests tests; then
+ return 1
+ fi
fi
if [[ ${#tests[@]} -eq 0 ]]; then
diff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.py
new file mode 100644
index 0000000000..c8d78e4feb
--- /dev/null
+++ b/scripts/tests/test_smoke_runner.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Exercise the real smoke runner's discovery admission and failure collection."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+RUNNER = ROOT / "scripts/tests/run.sh"
+
+
+class SmokeRunnerTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory(prefix="pulse-smoke-runner-")
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name)
+ self.tests = self.root / "scripts/tests"
+ self.tests.mkdir(parents=True)
+ self.runner = self.tests / "run.sh"
+ shutil.copyfile(RUNNER, self.runner)
+ self.runner.chmod(0o700)
+ self.bin = self.root / "bin"
+ self.bin.mkdir()
+ self.calls = self.root / "calls"
+ self.environment = {
+ **os.environ,
+ "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}",
+ "SMOKE_FIXTURE_CALLS": str(self.calls),
+ }
+
+ def python_fixture(self, name, status=0):
+ path = self.tests / name
+ path.write_text(
+ "import os\nfrom pathlib import Path\n"
+ f"with Path(os.environ['SMOKE_FIXTURE_CALLS']).open('a') as log: log.write({name!r} + '\\n')\n"
+ f"raise SystemExit({status})\n",
+ encoding="utf-8",
+ )
+ return path
+
+ def shell_fixture(self, name, status=0):
+ path = self.tests / name
+ path.write_text(
+ f"#!/bin/bash\nprintf '%s\\n' {shlex.quote(name)} >> \"$SMOKE_FIXTURE_CALLS\"\nexit {status}\n",
+ encoding="utf-8",
+ )
+ path.chmod(0o700)
+ return path
+
+ def producer(self, name, status, paths=()):
+ path = self.bin / name
+ lines = ["#!/bin/bash"]
+ # A failed sort consumes stdin like the real sorter, then can emit a
+ # valid prefix. Neither a find nor sort prefix is a complete inventory.
+ if name == "sort":
+ lines.append("cat >/dev/null")
+ lines.extend(f"printf '%s\\n' {shlex.quote(str(item))}" for item in paths)
+ lines.append(f"exit {status}")
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+ path.chmod(0o700)
+
+ def run_runner(self, *arguments):
+ return subprocess.run(
+ ["bash", str(self.runner), *map(str, arguments)],
+ cwd=self.root,
+ env=self.environment,
+ capture_output=True,
+ text=True,
+ timeout=15,
+ )
+
+ def observed_calls(self):
+ return self.calls.read_text(encoding="utf-8").splitlines() if self.calls.exists() else []
+
+ def assert_discovery_rejected(self, result):
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertIn("Failed to discover the complete smoke test inventory", result.stderr)
+ self.assertNotIn("Summary:", result.stdout)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_failed_find_cannot_admit_partial_passing_inventory(self):
+ passing = self.python_fixture("test_first.py")
+ self.python_fixture("test_second.py", 7)
+ self.producer("find", 23, [passing])
+ self.assert_discovery_rejected(self.run_runner())
+
+ def test_failed_sort_cannot_admit_partial_passing_inventory(self):
+ passing = self.python_fixture("test_first.py")
+ self.python_fixture("test_second.py", 7)
+ self.producer("sort", 24, [passing])
+ self.assert_discovery_rejected(self.run_runner())
+
+ def test_empty_failed_find_is_discovery_failure(self):
+ self.python_fixture("test_first.py")
+ self.producer("find", 23)
+ self.assert_discovery_rejected(self.run_runner())
+
+ def test_empty_failed_sort_is_discovery_failure(self):
+ self.python_fixture("test_first.py")
+ self.producer("sort", 24)
+ self.assert_discovery_rejected(self.run_runner())
+
+ def test_successful_empty_discovery_is_not_one_empty_test_or_a_pass(self):
+ result = self.run_runner()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("No tests found under", result.stderr)
+ self.assertNotIn("Summary:", result.stdout)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_full_inventory_preserves_sorted_python_and_shell_checks(self):
+ self.python_fixture("test_z.py")
+ self.python_fixture("test_a.py")
+ self.shell_fixture("test-b.sh")
+ self.python_fixture("not_a_test.py", 7)
+ nested = self.tests / "nested"
+ nested.mkdir()
+ (nested / "test_ignored.py").write_text("raise SystemExit(7)\n", encoding="utf-8")
+ result = self.run_runner()
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(self.observed_calls(), ["test-b.sh", "test_a.py", "test_z.py"])
+ self.assertIn("Summary: 3/3 passed", result.stdout)
+
+ def test_every_failure_is_collected_and_remains_gating(self):
+ self.shell_fixture("test-a.sh", 3)
+ self.python_fixture("test_a.py", 7)
+ self.python_fixture("test_z.py")
+ result = self.run_runner()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(self.observed_calls(), ["test-a.sh", "test_a.py", "test_z.py"])
+ self.assertIn("Summary: 1/3 passed", result.stdout)
+ self.assertIn("Failures: 2", result.stdout)
+
+ def test_explicit_subset_keeps_order_and_does_not_discover(self):
+ self.python_fixture("test_a.py")
+ self.shell_fixture("test-b.sh")
+ self.python_fixture("test_unselected.py", 7)
+ self.producer("find", 23)
+ self.producer("sort", 24)
+ result = self.run_runner("test_a.py", "test-b.sh")
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(self.observed_calls(), ["test_a.py", "test-b.sh"])
+ self.assertIn("Summary: 2/2 passed", result.stdout)
+
+ def test_unknown_explicit_test_stops_before_any_execution(self):
+ self.python_fixture("test_first.py")
+ result = self.run_runner("test_first.py", "test_missing.py")
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Unknown test: test_missing.py", result.stderr)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_failed_explicit_test_still_runs_remaining_selected_tests(self):
+ failing = self.python_fixture("test_first.py", 7)
+ passing = self.shell_fixture("test-second.sh")
+ result = self.run_runner(failing, passing)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(self.observed_calls(), ["test_first.py", "test-second.sh"])
+ self.assertIn("Summary: 1/2 passed", result.stdout)
+
+
+if __name__ == "__main__":
+ unittest.main()