Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Run complete script smoke checks in two isolated CI shards
pulse-triage[bot]82f6257d45c4
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.ymlindex 1e529672f0..c6d35ce61f 100644--- a/.github/workflows/build-and-test.yml+++ b/.github/workflows/build-and-test.yml@@ -555,12 +555,16 @@ jobs:fiecho "Every internal/api shard passed."- scripts-and-build:- name: Script smoke tests & backend build+ script-smoke:+ name: Script smoke tests (shard ${{ matrix.shard }}/2)needs: changesif: needs.changes.outputs.code == 'true'runs-on: ubuntu-24.04timeout-minutes: 30+ strategy:+ fail-fast: false+ matrix:+ shard: [0, 1]steps:- name: Checkout repositoryuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -583,7 +587,43 @@ jobs:printf '<!doctype html><title>ci embed stub</title>\n' > internal/api/frontend-modern/dist/index.html- name: Script smoke tests- run: scripts/tests/run.sh+ # Isolated runners preserve each recipe's deadlines and process cleanup.+ # Discover all suites in both shards; never maintain a second allowlist.+ run: scripts/tests/run.sh --smoke-shard ${{ matrix.shard }}++ scripts-and-build:+ name: Script smoke tests & backend build+ needs: [changes, script-smoke]+ if: always() && needs.changes.outputs.code == 'true'+ runs-on: ubuntu-24.04+ timeout-minutes: 30+ steps:+ - name: Require every script smoke shard+ env:+ SMOKE_RESULT: ${{ needs.script-smoke.result }}+ run: |+ if [ "$SMOKE_RESULT" != success ]; then+ echo "::error::Script smoke shards did not all pass (result: ${SMOKE_RESULT})"+ exit 1+ fi+ echo "Every script smoke shard passed."++ - name: Checkout repository+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - name: Set up Go+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0+ with:+ go-version-file: go.mod+ cache: true++ - name: Provide frontend embed stub+ run: |+ mkdir -p internal/api/frontend-modern/dist+ [ -f internal/api/frontend-modern/dist/index.html ] || \+ printf '<!doctype html><title>ci embed stub</title>\n' > internal/api/frontend-modern/dist/index.html- name: Build Pulse backendrun: go build ./cmd/pulsediff --git a/scripts/tests/run.sh b/scripts/tests/run.shindex 9717c10c24..ff2f1ac627 100755--- a/scripts/tests/run.sh+++ b/scripts/tests/run.sh@@ -10,11 +10,13 @@ TEST_DIR="${ROOT_DIR}/scripts/tests"usage() {cat <<'EOF'Usage: scripts/tests/run.sh [test-script ...]+ scripts/tests/run.sh --smoke-shard <0|1>scripts/tests/run.sh --docs-shard <0|1|2|3>Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subsetwhen specified. Documentation CI discovers every documentation suite and runsone of four disjoint, sorted shards; it does not maintain a second suite list.+Smoke CI partitions the complete inventory into two isolated, sorted shards.EOF}@@ -123,6 +125,20 @@ main() {tests+=("${doc_tests[index]}")doneprintf 'Documentation shard %s/4: %s of %s suites\n' "$2" "${#tests[@]}" "${#doc_tests[@]}"+ elif [[ "${1:-}" == --smoke-shard ]]; then+ if [[ $# -ne 2 || ! "${2:-}" =~ ^[0-1]$ ]]; then+ echo "Usage: scripts/tests/run.sh --smoke-shard <0|1>" >&2+ return 1+ fi+ local -a smoke_tests=()+ if ! discover_tests smoke_tests; then+ return 1+ fi+ local index+ for ((index = $2; index < ${#smoke_tests[@]}; index += 2)); do+ tests+=("${smoke_tests[index]}")+ done+ printf 'Smoke shard %s/2: %s of %s suites\n' "$2" "${#tests[@]}" "${#smoke_tests[@]}"elif [[ $# -gt 0 ]]; thenlocal arg resolvedfor arg in "$@"; dodiff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.pyindex 10a0ff9339..7710cac481 100644--- a/scripts/tests/test_smoke_runner.py+++ b/scripts/tests/test_smoke_runner.py@@ -231,6 +231,72 @@ class SmokeRunnerTest(unittest.TestCase):self.assertIn("No tests found", empty.stderr)self.assertEqual(self.observed_calls(), [])+ def test_smoke_shards_cover_all_shell_python_and_future_suites_once(self):+ names = ["test_a.py", "test_backend.py", "test_future.py", "test_recovery_docs.py",+ "test_security.py", "test_z.py"]+ for name in names:+ self.python_fixture(name)+ names.extend(["test-a.sh", "test-z.sh"])+ for name in names[-2:]:+ self.shell_fixture(name)+ self.python_fixture("not_a_test.py", 7)+ nested = self.tests / "nested"+ nested.mkdir()+ (nested / "test_nested.py").write_text("raise SystemExit(7)\n")+ seen = []+ for shard in range(2):+ before = len(self.observed_calls())+ result = self.run_runner("--smoke-shard", shard)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ calls = self.observed_calls()[before:]+ self.assertEqual(calls, sorted(names)[shard::2])+ seen.extend(calls)+ self.assertIn(f"Smoke shard {shard}/2: 4 of 8 suites", result.stdout)+ self.assertEqual(sorted(seen), sorted(names))+ self.assertEqual(len(seen), len(set(seen)))++ def test_smoke_shards_keep_failures_gating_and_continue_selected_suites(self):+ names = [f"test_{index}.py" for index in range(6)]+ for index, name in enumerate(names):+ self.python_fixture(name, 7 if index < 2 else 0)+ for shard in range(2):+ before = len(self.observed_calls())+ result = self.run_runner("--smoke-shard", shard)+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(self.observed_calls()[before:], names[shard::2])+ self.assertIn("Summary: 2/3 passed", result.stdout)+ self.assertIn("Failures: 1", result.stdout)++ def test_smoke_discovery_rejects_failed_find_and_sort_before_execution(self):+ passing = self.python_fixture("test_first.py")+ self.python_fixture("test_second.py", 7)+ for name in ("find", "sort"):+ with self.subTest(producer=name):+ self.producer(name, 23, [passing])+ self.assert_discovery_rejected(self.run_runner("--smoke-shard", 0))+ (self.bin / name).unlink()++ def test_invalid_smoke_shard_stops_before_discovery_or_execution(self):+ self.python_fixture("test_first.py")+ self.producer("find", 23)+ for args in ((), ("-1",), ("2",), ("00",), ("one",), ("0", "test_first.py")):+ with self.subTest(args=args):+ result = self.run_runner("--smoke-shard", *args)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Usage:", result.stderr)+ self.assertNotIn("Failed to discover", result.stderr)+ self.assertEqual(self.observed_calls(), [])++ def test_empty_smoke_inventory_or_shard_is_not_a_pass(self):+ empty = self.run_runner("--smoke-shard", 0)+ self.assertNotEqual(empty.returncode, 0)+ self.assertIn("No tests found", empty.stderr)+ self.python_fixture("test_first.py")+ empty = self.run_runner("--smoke-shard", 1)+ self.assertNotEqual(empty.returncode, 0)+ self.assertIn("No tests found", empty.stderr)+ self.assertEqual(self.observed_calls(), [])+if __name__ == "__main__":unittest.main()diff --git a/scripts/tests/test_smoke_workflow.py b/scripts/tests/test_smoke_workflow.pynew file mode 100644index 0000000000..88caf177ae--- /dev/null+++ b/scripts/tests/test_smoke_workflow.py@@ -0,0 +1,164 @@+#!/usr/bin/env python3+"""Every smoke suite remains required when CI runs two isolated inventories."""++from __future__ import annotations++import fnmatch+import os+from pathlib import Path+import re+import shlex+import shutil+import subprocess+import tempfile+import textwrap+import unittest++from test_public_docs_workflow import job_body, step_body+++ROOT = Path(__file__).resolve().parents[2]+WORKFLOW = ROOT / ".github/workflows/build-and-test.yml"+VERDICT_STEP = "Require every script smoke shard"+REQUIRED_NAME = "Script smoke tests & backend build"+++def smoke_suites() -> list[str]:+ return sorted(path.name for path in (ROOT / "scripts/tests").iterdir()+ if path.is_file() and (fnmatch.fnmatch(path.name, "test-*.sh")+ or fnmatch.fnmatch(path.name, "test_*.py")))+++def smoke_command(source: str, shard: int) -> str:+ body = step_body(job_body(source, "script-smoke"), "Script smoke tests")+ commands = re.findall(r"^ run: (.+)$", body, re.MULTILINE)+ if len(commands) != 1:+ raise AssertionError("expected one literal smoke-shard command")+ return commands[0].replace("${{ matrix.shard }}", str(shard))+++def verdict_command(source: str) -> str:+ body = step_body(job_body(source, "scripts-and-build"), VERDICT_STEP)+ marker = " run: |\n"+ if body.count(marker) != 1:+ raise AssertionError("expected one literal smoke verdict command")+ return textwrap.dedent(body.split(marker, 1)[1]).strip()+++class SmokeWorkflowTest(unittest.TestCase):+ def setUp(self):+ self.source = WORKFLOW.read_text(encoding="utf-8")++ def test_two_isolated_complete_shards_keep_source_setup_and_deadlines(self):+ smoke = job_body(self.source, "script-smoke")+ for required in (+ " needs: changes\n", " if: needs.changes.outputs.code == 'true'\n",+ " runs-on: ubuntu-24.04\n", " timeout-minutes: 30\n",+ " fail-fast: false\n", " shard: [0, 1]\n",+ " persist-credentials: false\n", " fetch-depth: 0\n",+ " go-version-file: go.mod\n", " cache: true\n",+ " - name: Provide frontend embed stub\n",+ ):+ self.assertIn(required, smoke)+ self.assertNotRegex(smoke, r"(?m)^\s*(?:include|exclude|continue-on-error):")+ self.assertEqual(re.findall(r"(?m)^\s*if:.*$", smoke),+ [" if: needs.changes.outputs.code == 'true'"])+ self.assertNotIn("continue-on-error:", job_body(self.source, "scripts-and-build"))+ self.assertIn("permissions:\n contents: read\n", self.source)+ for shard in range(2):+ self.assertEqual(shlex.split(smoke_command(self.source, shard)),+ ["scripts/tests/run.sh", "--smoke-shard", str(shard)])++ def test_existing_required_check_is_fail_closed(self):+ build = job_body(self.source, "scripts-and-build")+ self.assertIn(" needs: [changes, script-smoke]\n", build)+ self.assertIn(" if: always() && needs.changes.outputs.code == 'true'\n", build)+ self.assertEqual(self.source.count(f" name: {REQUIRED_NAME}\n"), 1)+ self.assertIn(" SMOKE_RESULT: ${{ needs.script-smoke.result }}\n", build)+ for state in ("success", "failure", "cancelled", "skipped", "", "unknown"):+ with self.subTest(state=state):+ result = subprocess.run(+ ["bash", "-e", "-o", "pipefail", "-c", verdict_command(self.source)],+ env=dict(os.environ, SMOKE_RESULT=state), text=True,+ capture_output=True, timeout=5,+ )+ self.assertEqual(result.returncode == 0, state == "success",+ result.stdout + result.stderr)++ def test_backend_build_waits_for_gate_and_is_still_required(self):+ build = job_body(self.source, "scripts-and-build")+ self.assertLess(build.index(VERDICT_STEP), build.index("Checkout repository"))+ self.assertLess(build.index(VERDICT_STEP), build.index("Build Pulse backend"))+ self.assertIn(" runs-on: ubuntu-24.04\n", build)+ self.assertIn(" timeout-minutes: 30\n", build)+ self.assertIn(" go-version-file: go.mod\n", build)+ self.assertIn(" persist-credentials: false\n", build)+ self.assertIn(" - name: Provide frontend embed stub\n", build)+ self.assertIn(" run: go build ./cmd/pulse\n",+ step_body(build, "Build Pulse backend"))+ self.assertEqual(re.findall(r"(?m)^\s*if:.*$", build),+ [" if: always() && needs.changes.outputs.code == 'true'"])++ def run_adapters(self, shard: int, failing: str = ""):+ """Run actual workflow/runner commands on every source-selected name.++ Only this fixture replaces suites with exit adapters; the full real+ suites run separately in the source-proof VM.+ """+ with tempfile.TemporaryDirectory() as temporary:+ root = Path(temporary)+ tests = root / "scripts/tests"+ tests.mkdir(parents=True)+ shutil.copyfile(ROOT / "scripts/tests/run.sh", tests / "run.sh")+ for name in smoke_suites():+ status = 7 if name == failing else 0+ if name.endswith(".sh"):+ code = f'#!/bin/bash\nprintf "%s\\n" {shlex.quote(name)} >> "$CALLS"\nexit {status}\n'+ else:+ code = ("import os\nfrom pathlib import Path\n"+ f"with Path(os.environ['CALLS']).open('a') as log: log.write({name!r} + '\\n')\n"+ f"raise SystemExit({status})\n")+ (tests / name).write_text(code, encoding="utf-8")+ (tests / name).chmod(0o700)+ calls = root / "calls"+ # Bash is the hosted step's interpreter; execute the runner itself.+ (tests / "run.sh").chmod(0o700)+ result = subprocess.run(+ ["bash", "-e", "-o", "pipefail", "-c", smoke_command(self.source, shard)],+ cwd=root, env=dict(os.environ, CALLS=str(calls)), text=True,+ capture_output=True, timeout=15,+ )+ return result, calls.read_text().splitlines() if calls.exists() else []++ def test_actual_workflow_commands_cover_every_current_suite_once(self):+ suites = smoke_suites()+ self.assertIn("test_smoke_workflow.py", suites)+ self.assertIn("test_performance_troubleshooting_docs.py", suites)+ self.assertTrue(any(name.endswith(".sh") for name in suites))+ seen = []+ for shard in range(2):+ result, calls = self.run_adapters(shard)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(calls, suites[shard::2])+ self.assertIn(f"Summary: {len(calls)}/{len(calls)} passed", result.stdout)+ seen.extend(calls)+ self.assertEqual(sorted(seen), suites)+ self.assertEqual(len(seen), len(set(seen)))++ def test_failure_in_either_shard_keeps_remaining_suites_and_required_gate(self):+ for shard in range(2):+ names = smoke_suites()[shard::2]+ result, calls = self.run_adapters(shard, names[0])+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(calls, names)+ self.assertIn(f"Summary: {len(names)-1}/{len(names)} passed", result.stdout)+ gate = subprocess.run(+ ["bash", "-e", "-o", "pipefail", "-c", verdict_command(self.source)],+ env=dict(os.environ, SMOKE_RESULT="failure"), text=True,+ capture_output=True, timeout=5,+ )+ self.assertNotEqual(gate.returncode, 0)+++if __name__ == "__main__":+ unittest.main()