Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Run complete script smoke checks in two isolated CI shards

pulse-triage[bot]82f6257d45c4
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1e529672f0..c6d35ce61f 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -555,12 +555,16 @@ jobs:
fi
echo "Every internal/api shard passed."
- scripts-and-build:
- name: Script smoke tests & backend build
+ script-smoke:
+ name: Script smoke tests (shard ${{ matrix.shard }}/2)
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ shard: [0, 1]
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -583,7 +587,43 @@ jobs:
printf '<!doctype html><title>ci embed stub</title>\n' > internal/api/frontend-modern/dist/index.html
- name: Script smoke tests
- run: scripts/tests/run.sh
+ # Isolated runners preserve each recipe's deadlines and process cleanup.
+ # Discover all suites in both shards; never maintain a second allowlist.
+ run: scripts/tests/run.sh --smoke-shard ${{ matrix.shard }}
+
+ scripts-and-build:
+ name: Script smoke tests & backend build
+ needs: [changes, script-smoke]
+ if: always() && needs.changes.outputs.code == 'true'
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ steps:
+ - name: Require every script smoke shard
+ env:
+ SMOKE_RESULT: ${{ needs.script-smoke.result }}
+ run: |
+ if [ "$SMOKE_RESULT" != success ]; then
+ echo "::error::Script smoke shards did not all pass (result: ${SMOKE_RESULT})"
+ exit 1
+ fi
+ echo "Every script smoke shard passed."
+
+ - name: Checkout repository
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Set up Go
+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+ with:
+ go-version-file: go.mod
+ cache: true
+
+ - name: Provide frontend embed stub
+ run: |
+ mkdir -p internal/api/frontend-modern/dist
+ [ -f internal/api/frontend-modern/dist/index.html ] || \
+ printf '<!doctype html><title>ci embed stub</title>\n' > internal/api/frontend-modern/dist/index.html
- name: Build Pulse backend
run: go build ./cmd/pulse
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
index 9717c10c24..ff2f1ac627 100755
--- a/scripts/tests/run.sh
+++ b/scripts/tests/run.sh
@@ -10,11 +10,13 @@ TEST_DIR="${ROOT_DIR}/scripts/tests"
usage() {
cat <<'EOF'
Usage: scripts/tests/run.sh [test-script ...]
+ scripts/tests/run.sh --smoke-shard <0|1>
scripts/tests/run.sh --docs-shard <0|1|2|3>
Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subset
when specified. Documentation CI discovers every documentation suite and runs
one of four disjoint, sorted shards; it does not maintain a second suite list.
+Smoke CI partitions the complete inventory into two isolated, sorted shards.
EOF
}
@@ -123,6 +125,20 @@ main() {
tests+=("${doc_tests[index]}")
done
printf 'Documentation shard %s/4: %s of %s suites\n' "$2" "${#tests[@]}" "${#doc_tests[@]}"
+ elif [[ "${1:-}" == --smoke-shard ]]; then
+ if [[ $# -ne 2 || ! "${2:-}" =~ ^[0-1]$ ]]; then
+ echo "Usage: scripts/tests/run.sh --smoke-shard <0|1>" >&2
+ return 1
+ fi
+ local -a smoke_tests=()
+ if ! discover_tests smoke_tests; then
+ return 1
+ fi
+ local index
+ for ((index = $2; index < ${#smoke_tests[@]}; index += 2)); do
+ tests+=("${smoke_tests[index]}")
+ done
+ printf 'Smoke shard %s/2: %s of %s suites\n' "$2" "${#tests[@]}" "${#smoke_tests[@]}"
elif [[ $# -gt 0 ]]; then
local arg resolved
for arg in "$@"; do
diff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.py
index 10a0ff9339..7710cac481 100644
--- a/scripts/tests/test_smoke_runner.py
+++ b/scripts/tests/test_smoke_runner.py
@@ -231,6 +231,72 @@ class SmokeRunnerTest(unittest.TestCase):
self.assertIn("No tests found", empty.stderr)
self.assertEqual(self.observed_calls(), [])
+ def test_smoke_shards_cover_all_shell_python_and_future_suites_once(self):
+ names = ["test_a.py", "test_backend.py", "test_future.py", "test_recovery_docs.py",
+ "test_security.py", "test_z.py"]
+ for name in names:
+ self.python_fixture(name)
+ names.extend(["test-a.sh", "test-z.sh"])
+ for name in names[-2:]:
+ self.shell_fixture(name)
+ self.python_fixture("not_a_test.py", 7)
+ nested = self.tests / "nested"
+ nested.mkdir()
+ (nested / "test_nested.py").write_text("raise SystemExit(7)\n")
+ seen = []
+ for shard in range(2):
+ before = len(self.observed_calls())
+ result = self.run_runner("--smoke-shard", shard)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ calls = self.observed_calls()[before:]
+ self.assertEqual(calls, sorted(names)[shard::2])
+ seen.extend(calls)
+ self.assertIn(f"Smoke shard {shard}/2: 4 of 8 suites", result.stdout)
+ self.assertEqual(sorted(seen), sorted(names))
+ self.assertEqual(len(seen), len(set(seen)))
+
+ def test_smoke_shards_keep_failures_gating_and_continue_selected_suites(self):
+ names = [f"test_{index}.py" for index in range(6)]
+ for index, name in enumerate(names):
+ self.python_fixture(name, 7 if index < 2 else 0)
+ for shard in range(2):
+ before = len(self.observed_calls())
+ result = self.run_runner("--smoke-shard", shard)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(self.observed_calls()[before:], names[shard::2])
+ self.assertIn("Summary: 2/3 passed", result.stdout)
+ self.assertIn("Failures: 1", result.stdout)
+
+ def test_smoke_discovery_rejects_failed_find_and_sort_before_execution(self):
+ passing = self.python_fixture("test_first.py")
+ self.python_fixture("test_second.py", 7)
+ for name in ("find", "sort"):
+ with self.subTest(producer=name):
+ self.producer(name, 23, [passing])
+ self.assert_discovery_rejected(self.run_runner("--smoke-shard", 0))
+ (self.bin / name).unlink()
+
+ def test_invalid_smoke_shard_stops_before_discovery_or_execution(self):
+ self.python_fixture("test_first.py")
+ self.producer("find", 23)
+ for args in ((), ("-1",), ("2",), ("00",), ("one",), ("0", "test_first.py")):
+ with self.subTest(args=args):
+ result = self.run_runner("--smoke-shard", *args)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Usage:", result.stderr)
+ self.assertNotIn("Failed to discover", result.stderr)
+ self.assertEqual(self.observed_calls(), [])
+
+ def test_empty_smoke_inventory_or_shard_is_not_a_pass(self):
+ empty = self.run_runner("--smoke-shard", 0)
+ self.assertNotEqual(empty.returncode, 0)
+ self.assertIn("No tests found", empty.stderr)
+ self.python_fixture("test_first.py")
+ empty = self.run_runner("--smoke-shard", 1)
+ self.assertNotEqual(empty.returncode, 0)
+ self.assertIn("No tests found", empty.stderr)
+ self.assertEqual(self.observed_calls(), [])
+
if __name__ == "__main__":
unittest.main()
diff --git a/scripts/tests/test_smoke_workflow.py b/scripts/tests/test_smoke_workflow.py
new file mode 100644
index 0000000000..88caf177ae
--- /dev/null
+++ b/scripts/tests/test_smoke_workflow.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Every smoke suite remains required when CI runs two isolated inventories."""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+from pathlib import Path
+import re
+import shlex
+import shutil
+import subprocess
+import tempfile
+import textwrap
+import unittest
+
+from test_public_docs_workflow import job_body, step_body
+
+
+ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW = ROOT / ".github/workflows/build-and-test.yml"
+VERDICT_STEP = "Require every script smoke shard"
+REQUIRED_NAME = "Script smoke tests & backend build"
+
+
+def smoke_suites() -> list[str]:
+ return sorted(path.name for path in (ROOT / "scripts/tests").iterdir()
+ if path.is_file() and (fnmatch.fnmatch(path.name, "test-*.sh")
+ or fnmatch.fnmatch(path.name, "test_*.py")))
+
+
+def smoke_command(source: str, shard: int) -> str:
+ body = step_body(job_body(source, "script-smoke"), "Script smoke tests")
+ commands = re.findall(r"^ run: (.+)$", body, re.MULTILINE)
+ if len(commands) != 1:
+ raise AssertionError("expected one literal smoke-shard command")
+ return commands[0].replace("${{ matrix.shard }}", str(shard))
+
+
+def verdict_command(source: str) -> str:
+ body = step_body(job_body(source, "scripts-and-build"), VERDICT_STEP)
+ marker = " run: |\n"
+ if body.count(marker) != 1:
+ raise AssertionError("expected one literal smoke verdict command")
+ return textwrap.dedent(body.split(marker, 1)[1]).strip()
+
+
+class SmokeWorkflowTest(unittest.TestCase):
+ def setUp(self):
+ self.source = WORKFLOW.read_text(encoding="utf-8")
+
+ def test_two_isolated_complete_shards_keep_source_setup_and_deadlines(self):
+ smoke = job_body(self.source, "script-smoke")
+ for required in (
+ " needs: changes\n", " if: needs.changes.outputs.code == 'true'\n",
+ " runs-on: ubuntu-24.04\n", " timeout-minutes: 30\n",
+ " fail-fast: false\n", " shard: [0, 1]\n",
+ " persist-credentials: false\n", " fetch-depth: 0\n",
+ " go-version-file: go.mod\n", " cache: true\n",
+ " - name: Provide frontend embed stub\n",
+ ):
+ self.assertIn(required, smoke)
+ self.assertNotRegex(smoke, r"(?m)^\s*(?:include|exclude|continue-on-error):")
+ self.assertEqual(re.findall(r"(?m)^\s*if:.*$", smoke),
+ [" if: needs.changes.outputs.code == 'true'"])
+ self.assertNotIn("continue-on-error:", job_body(self.source, "scripts-and-build"))
+ self.assertIn("permissions:\n contents: read\n", self.source)
+ for shard in range(2):
+ self.assertEqual(shlex.split(smoke_command(self.source, shard)),
+ ["scripts/tests/run.sh", "--smoke-shard", str(shard)])
+
+ def test_existing_required_check_is_fail_closed(self):
+ build = job_body(self.source, "scripts-and-build")
+ self.assertIn(" needs: [changes, script-smoke]\n", build)
+ self.assertIn(" if: always() && needs.changes.outputs.code == 'true'\n", build)
+ self.assertEqual(self.source.count(f" name: {REQUIRED_NAME}\n"), 1)
+ self.assertIn(" SMOKE_RESULT: ${{ needs.script-smoke.result }}\n", build)
+ for state in ("success", "failure", "cancelled", "skipped", "", "unknown"):
+ with self.subTest(state=state):
+ result = subprocess.run(
+ ["bash", "-e", "-o", "pipefail", "-c", verdict_command(self.source)],
+ env=dict(os.environ, SMOKE_RESULT=state), text=True,
+ capture_output=True, timeout=5,
+ )
+ self.assertEqual(result.returncode == 0, state == "success",
+ result.stdout + result.stderr)
+
+ def test_backend_build_waits_for_gate_and_is_still_required(self):
+ build = job_body(self.source, "scripts-and-build")
+ self.assertLess(build.index(VERDICT_STEP), build.index("Checkout repository"))
+ self.assertLess(build.index(VERDICT_STEP), build.index("Build Pulse backend"))
+ self.assertIn(" runs-on: ubuntu-24.04\n", build)
+ self.assertIn(" timeout-minutes: 30\n", build)
+ self.assertIn(" go-version-file: go.mod\n", build)
+ self.assertIn(" persist-credentials: false\n", build)
+ self.assertIn(" - name: Provide frontend embed stub\n", build)
+ self.assertIn(" run: go build ./cmd/pulse\n",
+ step_body(build, "Build Pulse backend"))
+ self.assertEqual(re.findall(r"(?m)^\s*if:.*$", build),
+ [" if: always() && needs.changes.outputs.code == 'true'"])
+
+ def run_adapters(self, shard: int, failing: str = ""):
+ """Run actual workflow/runner commands on every source-selected name.
+
+ Only this fixture replaces suites with exit adapters; the full real
+ suites run separately in the source-proof VM.
+ """
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ tests = root / "scripts/tests"
+ tests.mkdir(parents=True)
+ shutil.copyfile(ROOT / "scripts/tests/run.sh", tests / "run.sh")
+ for name in smoke_suites():
+ status = 7 if name == failing else 0
+ if name.endswith(".sh"):
+ code = f'#!/bin/bash\nprintf "%s\\n" {shlex.quote(name)} >> "$CALLS"\nexit {status}\n'
+ else:
+ code = ("import os\nfrom pathlib import Path\n"
+ f"with Path(os.environ['CALLS']).open('a') as log: log.write({name!r} + '\\n')\n"
+ f"raise SystemExit({status})\n")
+ (tests / name).write_text(code, encoding="utf-8")
+ (tests / name).chmod(0o700)
+ calls = root / "calls"
+ # Bash is the hosted step's interpreter; execute the runner itself.
+ (tests / "run.sh").chmod(0o700)
+ result = subprocess.run(
+ ["bash", "-e", "-o", "pipefail", "-c", smoke_command(self.source, shard)],
+ cwd=root, env=dict(os.environ, CALLS=str(calls)), text=True,
+ capture_output=True, timeout=15,
+ )
+ return result, calls.read_text().splitlines() if calls.exists() else []
+
+ def test_actual_workflow_commands_cover_every_current_suite_once(self):
+ suites = smoke_suites()
+ self.assertIn("test_smoke_workflow.py", suites)
+ self.assertIn("test_performance_troubleshooting_docs.py", suites)
+ self.assertTrue(any(name.endswith(".sh") for name in suites))
+ seen = []
+ for shard in range(2):
+ result, calls = self.run_adapters(shard)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(calls, suites[shard::2])
+ self.assertIn(f"Summary: {len(calls)}/{len(calls)} passed", result.stdout)
+ seen.extend(calls)
+ self.assertEqual(sorted(seen), suites)
+ self.assertEqual(len(seen), len(set(seen)))
+
+ def test_failure_in_either_shard_keeps_remaining_suites_and_required_gate(self):
+ for shard in range(2):
+ names = smoke_suites()[shard::2]
+ result, calls = self.run_adapters(shard, names[0])
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(calls, names)
+ self.assertIn(f"Summary: {len(names)-1}/{len(names)} passed", result.stdout)
+ gate = subprocess.run(
+ ["bash", "-e", "-o", "pipefail", "-c", verdict_command(self.source)],
+ env=dict(os.environ, SMOKE_RESULT="failure"), text=True,
+ capture_output=True, timeout=5,
+ )
+ self.assertNotEqual(gate.returncode, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()