Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Run complete documentation safety suites on docs-only changes
pulse-triage[bot]54f40a6a7634
diff --git a/.github/workflows/public-docs.yml b/.github/workflows/public-docs.ymlindex 700293558e..c5ef3af498 100644--- a/.github/workflows/public-docs.yml+++ b/.github/workflows/public-docs.yml@@ -7,23 +7,11 @@ on:- "docs/**"- "frontend-modern/public/docs/**"- ".github/ISSUE_TEMPLATE/**"+ - ".github/PULL_REQUEST_TEMPLATE.md"- "scripts/check_public_docs.py"- "scripts/check_docs_mirror.py"- "scripts/readme_entrypoints_test.py"- - "scripts/tests/run.sh"- - "scripts/tests/test_public_docs_workflow.py"- - "scripts/tests/test_production_rollout_docs.py"- - "scripts/tests/test_pmg_docs.py"- - "scripts/tests/test_agent_profile_docs.py"- - "scripts/tests/test_pbs_docs.py"- - "scripts/tests/test_audit_recovery_docs.py"- - "scripts/tests/test_notification_troubleshooting_docs.py"- - "scripts/tests/test_webhook_verification_docs.py"- - "scripts/tests/test_pve_backup_troubleshooting_docs.py"- - "scripts/tests/test_vm_disk_diagnostics.py"- - "scripts/tests/test_memory_troubleshooting_docs.py"- - "scripts/tests/test_performance_troubleshooting_docs.py"- - "scripts/tests/test_troubleshooting_logs.py"+ - "scripts/tests/**"- "scripts/test-vm-disk.sh"- ".github/workflows/public-docs.yml"push:@@ -34,23 +22,11 @@ on:- "docs/**"- "frontend-modern/public/docs/**"- ".github/ISSUE_TEMPLATE/**"+ - ".github/PULL_REQUEST_TEMPLATE.md"- "scripts/check_public_docs.py"- "scripts/check_docs_mirror.py"- "scripts/readme_entrypoints_test.py"- - "scripts/tests/run.sh"- - "scripts/tests/test_public_docs_workflow.py"- - "scripts/tests/test_production_rollout_docs.py"- - "scripts/tests/test_pmg_docs.py"- - "scripts/tests/test_agent_profile_docs.py"- - "scripts/tests/test_pbs_docs.py"- - "scripts/tests/test_audit_recovery_docs.py"- - "scripts/tests/test_notification_troubleshooting_docs.py"- - "scripts/tests/test_webhook_verification_docs.py"- - "scripts/tests/test_pve_backup_troubleshooting_docs.py"- - "scripts/tests/test_vm_disk_diagnostics.py"- - "scripts/tests/test_memory_troubleshooting_docs.py"- - "scripts/tests/test_performance_troubleshooting_docs.py"- - "scripts/tests/test_troubleshooting_logs.py"+ - "scripts/tests/**"- "scripts/test-vm-disk.sh"- ".github/workflows/public-docs.yml"@@ -62,7 +38,30 @@ concurrency:cancel-in-progress: truejobs:+ recipes:+ name: Documentation recipes (${{ matrix.shard }})+ runs-on: ubuntu-24.04+ timeout-minutes: 10+ strategy:+ fail-fast: false+ matrix:+ shard: [0, 1, 2, 3]+ steps:+ - name: Check out repository+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ # Build and Test skips smoke for docs-only changes. Discover the complete+ # current docs suite here, including privacy, authentication and recovery.+ # Isolated shards retain all recipe deadlines without a longer CI path.+ - name: Exercise documentation safety and diagnostic recipes+ run: scripts/tests/run.sh --docs-shard "${{ matrix.shard }}"++ # Preserve the existing required check name and fail closed on every shard.check:+ needs: recipes+ if: ${{ always() }}runs-on: ubuntu-24.04timeout-minutes: 10steps:@@ -71,6 +70,15 @@ jobs:with:persist-credentials: false+ - name: Require every documentation recipe shard+ env:+ RECIPE_RESULT: ${{ needs.recipes.result }}+ run: |+ if [[ "$RECIPE_RESULT" != success ]]; then+ echo "Documentation recipe shards did not all succeed: $RECIPE_RESULT" >&2+ exit 1+ fi+- name: Validate public documentationrun: python3 scripts/check_public_docs.py@@ -82,22 +90,3 @@ jobs:- name: Check documentation safety workflow contractrun: python3 scripts/tests/test_public_docs_workflow.py-- # Build and Test intentionally skips its smoke suite for docs-only- # changes. Exercise the existing safety guidance and copied recipes here- # too, without installing frontend/Go dependencies or calling providers.- - name: Exercise backup safety and diagnostic recipes- run: |- scripts/tests/run.sh \- test_production_rollout_docs.py \- test_pmg_docs.py \- test_agent_profile_docs.py \- test_pbs_docs.py \- test_audit_recovery_docs.py \- test_notification_troubleshooting_docs.py \- test_webhook_verification_docs.py \- test_pve_backup_troubleshooting_docs.py \- test_vm_disk_diagnostics.py \- test_memory_troubleshooting_docs.py \- test_performance_troubleshooting_docs.py \- test_troubleshooting_logs.pydiff --git a/scripts/tests/run.sh b/scripts/tests/run.shindex b066faa26c..9717c10c24 100755--- a/scripts/tests/run.sh+++ b/scripts/tests/run.sh@@ -10,20 +10,30 @@ TEST_DIR="${ROOT_DIR}/scripts/tests"usage() {cat <<'EOF'Usage: scripts/tests/run.sh [test-script ...]+ scripts/tests/run.sh --docs-shard <0|1|2|3>Run all scripts/tests/test-*.sh and scripts/tests/test_*.py tests or a subset-when specified.+when specified. Documentation CI discovers every documentation suite and runs+one of four disjoint, sorted shards; it does not maintain a second suite list.EOF}discover_tests() {local -n ref=$1+ local mode="${2:-all}"local inventory+ local -a patterns=(-name 'test-*.sh' -o -name 'test_*.py')+ if [[ "${mode}" == docs ]]; then+ # Keep the established safety helpers whose names predate the docs suffix.+ patterns=(-name 'test_*docs*.py' -o -name 'test_vm_disk_diagnostics.py'+ -o -name 'test_troubleshooting_logs.py'+ -o -name 'test-retired-trial-acquisition-docs.sh')+ fi# Process substitution hides producer failures from mapfile. Admit the# complete discovery pipeline before running any tests, never a partial list.if ! inventory="$(find "${TEST_DIR}" -maxdepth 1 -type f \- \( -name 'test-*.sh' -o -name 'test_*.py' \) | sort+ \( "${patterns[@]}" \) | LC_ALL=C sort)"; thenecho "Failed to discover the complete smoke test inventory under ${TEST_DIR}" >&2return 1@@ -99,7 +109,21 @@ main() {filocal -a tests=()- if [[ $# -gt 0 ]]; then+ if [[ "${1:-}" == --docs-shard ]]; then+ if [[ $# -ne 2 || ! "${2:-}" =~ ^[0-3]$ ]]; then+ echo "Usage: scripts/tests/run.sh --docs-shard <0|1|2|3>" >&2+ return 1+ fi+ local -a doc_tests=()+ if ! discover_tests doc_tests docs; then+ return 1+ fi+ local index+ for ((index = $2; index < ${#doc_tests[@]}; index += 4)); do+ tests+=("${doc_tests[index]}")+ done+ printf 'Documentation shard %s/4: %s of %s suites\n' "$2" "${#tests[@]}" "${#doc_tests[@]}"+ elif [[ $# -gt 0 ]]; thenlocal arg resolvedfor arg in "$@"; doif ! resolved="$(resolve_test_path "${arg}")"; thendiff --git a/scripts/tests/test_public_docs_workflow.py b/scripts/tests/test_public_docs_workflow.pyindex 76e6e8441b..411f722e17 100644--- a/scripts/tests/test_public_docs_workflow.py+++ b/scripts/tests/test_public_docs_workflow.py@@ -1,13 +1,13 @@#!/usr/bin/env python3-"""Docs-only CI must exercise safety guidance, not just its Markdown shape.+"""Docs-only CI must execute every existing safety suite and fail closed.-Read the canonical block-YAML surface (also checked by workflow trust), then-execute its actual smoke command with disposable test adapters. No provider,-host service, backup or guest-agent operation is used.+Exercise the actual workflow commands with disposable test adapters. Real+recipes run separately in the source-proof VM, never against customer services."""from __future__ import annotations+import fnmatchimport osfrom pathlib import Pathimport re@@ -21,21 +21,38 @@ import unittestROOT = Path(__file__).resolve().parents[2]WORKFLOW = ROOT / ".github/workflows/public-docs.yml"-SAFETY_STEP = "Exercise backup safety and diagnostic recipes"-TESTS = (- "test_production_rollout_docs.py",- "test_pmg_docs.py",- "test_agent_profile_docs.py",- "test_pbs_docs.py",- "test_audit_recovery_docs.py",- "test_notification_troubleshooting_docs.py",- "test_webhook_verification_docs.py",- "test_pve_backup_troubleshooting_docs.py",- "test_vm_disk_diagnostics.py",- "test_memory_troubleshooting_docs.py",- "test_performance_troubleshooting_docs.py",- "test_troubleshooting_logs.py",+SAFETY_STEP = "Exercise documentation safety and diagnostic recipes"+VERDICT_STEP = "Require every documentation recipe shard"+# Independent witnesses: all twelve formerly covered suites, plus the critical+# privacy/access/recovery checks that the former literal workflow omitted.+REQUIRED_SUITES = (+ "test_production_rollout_docs.py", "test_pmg_docs.py",+ "test_agent_profile_docs.py", "test_pbs_docs.py", "test_audit_recovery_docs.py",+ "test_notification_troubleshooting_docs.py", "test_webhook_verification_docs.py",+ "test_pve_backup_troubleshooting_docs.py", "test_vm_disk_diagnostics.py",+ "test_memory_troubleshooting_docs.py", "test_performance_troubleshooting_docs.py",+ "test_troubleshooting_logs.py", "test_admin_docs.py", "test_api_auth_docs.py",+ "test_authentication_diagnostic_docs.py", "test_update_recovery_docs.py",+ "test_guest_disk_reading_docs.py", "test_guest_network_troubleshooting_docs.py",+ "test_truenas_docs.py", "test_repo_docs_link_drift.py", "test_security_docs.py",)+LEGACY_SUITES = {+ "test_vm_disk_diagnostics.py", "test_troubleshooting_logs.py",+ "test-retired-trial-acquisition-docs.sh",+}+++def documentation_suites() -> list[str]:+ return sorted(path.name for path in (ROOT / "scripts/tests").iterdir()+ if path.is_file() and (fnmatch.fnmatch(path.name, "test_*docs*.py")+ or path.name in LEGACY_SUITES))+++def job_body(source: str, name: str) -> str:+ marker = f"\n {name}:\n"+ if source.count(marker) != 1:+ raise AssertionError(f"expected exactly one {name} job")+ return re.split(r"\n [\w-]+:\n", source.split(marker, 1)[1], maxsplit=1)[0]def step_body(source: str, name: str) -> str:@@ -45,11 +62,19 @@ def step_body(source: str, name: str) -> str:return source.split(marker, 1)[1].split("\n - name:", 1)[0]-def safety_command(source: str) -> str:- body = step_body(source, SAFETY_STEP)+def safety_command(source: str, shard: int) -> str:+ body = step_body(job_body(source, "recipes"), SAFETY_STEP)+ run = re.findall(r"^ run: (.+)$", body, re.MULTILINE)+ if len(run) != 1:+ raise AssertionError("expected one literal docs-shard command")+ return run[0].replace("${{ matrix.shard }}", str(shard))+++def verdict_command(source: str) -> str:+ body = step_body(job_body(source, "check"), VERDICT_STEP)marker = " run: |\n"if body.count(marker) != 1:- raise AssertionError("expected one literal safety command")+ raise AssertionError("expected one literal verdict command")return textwrap.dedent(body.split(marker, 1)[1]).strip()@@ -57,15 +82,14 @@ class PublicDocsWorkflowTest(unittest.TestCase):def setUp(self):self.source = WORKFLOW.read_text(encoding="utf-8")- def test_both_events_cover_docs_checks_runner_and_helper(self):+ def test_both_events_cover_all_suite_and_helper_changes(self):triggers = self.source.split("\non:\n", 1)[1].split("\npermissions:", 1)[0]required = {"*.md", "docs/**", "frontend-modern/public/docs/**",+ ".github/ISSUE_TEMPLATE/**", ".github/PULL_REQUEST_TEMPLATE.md","scripts/check_public_docs.py", "scripts/check_docs_mirror.py",- "scripts/readme_entrypoints_test.py", "scripts/tests/run.sh",- "scripts/tests/test_public_docs_workflow.py", "scripts/test-vm-disk.sh",- ".github/workflows/public-docs.yml",- *(f"scripts/tests/{name}" for name in TESTS),+ "scripts/readme_entrypoints_test.py", "scripts/tests/**",+ "scripts/test-vm-disk.sh", ".github/workflows/public-docs.yml",}for event in ("pull_request", "push"):with self.subTest(event=event):@@ -76,15 +100,16 @@ class PublicDocsWorkflowTest(unittest.TestCase):self.assertEqual(len(paths), len(set(paths)))self.assertIn(" push:\n branches:\n - main\n", triggers)- def test_exact_existing_suites_run_without_dependency_setup(self):- command = safety_command(self.source).replace("\\\n", " ")- self.assertEqual(shlex.split(command), ["scripts/tests/run.sh", *TESTS])- for name in TESTS:- self.assertTrue((ROOT / "scripts/tests" / name).is_file())+ def test_shards_discover_current_suites_without_dependency_setup(self):+ self.assertTrue(set(REQUIRED_SUITES).issubset(documentation_suites()))+ for shard in range(4):+ self.assertEqual(shlex.split(safety_command(self.source, shard)),+ ["scripts/tests/run.sh", "--docs-shard", str(shard)])self.assertNotIn("actions/setup-", self.source)self.assertNotRegex(self.source, r"\b(?:npm|pip|go) (?:ci|install|build|test)\b")def test_existing_document_admissions_are_preserved(self):+ check = job_body(self.source, "check")for name, command in (("Validate public documentation", "python3 scripts/check_public_docs.py"),("Check getting-started safety and current plans", "python3 scripts/readme_entrypoints_test.py"),@@ -92,49 +117,77 @@ class PublicDocsWorkflowTest(unittest.TestCase):("Check documentation safety workflow contract", "python3 scripts/tests/test_public_docs_workflow.py"),):with self.subTest(step=name):- self.assertIn(f" run: {command}\n", step_body(self.source, name))-- def test_safety_checks_cannot_be_conditionally_skipped_or_softened(self):- self.assertNotRegex(self.source, r"(?m)^\s*(?:if|continue-on-error):")+ self.assertIn(f" run: {command}\n", step_body(check, name))++ def test_all_four_isolated_shards_are_mandatory_and_bounded(self):+ recipes = job_body(self.source, "recipes")+ self.assertIn(" shard: [0, 1, 2, 3]\n", recipes)+ self.assertIn(" fail-fast: false\n", recipes)+ self.assertNotRegex(recipes, r"(?m)^\s*(?:if|continue-on-error|exclude):")+ self.assertNotIn("continue-on-error:", self.source)+ for name in ("recipes", "check"):+ job = job_body(self.source, name)+ self.assertIn(" runs-on: ubuntu-24.04\n", job)+ self.assertIn(" timeout-minutes: 10\n", job)+ self.assertIn(" persist-credentials: false\n", job)self.assertIn("permissions:\n contents: read\n", self.source)- self.assertIn(" persist-credentials: false\n", self.source)- self.assertIn(" timeout-minutes: 10\n", self.source)- def run_adapters(self, failing: str = ""):+ def test_required_check_runs_even_when_shards_fail_or_are_skipped(self):+ check = job_body(self.source, "check")+ self.assertIn(" needs: recipes\n", check)+ self.assertIn(" if: ${{ always() }}\n", check)+ self.assertNotRegex(check.replace(" if: ${{ always() }}\n", ""), r"(?m)^\s*if:")+ self.assertIn(" RECIPE_RESULT: ${{ needs.recipes.result }}\n", check)+ self.assertLess(check.index(VERDICT_STEP), check.index("Validate public documentation"))+ for state in ("success", "failure", "cancelled", "skipped", "", "unknown"):+ with self.subTest(state=state):+ result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", verdict_command(self.source)],+ env=dict(os.environ, RECIPE_RESULT=state), text=True,+ capture_output=True, timeout=5)+ self.assertEqual(result.returncode == 0, state == "success", result.stdout + result.stderr)++ def run_adapters(self, shard: int, failing: str = ""):with tempfile.TemporaryDirectory() as temporary:root = Path(temporary)tests = root / "scripts/tests"tests.mkdir(parents=True)shutil.copyfile(ROOT / "scripts/tests/run.sh", tests / "run.sh")(tests / "run.sh").chmod(0o700)- for name in TESTS:- (tests / name).write_text(- "import os\nfrom pathlib import Path\n"- f"with Path(os.environ['CALLS']).open('a') as log: log.write({name!r} + '\\n')\n"- f"raise SystemExit({1 if name == failing else 0})\n",- encoding="utf-8",- )+ for name in documentation_suites():+ status = 1 if name == failing else 0+ if name.endswith(".sh"):+ code = f'#!/bin/bash\nprintf "%s\\n" {shlex.quote(name)} >> "$CALLS"\nexit {status}\n'+ else:+ code = ("import os\nfrom pathlib import Path\n"+ f"with Path(os.environ['CALLS']).open('a') as log: log.write({name!r} + '\\n')\n"+ f"raise SystemExit({status})\n")+ (tests / name).write_text(code, encoding="utf-8")+ (tests / name).chmod(0o700)calls = root / "calls"- result = subprocess.run(- ["bash", "-e", "-o", "pipefail", "-c", safety_command(self.source)],- cwd=root, env=dict(os.environ, CALLS=str(calls)),- text=True, capture_output=True, timeout=15,- )- return result, calls.read_text().splitlines()-- def test_actual_workflow_command_runs_every_suite(self):- result, calls = self.run_adapters()- self.assertEqual(result.returncode, 0, result.stdout + result.stderr)- self.assertEqual(calls, list(TESTS))- self.assertIn(f"Summary: {len(TESTS)}/{len(TESTS)} passed", result.stdout)-- def test_each_failed_suite_is_a_failed_workflow_command(self):- for failing in TESTS:+ result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", safety_command(self.source, shard)],+ cwd=root, env=dict(os.environ, CALLS=str(calls)),+ text=True, capture_output=True, timeout=15)+ return result, calls.read_text().splitlines() if calls.exists() else []++ def test_actual_workflow_commands_cover_every_suite_once(self):+ seen = []+ for shard in range(4):+ result, calls = self.run_adapters(shard)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(calls, documentation_suites()[shard::4])+ self.assertIn(f"Summary: {len(calls)}/{len(calls)} passed", result.stdout)+ seen.extend(calls)+ self.assertEqual(sorted(seen), documentation_suites())+ self.assertEqual(len(seen), len(set(seen)))++ def test_each_failed_suite_fails_its_actual_workflow_command(self):+ for index, failing in enumerate(documentation_suites()):with self.subTest(failing=failing):- result, calls = self.run_adapters(failing)+ shard = index % 4+ result, calls = self.run_adapters(shard, failing)self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)- self.assertEqual(calls, list(TESTS))- self.assertIn(f"Summary: {len(TESTS)-1}/{len(TESTS)} passed", result.stdout)+ self.assertEqual(calls, documentation_suites()[shard::4])+ self.assertIn(f"Summary: {len(calls)-1}/{len(calls)} passed", result.stdout)self.assertIn("Failures: 1", result.stdout)diff --git a/scripts/tests/test_smoke_runner.py b/scripts/tests/test_smoke_runner.pyindex c8d78e4feb..10a0ff9339 100644--- a/scripts/tests/test_smoke_runner.py+++ b/scripts/tests/test_smoke_runner.py@@ -163,6 +163,74 @@ class SmokeRunnerTest(unittest.TestCase):self.assertEqual(self.observed_calls(), ["test_first.py", "test-second.sh"])self.assertIn("Summary: 1/2 passed", result.stdout)+ def test_docs_shards_cover_current_and_future_suites_once_without_unrelated_tests(self):+ names = ["test_admin_docs.py", "test_api_auth_docs.py", "test_docs_reader.py",+ "test_future_docs.py", "test_recovery_docs.py", "test_security_docs.py",+ "test_troubleshooting_logs.py", "test_vm_disk_diagnostics.py"]+ for name in names:+ self.python_fixture(name)+ names.append("test-retired-trial-acquisition-docs.sh")+ self.shell_fixture(names[-1])+ self.python_fixture("test_backend.py", 7)+ self.shell_fixture("test-agent-runtime.sh", 7)+ nested = self.tests / "nested"+ nested.mkdir()+ (nested / "test_nested_docs.py").write_text("raise SystemExit(7)\n")+ seen = []+ for shard in range(4):+ before = len(self.observed_calls())+ result = self.run_runner("--docs-shard", shard)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ calls = self.observed_calls()[before:]+ self.assertEqual(calls, sorted(names)[shard::4])+ seen.extend(calls)+ self.assertIn(f"Documentation shard {shard}/4:", result.stdout)+ self.assertEqual(sorted(seen), sorted(names))+ self.assertEqual(len(seen), len(set(seen)))++ def test_every_doc_failure_remains_gating_and_collects_the_remaining_shard(self):+ names = [f"test_{index}_docs.py" for index in range(8)]+ for index, name in enumerate(names):+ self.python_fixture(name, 7 if index < 4 else 0)+ for shard in range(4):+ before = len(self.observed_calls())+ result = self.run_runner("--docs-shard", shard)+ self.assertNotEqual(result.returncode, 0)+ self.assertEqual(self.observed_calls()[before:], names[shard::4])+ self.assertIn("Summary: 1/2 passed", result.stdout)+ self.assertIn("Failures: 1", result.stdout)++ def test_docs_discovery_rejects_failed_find_and_sort_before_execution(self):+ passing = self.python_fixture("test_first_docs.py")+ self.python_fixture("test_second_docs.py", 7)+ for name in ("find", "sort"):+ with self.subTest(producer=name):+ self.producer(name, 23, [passing])+ self.assert_discovery_rejected(self.run_runner("--docs-shard", 0))+ (self.bin / name).unlink()++ def test_invalid_doc_shard_arguments_stop_before_discovery_or_execution(self):+ self.python_fixture("test_first_docs.py")+ self.producer("find", 23)+ for args in ((), ("-1",), ("4",), ("00",), ("one",), ("0", "test_first_docs.py")):+ with self.subTest(args=args):+ result = self.run_runner("--docs-shard", *args)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Usage:", result.stderr)+ self.assertNotIn("Failed to discover", result.stderr)+ self.assertEqual(self.observed_calls(), [])++ def test_empty_doc_inventory_or_shard_is_not_a_pass(self):+ self.python_fixture("test_unrelated.py")+ empty = self.run_runner("--docs-shard", 0)+ self.assertNotEqual(empty.returncode, 0)+ self.assertIn("No tests found", empty.stderr)+ self.python_fixture("test_first_docs.py")+ empty = self.run_runner("--docs-shard", 3)+ self.assertNotEqual(empty.returncode, 0)+ self.assertIn("No tests found", empty.stderr)+ self.assertEqual(self.observed_calls(), [])+if __name__ == "__main__":unittest.main()