Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

11.9 KB
Raw Blame
#!/usr/bin/env python3
"""Docs-only CI must execute every existing safety suite and fail closed.

Exercise the actual workflow commands with disposable test adapters. Real
recipes run separately in the source-proof VM, never against customer services.
"""

from __future__ import annotations

import fnmatch
import os
from pathlib import Path
import re
import shlex
import shutil
import subprocess
import sys
import tempfile
import textwrap
import unittest


ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".github/workflows/public-docs.yml"
SAFETY_STEP = "Exercise documentation safety and diagnostic recipes"
VERDICT_STEP = "Require every documentation recipe shard"
# Independent witnesses: all twelve formerly covered suites, plus the critical
# privacy/access/recovery checks that the former literal workflow omitted.
REQUIRED_SUITES = (
    "test_production_rollout_docs.py", "test_pmg_docs.py",
    "test_agent_profile_docs.py", "test_pbs_docs.py", "test_audit_recovery_docs.py",
    "test_notification_troubleshooting_docs.py", "test_webhook_verification_docs.py",
    "test_pve_backup_troubleshooting_docs.py", "test_vm_disk_diagnostics.py",
    "test_memory_troubleshooting_docs.py", "test_performance_troubleshooting_docs.py",
    "test_troubleshooting_logs.py", "test_admin_docs.py", "test_api_auth_docs.py",
    "test_authentication_diagnostic_docs.py", "test_update_recovery_docs.py",
    "test_guest_disk_reading_docs.py", "test_guest_network_troubleshooting_docs.py",
    "test_truenas_docs.py", "test_repo_docs_link_drift.py", "test_security_docs.py",
)
LEGACY_SUITES = {
    "test_vm_disk_diagnostics.py", "test_troubleshooting_logs.py",
    "test-retired-trial-acquisition-docs.sh",
}


def documentation_suites() -> list[str]:
    return sorted(path.name for path in (ROOT / "scripts/tests").iterdir()
                  if path.is_file() and (fnmatch.fnmatch(path.name, "test_*docs*.py")
                                         or path.name in LEGACY_SUITES))


def job_body(source: str, name: str) -> str:
    marker = f"\n  {name}:\n"
    if source.count(marker) != 1:
        raise AssertionError(f"expected exactly one {name} job")
    return re.split(r"\n  [\w-]+:\n", source.split(marker, 1)[1], maxsplit=1)[0]


def step_body(source: str, name: str) -> str:
    marker = f"      - name: {name}\n"
    if source.count(marker) != 1:
        raise AssertionError(f"expected exactly one {name} step")
    return source.split(marker, 1)[1].split("\n      - name:", 1)[0]


def safety_command(source: str, shard: int) -> str:
    body = step_body(job_body(source, "recipes"), SAFETY_STEP)
    run = re.findall(r"^        run: (.+)$", body, re.MULTILINE)
    if len(run) != 1:
        raise AssertionError("expected one literal docs-shard command")
    return run[0].replace("${{ matrix.shard }}", str(shard))


def verdict_command(source: str) -> str:
    body = step_body(job_body(source, "check"), VERDICT_STEP)
    marker = "        run: |\n"
    if body.count(marker) != 1:
        raise AssertionError("expected one literal verdict command")
    return textwrap.dedent(body.split(marker, 1)[1]).strip()


def workflow_fixture_environment(root: Path, calls: Path) -> dict[str, str]:
    # These exit adapters and the real selector need only the standard library.
    # Use this suite's interpreter without repeating site/package startup for
    # every suite in every single-failure control. Real safety suites still run
    # separately through the unchanged production runner and environment.
    tools = root / "tools"
    tools.mkdir()
    python = tools / "python3"
    python.write_text(
        f'#!/bin/sh\nexec {shlex.quote(sys.executable)} -I -S "$@"\n', encoding="utf-8")
    python.chmod(0o700)
    return dict(os.environ, PATH=f"{tools}{os.pathsep}{os.environ['PATH']}", CALLS=str(calls))


class PublicDocsWorkflowTest(unittest.TestCase):
    def setUp(self):
        self.source = WORKFLOW.read_text(encoding="utf-8")

    def test_both_events_cover_all_suite_and_helper_changes(self):
        triggers = self.source.split("\non:\n", 1)[1].split("\npermissions:", 1)[0]
        required = {
            "*.md", "docs/**", "frontend-modern/public/docs/**",
            ".github/ISSUE_TEMPLATE/**", ".github/PULL_REQUEST_TEMPLATE.md",
            "scripts/check_public_docs.py", "scripts/check_docs_mirror.py",
            "scripts/readme_entrypoints_test.py", "scripts/tests/**",
            "scripts/select-smoke-test-shard.py", ".github/scripts/docs-test-seconds.txt",
            "scripts/test-vm-disk.sh", ".github/workflows/public-docs.yml",
        }
        for event in ("pull_request", "push"):
            with self.subTest(event=event):
                block = triggers.split(f"  {event}:\n", 1)[1]
                block = re.split(r"\n  [a-z_]+:\n", block, maxsplit=1)[0]
                paths = re.findall(r'^      - "([^"]+)"$', block, re.MULTILINE)
                self.assertTrue(required.issubset(paths), required.difference(paths))
                self.assertEqual(len(paths), len(set(paths)))
        self.assertIn("  push:\n    branches:\n      - main\n", triggers)

    def test_shards_discover_current_suites_without_dependency_setup(self):
        self.assertTrue(set(REQUIRED_SUITES).issubset(documentation_suites()))
        for shard in range(4):
            self.assertEqual(shlex.split(safety_command(self.source, shard)),
                             ["scripts/tests/run.sh", "--docs-shard", str(shard)])
        self.assertNotIn("actions/setup-", self.source)
        self.assertNotRegex(self.source, r"\b(?:npm|pip|go) (?:ci|install|build|test)\b")

    def test_existing_document_admissions_are_preserved(self):
        check = job_body(self.source, "check")
        for name, command in (
            ("Validate public documentation", "python3 scripts/check_public_docs.py"),
            ("Check getting-started safety and current plans", "python3 scripts/readme_entrypoints_test.py"),
            ("Check shipped docs mirror sync", "python3 scripts/check_docs_mirror.py"),
            ("Check documentation safety workflow contract", "python3 scripts/tests/test_public_docs_workflow.py"),
        ):
            with self.subTest(step=name):
                self.assertIn(f"        run: {command}\n", step_body(check, name))

    def test_all_four_isolated_shards_are_mandatory_and_bounded(self):
        recipes = job_body(self.source, "recipes")
        self.assertIn("        shard: [0, 1, 2, 3]\n", recipes)
        self.assertIn("      fail-fast: false\n", recipes)
        self.assertNotRegex(recipes, r"(?m)^\s*(?:if|continue-on-error|exclude):")
        self.assertNotIn("continue-on-error:", self.source)
        for name in ("recipes", "check"):
            job = job_body(self.source, name)
            self.assertIn("    runs-on: ubuntu-24.04\n", job)
            self.assertIn("    timeout-minutes: 10\n", job)
            self.assertIn("          persist-credentials: false\n", job)
        self.assertIn("permissions:\n  contents: read\n", self.source)

    def test_required_check_runs_even_when_shards_fail_or_are_skipped(self):
        check = job_body(self.source, "check")
        self.assertIn("    needs: recipes\n", check)
        self.assertIn("    if: ${{ always() }}\n", check)
        self.assertNotRegex(check.replace("    if: ${{ always() }}\n", ""), r"(?m)^\s*if:")
        self.assertIn("          RECIPE_RESULT: ${{ needs.recipes.result }}\n", check)
        self.assertLess(check.index(VERDICT_STEP), check.index("Validate public documentation"))
        for state in ("success", "failure", "cancelled", "skipped", "", "unknown"):
            with self.subTest(state=state):
                result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", verdict_command(self.source)],
                                        env=dict(os.environ, RECIPE_RESULT=state), text=True,
                                        capture_output=True, timeout=5)
                self.assertEqual(result.returncode == 0, state == "success", result.stdout + result.stderr)

    def test_exit_adapters_use_the_selected_interpreter_without_site_startup(self):
        with tempfile.TemporaryDirectory() as temporary:
            root = Path(temporary)
            result = subprocess.run(
                ["python3", "-c", "import sys; print(sys.executable); "
                 "print(sys.flags.isolated, sys.flags.no_site)"],
                env=workflow_fixture_environment(root, root / "calls"),
                text=True, capture_output=True, timeout=5)
            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
            self.assertEqual(result.stdout.splitlines(), [sys.executable, "1 1"])

    def run_adapters(self, shard: int, failing: str = ""):
        with tempfile.TemporaryDirectory() as temporary:
            root = Path(temporary)
            tests = root / "scripts/tests"
            tests.mkdir(parents=True)
            shutil.copyfile(ROOT / "scripts/tests/run.sh", tests / "run.sh")
            (tests / "run.sh").chmod(0o700)
            shutil.copyfile(ROOT / "scripts/select-smoke-test-shard.py",
                            root / "scripts/select-smoke-test-shard.py")
            hints = root / ".github/scripts/docs-test-seconds.txt"
            hints.parent.mkdir(parents=True)
            shutil.copyfile(ROOT / ".github/scripts/docs-test-seconds.txt", hints)
            for name in documentation_suites():
                status = 1 if name == failing else 0
                if name.endswith(".sh"):
                    code = f'#!/bin/bash\nprintf "%s\\n" {shlex.quote(name)} >> "$CALLS"\nexit {status}\n'
                else:
                    code = ("import os\n"
                            f"with open(os.environ['CALLS'], 'a', encoding='utf-8') as log: log.write({name!r} + '\\n')\n"
                            f"raise SystemExit({status})\n")
                (tests / name).write_text(code, encoding="utf-8")
                (tests / name).chmod(0o700)
            calls = root / "calls"
            result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", safety_command(self.source, shard)],
                                    cwd=root, env=workflow_fixture_environment(root, calls),
                                    text=True, capture_output=True, timeout=15)
            return result, calls.read_text().splitlines() if calls.exists() else []

    def test_actual_workflow_commands_cover_every_suite_once(self):
        seen = []
        for shard in range(4):
            result, calls = self.run_adapters(shard)
            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
            self.assertTrue(calls)
            self.assertEqual(calls, sorted(calls))
            self.assertIn(f"Summary: {len(calls)}/{len(calls)} passed", result.stdout)
            seen.extend(calls)
        self.assertEqual(sorted(seen), documentation_suites())
        self.assertEqual(len(seen), len(set(seen)))

    def test_each_failed_suite_fails_its_actual_workflow_command(self):
        seen = []
        for shard in range(4):
            # Observe actual membership, not obsolete filename parity. The
            # complete-union assertion independently catches missing coverage.
            selected, names = self.run_adapters(shard)
            self.assertEqual(selected.returncode, 0, selected.stdout + selected.stderr)
            self.assertTrue(names)
            for failing in names:
                with self.subTest(failing=failing, shard=shard):
                    result, calls = self.run_adapters(shard, failing)
                    self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
                    self.assertEqual(calls, names)
                    self.assertIn(f"Summary: {len(calls)-1}/{len(calls)} passed", result.stdout)
                    self.assertIn("Failures: 1", result.stdout)
                    seen.append(failing)
        self.assertEqual(sorted(seen), documentation_suites())
        self.assertEqual(len(seen), len(set(seen)))


if __name__ == "__main__":
    unittest.main()