Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.2 KB
Raw Blame
"""Tests for the check-release-gate-needs pre-commit hook.

Two tripwires live in that hook:

* The original one (#4817): a SARIF-uploading scan missing from either
  release-gate.yml needs list silently bypasses the code-scanning alert
  gate (grype did exactly that for ~1.5 years).
* The repo-wide local-ref existence pass added with the self-repository
  (`uses: $/...`) conversion: every local callee must exist, including
  those referenced by callers outside the changed-file list. This
  complements the actionlint adapter's workflow-interface checks.
"""

import importlib.util
from pathlib import Path

import pytest

HOOKS_DIR = Path(__file__).parent.parent.parent / ".pre-commit-hooks"


def _load_hook():
    """Import the dash-named hook script as a module."""
    spec = importlib.util.spec_from_file_location(
        "check_release_gate_needs", HOOKS_DIR / "check-release-gate-needs.py"
    )
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


@pytest.fixture
def hook():
    return _load_hook()


def _write_workflow(root: Path, name: str, text: str) -> Path:
    wf_dir = root / ".github" / "workflows"
    wf_dir.mkdir(parents=True, exist_ok=True)
    path = wf_dir / name
    path.write_text(text, encoding="utf-8")
    return path


# A release-gate skeleton with one inline SARIF uploader and one
# workflow-calling uploader, both present in both consumer needs lists.
CONSUMERS = ("check-code-scanning-alerts", "release-gate-summary")


def _release_gate(needs_c1: str, needs_c2: str) -> str:
    return (
        "jobs:\n"
        "  grype-scan:\n"
        "    uses: $/.github/workflows/grype.yml\n"
        "  codeql:\n"
        "    runs-on: ubuntu-latest\n"
        "    steps:\n"
        "      - uses: github/codeql-action/analyze@abc\n"
        f"  {CONSUMERS[0]}:\n"
        f"    needs: {needs_c1}\n"
        "    runs-on: ubuntu-latest\n"
        "    steps: []\n"
        f"  {CONSUMERS[1]}:\n"
        f"    needs: {needs_c2}\n"
        "    runs-on: ubuntu-latest\n"
        "    steps: []\n"
    )


RELEASE_GATE_OK = _release_gate("[grype-scan, codeql]", "[grype-scan, codeql]")


@pytest.fixture(autouse=True)
def _isolate_repo_root(hook, tmp_path):
    """Point the hook's module globals at the per-test tmp tree.

    Without this, the hook module resolves REPO_ROOT from its own file
    location and the tests would silently audit the REAL repository.
    """
    hook.REPO_ROOT = tmp_path
    hook.RELEASE_GATE = tmp_path / ".github" / "workflows" / "release-gate.yml"


def _make_tree(tmp_path: Path, release_gate: str = RELEASE_GATE_OK) -> Path:
    """A minimal repo tree: a callee for grype plus the release gate.

    The grype stub carries a SARIF upload marker so the hook classifies
    the workflow-calling job as a code-scanning uploader.
    """
    wf_dir = tmp_path / ".github" / "workflows"
    wf_dir.mkdir(parents=True, exist_ok=True)
    (wf_dir / "grype.yml").write_text(
        "jobs: {}\n# github/codeql-action/upload-sarif\n", encoding="utf-8"
    )
    (wf_dir / "release-gate.yml").write_text(release_gate, encoding="utf-8")
    return tmp_path


def _run_main(hook, capsys) -> tuple[int, str]:
    code = hook.main()
    out = capsys.readouterr().out
    return code, out


# =========================================================================
# check_local_workflow_refs — the repo-wide existence pass
# =========================================================================


class TestLocalWorkflowRefs:
    def test_clean_tree_yields_no_errors(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "ci-gate.yml",
            "jobs:\n  pre-commit:\n    uses: $/.github/workflows/grype.yml\n",
        )
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert errors == []

    def test_missing_self_repo_callee_is_flagged(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "ci-gate.yml",
            "jobs:\n  pre-commit:\n"
            "    uses: $/.github/workflows/pre-commit-typo.yml\n",
        )
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert len(errors) == 1
        assert "ci-gate.yml" in errors[0]
        assert "pre-commit-typo.yml" in errors[0]

    def test_legacy_dot_slash_callee_is_also_checked(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "old.yml",
            "jobs:\n  x:\n    uses: ./.github/workflows/nope.yml\n",
        )
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert len(errors) == 1
        assert "nope.yml" in errors[0]

    def test_remote_refs_and_step_uses_are_ignored(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "mixed.yml",
            "jobs:\n"
            "  remote:\n"
            "    uses: org/repo/.github/workflows/x.yml@0123456789abcdef0123456789abcdef01234567\n"
            "  local-job:\n"
            "    runs-on: ubuntu-latest\n"
            "    steps:\n"
            "      - uses: $/./irrelevant-step-action\n"
            "      - uses: actions/checkout@v5\n",
        )
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        # The step-level "$/./..." must NOT be treated as a job-level local
        # workflow ref (job.get("uses") only reads the job mapping).
        assert errors == []

    def test_unparseable_yaml_is_reported_not_raised(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(tmp_path, "broken.yml", "jobs: [unclosed\n")
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert any("broken.yml" in e and "parse" in e for e in errors)

    def test_malformed_job_shapes_do_not_crash(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "weird.yml",
            "jobs:\n"
            "  null-job:\n"
            "  str-job: just-a-string\n"
            "  ok-job:\n"
            "    uses: $/.github/workflows/grype.yml\n",
        )
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert errors == []

    def test_yaml_extension_workflows_are_globbed(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "dots.yaml",
            "jobs:\n  x:\n    uses: $/.github/workflows/nope.yaml\n",
        )
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert len(errors) == 1
        assert "dots.yaml" in errors[0]
        assert "nope.yaml" in errors[0]

    def test_empty_uses_value_is_ignored(self, hook, tmp_path):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "empty.yml",
            'jobs:\n  x:\n    uses: ""\n',
        )
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert errors == []

    def test_release_gate_is_skipped_no_duplicate_error(self, hook, tmp_path):
        # A missing callee in release-gate.yml is reported exactly once (by
        # the release-gate pass inside main()), not twice.
        _make_tree(
            tmp_path,
            RELEASE_GATE_OK.replace(
                "$/.github/workflows/grype.yml",
                "$/.github/workflows/grype-typo.yml",
            ),
        )
        # grype-typo.yml must not exist
        assert not (
            tmp_path / ".github" / "workflows" / "grype-typo.yml"
        ).exists()
        errors: list[str] = []
        hook.check_local_workflow_refs(errors)
        assert errors == []  # repo-wide pass skips release-gate.yml


# =========================================================================
# main() — both tripwires together
# =========================================================================


class TestMain:
    def test_clean_tree_passes(self, hook, tmp_path, capsys):
        _make_tree(tmp_path)
        code, _ = _run_main(hook, capsys)
        assert code == 0

    def test_missing_callee_fails_with_pointer(self, hook, tmp_path, capsys):
        _make_tree(
            tmp_path,
            RELEASE_GATE_OK.replace(
                "$/.github/workflows/grype.yml",
                "$/.github/workflows/grype-typo.yml",
            ),
        )
        code, out = _run_main(hook, capsys)
        assert code == 1
        assert "grype-typo.yml" in out
        # Reported once, by the release-gate pass (repo-wide pass skips
        # release-gate.yml — see TestLocalWorkflowRefs above).
        assert out.count("grype-typo.yml") == 1

    def test_repo_wide_missing_callee_fails(self, hook, tmp_path, capsys):
        _make_tree(tmp_path)
        _write_workflow(
            tmp_path,
            "ci-gate.yml",
            "jobs:\n  pre-commit:\n"
            "    uses: $/.github/workflows/pre-commit-typo.yml\n",
        )
        code, out = _run_main(hook, capsys)
        assert code == 1
        assert "pre-commit-typo.yml" in out
        assert "FIX" in out

    def test_needs_list_drift_still_detected(self, hook, tmp_path, capsys):
        # The original tripwire: grype-scan removed from ONE needs list
        # (the first consumer) while the second keeps it.
        gate = _release_gate("[codeql]", "[grype-scan, codeql]")
        _make_tree(tmp_path, gate)
        code, out = _run_main(hook, capsys)
        assert code == 1
        assert "grype-scan" in out
        assert CONSUMERS[0] in out

    def test_no_uploader_jobs_at_all_fails_loudly(self, hook, tmp_path, capsys):
        # If the detector stops finding uploaders (e.g. a marker rename),
        # fail closed instead of vacuously passing.
        _make_tree(
            tmp_path,
            "jobs:\n  nothing:\n    runs-on: ubuntu-latest\n    steps: []\n",
        )
        code, out = _run_main(hook, capsys)
        assert code == 1
        assert "No SARIF-uploading jobs" in out

    def test_missing_release_gate_clean_error_not_traceback(
        self, hook, tmp_path, capsys
    ):
        # No release-gate.yml at all: the OSError path prints a clean
        # message and exits 1 (main returns int; no exception escapes).
        wf_dir = tmp_path / ".github" / "workflows"
        wf_dir.mkdir(parents=True, exist_ok=True)
        (wf_dir / "grype.yml").write_text(
            "jobs: {}\n# github/codeql-action/upload-sarif\n", encoding="utf-8"
        )
        code, out = _run_main(hook, capsys)
        assert code == 1
        assert "Could not read or parse" in out

    def test_non_dict_jobs_in_release_gate_clean_error(
        self, hook, tmp_path, capsys
    ):
        # `jobs:` as a list is invalid for this hook's purposes; it must
        # fail with a pointer, not an AttributeError traceback (#round-4
        # review finding).
        _make_tree(tmp_path, "jobs: [not, a, mapping]\n")
        code, out = _run_main(hook, capsys)
        assert code == 1
        assert "'jobs' is not a mapping" in out

    def test_consumer_names_match_hook(self, hook):
        # The fixtures above hardcode the consumer job names; if the hook's
        # CONSUMER_JOBS ever changes, these tests must follow.
        assert CONSUMERS == hook.CONSUMER_JOBS