Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

13.4 KB
Raw Blame
"""Contracts for repository labels and label-driven workflows.

These tests keep the declarative catalog, PR triage, AI-review policy, and
label-triggered checks from drifting apart. They intentionally require no
GitHub API access.
"""

import json
import re
from itertools import chain
from pathlib import Path

import yaml


ROOT = Path(__file__).resolve().parents[2]
LABELS_PATH = ROOT / ".github" / "labels.yml"
POLICY_PATH = ROOT / ".github" / "ai-review-label-policy.json"
RELEASE_PATH = ROOT / ".github" / "release.yml"
TRIAGE_PATH = ROOT / ".github" / "workflows" / "pr-triage.yml"
AI_REVIEW_PATH = ROOT / ".github" / "workflows" / "ai-code-reviewer.yml"


def _catalog():
    entries = yaml.safe_load(LABELS_PATH.read_text())
    return {entry["name"]: entry for entry in entries}, entries


def _policy():
    return json.loads(POLICY_PATH.read_text())


def _sections():
    """Map each labels.yml section header to the label names declared under it.

    Sections are the `#` comment blocks in the file; a blank line ends one. The
    grouping is what lets the tests below assert a *property* of a section
    ("nothing a human or a workflow owns is AI-selectable") instead of freezing
    a list of names that silently goes stale when a label is added.
    """
    sections = {}
    header = None
    pending = []
    for line in LABELS_PATH.read_text().splitlines():
        stripped = line.strip()
        if stripped.startswith("#"):
            pending.append(stripped.lstrip("#").strip())
        elif not stripped:
            pending = []
        elif stripped.startswith("- name:"):
            if pending:
                header = " ".join(pending)
                pending = []
            name = yaml.safe_load(stripped[len("- name:") :].strip())
            sections.setdefault(header, []).append(name)
    return sections


def _labels_in_sections(*keys):
    sections = _sections()
    found = set()
    for key in keys:
        matching = [
            names
            for header, names in sections.items()
            if header and key in header
        ]
        assert matching, (
            f"labels.yml has no section whose header mentions {key!r}"
        )
        for names in matching:
            found.update(names)
    return found


def _label_trigger_set():
    """Labels that any workflow uses to decide whether to run."""
    referenced = set()
    for path in (ROOT / ".github" / "workflows").glob("*.yml"):
        text = path.read_text()
        referenced.update(
            re.findall(r"github\.event\.label\.name\s*==\s*'([^']+)'", text)
        )
        referenced.update(
            re.findall(
                r"contains\(github\.event\.pull_request\.labels\.\*\.name,\s*'([^']+)'\)",
                text,
            )
        )
    return referenced


def test_label_catalog_is_valid_and_has_no_placeholder_descriptions():
    catalog, entries = _catalog()

    assert len(catalog) == len(entries), "label names must be unique"
    for name, entry in catalog.items():
        assert re.fullmatch(r"[0-9a-fA-F]{6}", entry["color"]), name
        assert entry["description"].strip(), name
        assert len(entry["description"]) <= 100, name
        assert all(ord(char) <= 0xFFFF for char in entry["description"]), (
            f"{name}: the pinned label synchronizer rejects non-BMP descriptions"
        )
        assert entry["description"] != "Auto-created by AI reviewer", name


def test_every_pr_triage_label_is_declared():
    catalog, _ = _catalog()
    workflow = TRIAGE_PATH.read_text()

    topic_labels = set(re.findall(r",\s*'([^']+)'\],", workflow))
    state_labels = set(
        re.findall(
            r"(?:labels\.push|removeLabel|current\.includes)\('([^']+)'\)",
            workflow,
        )
    )
    state_labels.update(re.findall(r"addLabels\(\['([^']+)'\]\)", workflow))
    referenced = topic_labels | state_labels

    assert referenced
    assert referenced <= catalog.keys(), referenced - catalog.keys()


def test_ai_label_policy_only_uses_declared_labels():
    catalog, _ = _catalog()
    policy = _policy()
    apply = set(policy["apply"])
    recommend = set(policy["recommend"])
    aliases = policy["aliases"]
    referenced = apply | recommend | set(aliases) | set(aliases.values())

    assert referenced <= catalog.keys(), referenced - catalog.keys()
    assert not (apply & recommend)


def test_every_catalog_label_belongs_to_exactly_one_section():
    # Safety net for _sections()'s line-prefix scan: it keys off the literal
    # `- name:` prefix, so a label whose YAML fields are reordered (a
    # perfectly valid `- color: ...` / `name: ...` block) or otherwise fails
    # to match is silently dropped from every section instead of raising.
    # Any test built on top of _sections()/_labels_in_sections() (below) would
    # then just see one fewer label without noticing. Asserting totality here
    # means that drop always fails loudly, in one place, by name.
    catalog, _ = _catalog()
    assert set(chain(*_sections().values())) == catalog.keys()


def test_ai_label_policy_excludes_human_and_workflow_control_state():
    # Inverted derivation: forbidden is *everything in the catalog that is
    # not* in a section the AI reviewer is explicitly allowed to select from,
    # rather than an enumeration of the sections that are off-limits. That
    # makes "forbidden" the safe default — a label under a new or renamed
    # section (a header this test has never heard of) lands in forbidden
    # until someone deliberately adds its section to the allowlist below,
    # instead of silently escaping the check the way an enumerated forbidden
    # list would (a stale name — there used to be an undeclared
    # `code-ready-preliminary` here — could also sit in that list doing
    # nothing).
    catalog, _ = _catalog()
    policy = _policy()
    selectable = set(policy["apply"]) | set(policy["recommend"])
    ai_selectable = _labels_in_sections(
        "PR topic labels",
        "Semantic PR labels the AI reviewer may apply",
        "AI-reviewer recommendations for opt-in",
    )
    forbidden = catalog.keys() - ai_selectable

    # Guard against the derivation silently collapsing to a near-empty set.
    assert len(forbidden) >= 17, sorted(forbidden)
    assert {"code-ready", "needs-rework", "ai_code_review"} <= forbidden
    assert selectable.isdisjoint(forbidden), selectable & forbidden


def test_ai_reviewer_never_creates_model_suggested_labels():
    workflow = AI_REVIEW_PATH.read_text()

    assert "gh label create" not in workflow
    assert "Auto-created by AI reviewer" not in workflow
    assert ".github/ai-review-label-policy.json" in workflow


def test_pull_request_target_reviewer_keeps_the_trusted_checkout_invariant():
    # Structural, not textual, but narrowly scoped: this covers actions/
    # checkout's `with.ref` and `with.persist-credentials` only. A textual
    # search for one spelling of the head ref missed the others
    # (`github.head_ref`, `refs/pull/N/merge`, a pre-fetched `origin/pr-head`,
    # a quoted expression); asserting that no `ref:` is pinned at all catches
    # every spelling of *that shape* — an actions/checkout step pinning a ref —
    # including ones nobody has thought of yet. A deliberate literal base ref
    # would be safe but still has to come through here. It does NOT catch a
    # `run:` step doing its own `git checkout` of the fetched PR-head ref (the
    # second assertion below covers exactly that one shape) or a `uses:`-only
    # job with no `steps:` at all.
    workflow = yaml.safe_load(AI_REVIEW_PATH.read_text())
    # PyYAML resolves the bare `on:` key to the boolean True.
    triggers = workflow.get("on", workflow.get(True))
    assert "pull_request_target" in triggers

    checkouts = [
        step
        for job in workflow["jobs"].values()
        for step in job.get("steps", [])
        if str(step.get("uses", "")).startswith("actions/checkout@")
    ]

    assert checkouts, "the reviewer must check out the trusted base branch"
    for step in checkouts:
        options = step.get("with") or {}
        assert "ref" not in options, (
            f"checkout step {step.get('name')!r} pins ref: {options.get('ref')!r}; "
            "pull_request_target must keep its default (base branch) checkout "
            "so fork-controlled files are never executed with secrets in scope"
        )
        assert options.get("persist-credentials") is False, step.get("name")

    # The PR head is legitimately fetched (as data, via `git fetch ...
    # refs/pull/$PR_NUMBER/head`) for the diff; it must never also be checked
    # out. Revert: replace the `git fetch` + `git diff origin/$BASE_REF...
    # origin/pr-head` pair with `git checkout origin/pr-head` (or any `git
    # checkout` of a `refs/pull/...` ref) in a `run:` step and this fails.
    reviewer_job = workflow["jobs"]["comprehensive-review"]
    for step in reviewer_job.get("steps", []):
        run = step.get("run")
        if not isinstance(run, str):
            continue
        for line in run.splitlines():
            assert not (
                "git checkout" in line
                and ("origin/pr-head" in line or "refs/pull/" in line)
            ), (
                f"step {step.get('name')!r} run line checks out a PR head ref: {line!r}"
            )


def test_label_triggered_workflows_only_reference_declared_labels():
    catalog, _ = _catalog()
    referenced = _label_trigger_set()

    assert referenced <= catalog.keys(), referenced - catalog.keys()
    assert "test:notes" in referenced
    assert "test:ui" not in referenced


def test_no_workflow_trigger_label_is_ai_applicable():
    """The central invariant of the AI label policy.

    Every label that decides whether some workflow runs must stay out of
    `apply` — that array is what the reviewer writes straight onto the PR with
    `gh issue edit --add-label`, from output steered by an untrusted diff.
    `recommend` is the correct home for them: it only prints a suggestion.
    Move `test:notes` from `recommend` to `apply` and this test fails, where
    every other contract here still passes.
    """
    policy = _policy()
    triggers = _label_trigger_set()

    assert triggers, "no label-triggered workflows found — the sweep broke"
    assert triggers.isdisjoint(policy["apply"]), triggers & set(policy["apply"])


def test_rejected_label_suggestions_reach_the_run_summary():
    # Structural: pulled from this one step's parsed `run:` string, not a
    # `workflow.index()` window over the whole file — that whole-file version
    # both fails spuriously if any *earlier* step also writes to
    # $GITHUB_STEP_SUMMARY and would pass just as well if the block moved
    # into an unrelated step. The helper writes rejected_labels.json, but only
    # a count ever reached stderr, so a maintainer could not see *which*
    # suggestions were dropped without digging through the job log. Delete
    # the summary block, or wrap it in a constant-false guard
    # (`if false; then ... fi`) so it can never run, and this fails.
    workflow = yaml.safe_load(AI_REVIEW_PATH.read_text())
    job = workflow["jobs"]["comprehensive-review"]
    step = next(step for step in job["steps"] if step.get("id") == "ai-review")
    run = step.get("run", "")

    assert "GITHUB_STEP_SUMMARY" in run, (
        "the reviewer job must write the dropped suggestions somewhere a "
        "maintainer sees without opening the job log"
    )
    assert "rejected_labels.json" in run
    assert "Suggested but not applied" in run
    assert "if false" not in run, (
        "the summary block must not be dead-guarded behind a constant-false "
        "condition"
    )


def test_release_categories_only_reference_declared_labels():
    catalog, _ = _catalog()
    release_config = yaml.safe_load(RELEASE_PATH.read_text())
    referenced = {
        label
        for category in release_config["changelog"]["categories"]
        for label in category["labels"]
        if label != "*"
    }

    assert referenced <= catalog.keys(), referenced - catalog.keys()


def test_ci_recommendation_descriptions_explain_the_human_handoff():
    catalog, _ = _catalog()
    policy = _policy()

    for name in policy["recommend"]:
        description = catalog[name]["description"]
        assert "AI reviewer may recommend" in description, name
        assert "apply or re-add" in description, name


def test_legacy_aliases_point_to_canonical_labels():
    policy = _policy()

    assert policy["aliases"]["bug"] == "bugfix"
    assert policy["aliases"]["testing"] == "tests"
    assert policy["aliases"]["test:e2e"] == "test:puppeteer"


def test_requested_changes_option_is_visible_to_the_failure_step():
    workflow = yaml.safe_load(AI_REVIEW_PATH.read_text())
    job = workflow["jobs"]["comprehensive-review"]
    step = next(
        step
        for step in job["steps"]
        if step.get("name") == "Fail Workflow if Requested"
    )
    condition = step["if"]
    option = "FAIL_ON_REQUESTED_CHANGES"
    # A flag declared on a sibling step is not visible here. Both a direct
    # repository variable and a workflow/job/this-step env binding are valid.
    references = re.findall(rf"\b(env|vars)\.{option}\b", condition)
    assert references, "failure must remain an explicit configured opt-in"
    available_env = {
        **workflow.get("env", {}),
        **job.get("env", {}),
        **step.get("env", {}),
    }
    if "env" in references:
        assert option in available_env, "failure flag is scoped to another step"
    assert "steps.ai-review.outputs.DECISION" in condition