~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Contracts for repository labels and label-driven workflows.
These tests keep the declarative catalog, PR triage, AI-review policy, and
label-triggered checks from drifting apart. They intentionally require no
GitHub API access.
"""
import json
import re
from itertools import chain
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[2]
LABELS_PATH = ROOT / ".github" / "labels.yml"
POLICY_PATH = ROOT / ".github" / "ai-review-label-policy.json"
RELEASE_PATH = ROOT / ".github" / "release.yml"
TRIAGE_PATH = ROOT / ".github" / "workflows" / "pr-triage.yml"
AI_REVIEW_PATH = ROOT / ".github" / "workflows" / "ai-code-reviewer.yml"
def _catalog():
entries = yaml.safe_load(LABELS_PATH.read_text())
return {entry["name"]: entry for entry in entries}, entries
def _policy():
return json.loads(POLICY_PATH.read_text())
def _sections():
"""Map each labels.yml section header to the label names declared under it.
Sections are the `#` comment blocks in the file; a blank line ends one. The
grouping is what lets the tests below assert a *property* of a section
("nothing a human or a workflow owns is AI-selectable") instead of freezing
a list of names that silently goes stale when a label is added.
"""
sections = {}
header = None
pending = []
for line in LABELS_PATH.read_text().splitlines():
stripped = line.strip()
if stripped.startswith("#"):
pending.append(stripped.lstrip("#").strip())
elif not stripped:
pending = []
elif stripped.startswith("- name:"):
if pending:
header = " ".join(pending)
pending = []
name = yaml.safe_load(stripped[len("- name:") :].strip())
sections.setdefault(header, []).append(name)
return sections
def _labels_in_sections(*keys):
sections = _sections()
found = set()
for key in keys:
matching = [
names
for header, names in sections.items()
if header and key in header
]
assert matching, (
f"labels.yml has no section whose header mentions {key!r}"
)
for names in matching:
found.update(names)
return found
def _label_trigger_set():
"""Labels that any workflow uses to decide whether to run."""
referenced = set()
for path in (ROOT / ".github" / "workflows").glob("*.yml"):
text = path.read_text()
referenced.update(
re.findall(r"github\.event\.label\.name\s*==\s*'([^']+)'", text)
)
referenced.update(
re.findall(
r"contains\(github\.event\.pull_request\.labels\.\*\.name,\s*'([^']+)'\)",
text,
)
)
return referenced
def test_label_catalog_is_valid_and_has_no_placeholder_descriptions():
catalog, entries = _catalog()
assert len(catalog) == len(entries), "label names must be unique"
for name, entry in catalog.items():
assert re.fullmatch(r"[0-9a-fA-F]{6}", entry["color"]), name
assert entry["description"].strip(), name
assert len(entry["description"]) <= 100, name
assert all(ord(char) <= 0xFFFF for char in entry["description"]), (
f"{name}: the pinned label synchronizer rejects non-BMP descriptions"
)
assert entry["description"] != "Auto-created by AI reviewer", name
def test_every_pr_triage_label_is_declared():
catalog, _ = _catalog()
workflow = TRIAGE_PATH.read_text()
topic_labels = set(re.findall(r",\s*'([^']+)'\],", workflow))
state_labels = set(
re.findall(
r"(?:labels\.push|removeLabel|current\.includes)\('([^']+)'\)",
workflow,
)
)
state_labels.update(re.findall(r"addLabels\(\['([^']+)'\]\)", workflow))
referenced = topic_labels | state_labels
assert referenced
assert referenced <= catalog.keys(), referenced - catalog.keys()
def test_ai_label_policy_only_uses_declared_labels():
catalog, _ = _catalog()
policy = _policy()
apply = set(policy["apply"])
recommend = set(policy["recommend"])
aliases = policy["aliases"]
referenced = apply | recommend | set(aliases) | set(aliases.values())
assert referenced <= catalog.keys(), referenced - catalog.keys()
assert not (apply & recommend)
def test_every_catalog_label_belongs_to_exactly_one_section():
# Safety net for _sections()'s line-prefix scan: it keys off the literal
# `- name:` prefix, so a label whose YAML fields are reordered (a
# perfectly valid `- color: ...` / `name: ...` block) or otherwise fails
# to match is silently dropped from every section instead of raising.
# Any test built on top of _sections()/_labels_in_sections() (below) would
# then just see one fewer label without noticing. Asserting totality here
# means that drop always fails loudly, in one place, by name.
catalog, _ = _catalog()
assert set(chain(*_sections().values())) == catalog.keys()
def test_ai_label_policy_excludes_human_and_workflow_control_state():
# Inverted derivation: forbidden is *everything in the catalog that is
# not* in a section the AI reviewer is explicitly allowed to select from,
# rather than an enumeration of the sections that are off-limits. That
# makes "forbidden" the safe default — a label under a new or renamed
# section (a header this test has never heard of) lands in forbidden
# until someone deliberately adds its section to the allowlist below,
# instead of silently escaping the check the way an enumerated forbidden
# list would (a stale name — there used to be an undeclared
# `code-ready-preliminary` here — could also sit in that list doing
# nothing).
catalog, _ = _catalog()
policy = _policy()
selectable = set(policy["apply"]) | set(policy["recommend"])
ai_selectable = _labels_in_sections(
"PR topic labels",
"Semantic PR labels the AI reviewer may apply",
"AI-reviewer recommendations for opt-in",
)
forbidden = catalog.keys() - ai_selectable
# Guard against the derivation silently collapsing to a near-empty set.
assert len(forbidden) >= 17, sorted(forbidden)
assert {"code-ready", "needs-rework", "ai_code_review"} <= forbidden
assert selectable.isdisjoint(forbidden), selectable & forbidden
def test_ai_reviewer_never_creates_model_suggested_labels():
workflow = AI_REVIEW_PATH.read_text()
assert "gh label create" not in workflow
assert "Auto-created by AI reviewer" not in workflow
assert ".github/ai-review-label-policy.json" in workflow
def test_pull_request_target_reviewer_keeps_the_trusted_checkout_invariant():
# Structural, not textual, but narrowly scoped: this covers actions/
# checkout's `with.ref` and `with.persist-credentials` only. A textual
# search for one spelling of the head ref missed the others
# (`github.head_ref`, `refs/pull/N/merge`, a pre-fetched `origin/pr-head`,
# a quoted expression); asserting that no `ref:` is pinned at all catches
# every spelling of *that shape* — an actions/checkout step pinning a ref —
# including ones nobody has thought of yet. A deliberate literal base ref
# would be safe but still has to come through here. It does NOT catch a
# `run:` step doing its own `git checkout` of the fetched PR-head ref (the
# second assertion below covers exactly that one shape) or a `uses:`-only
# job with no `steps:` at all.
workflow = yaml.safe_load(AI_REVIEW_PATH.read_text())
# PyYAML resolves the bare `on:` key to the boolean True.
triggers = workflow.get("on", workflow.get(True))
assert "pull_request_target" in triggers
checkouts = [
step
for job in workflow["jobs"].values()
for step in job.get("steps", [])
if str(step.get("uses", "")).startswith("actions/checkout@")
]
assert checkouts, "the reviewer must check out the trusted base branch"
for step in checkouts:
options = step.get("with") or {}
assert "ref" not in options, (
f"checkout step {step.get('name')!r} pins ref: {options.get('ref')!r}; "
"pull_request_target must keep its default (base branch) checkout "
"so fork-controlled files are never executed with secrets in scope"
)
assert options.get("persist-credentials") is False, step.get("name")
# The PR head is legitimately fetched (as data, via `git fetch ...
# refs/pull/$PR_NUMBER/head`) for the diff; it must never also be checked
# out. Revert: replace the `git fetch` + `git diff origin/$BASE_REF...
# origin/pr-head` pair with `git checkout origin/pr-head` (or any `git
# checkout` of a `refs/pull/...` ref) in a `run:` step and this fails.
reviewer_job = workflow["jobs"]["comprehensive-review"]
for step in reviewer_job.get("steps", []):
run = step.get("run")
if not isinstance(run, str):
continue
for line in run.splitlines():
assert not (
"git checkout" in line
and ("origin/pr-head" in line or "refs/pull/" in line)
), (
f"step {step.get('name')!r} run line checks out a PR head ref: {line!r}"
)
def test_label_triggered_workflows_only_reference_declared_labels():
catalog, _ = _catalog()
referenced = _label_trigger_set()
assert referenced <= catalog.keys(), referenced - catalog.keys()
assert "test:notes" in referenced
assert "test:ui" not in referenced
def test_no_workflow_trigger_label_is_ai_applicable():
"""The central invariant of the AI label policy.
Every label that decides whether some workflow runs must stay out of
`apply` — that array is what the reviewer writes straight onto the PR with
`gh issue edit --add-label`, from output steered by an untrusted diff.
`recommend` is the correct home for them: it only prints a suggestion.
Move `test:notes` from `recommend` to `apply` and this test fails, where
every other contract here still passes.
"""
policy = _policy()
triggers = _label_trigger_set()
assert triggers, "no label-triggered workflows found — the sweep broke"
assert triggers.isdisjoint(policy["apply"]), triggers & set(policy["apply"])
def test_rejected_label_suggestions_reach_the_run_summary():
# Structural: pulled from this one step's parsed `run:` string, not a
# `workflow.index()` window over the whole file — that whole-file version
# both fails spuriously if any *earlier* step also writes to
# $GITHUB_STEP_SUMMARY and would pass just as well if the block moved
# into an unrelated step. The helper writes rejected_labels.json, but only
# a count ever reached stderr, so a maintainer could not see *which*
# suggestions were dropped without digging through the job log. Delete
# the summary block, or wrap it in a constant-false guard
# (`if false; then ... fi`) so it can never run, and this fails.
workflow = yaml.safe_load(AI_REVIEW_PATH.read_text())
job = workflow["jobs"]["comprehensive-review"]
step = next(step for step in job["steps"] if step.get("id") == "ai-review")
run = step.get("run", "")
assert "GITHUB_STEP_SUMMARY" in run, (
"the reviewer job must write the dropped suggestions somewhere a "
"maintainer sees without opening the job log"
)
assert "rejected_labels.json" in run
assert "Suggested but not applied" in run
assert "if false" not in run, (
"the summary block must not be dead-guarded behind a constant-false "
"condition"
)
def test_release_categories_only_reference_declared_labels():
catalog, _ = _catalog()
release_config = yaml.safe_load(RELEASE_PATH.read_text())
referenced = {
label
for category in release_config["changelog"]["categories"]
for label in category["labels"]
if label != "*"
}
assert referenced <= catalog.keys(), referenced - catalog.keys()
def test_ci_recommendation_descriptions_explain_the_human_handoff():
catalog, _ = _catalog()
policy = _policy()
for name in policy["recommend"]:
description = catalog[name]["description"]
assert "AI reviewer may recommend" in description, name
assert "apply or re-add" in description, name
def test_legacy_aliases_point_to_canonical_labels():
policy = _policy()
assert policy["aliases"]["bug"] == "bugfix"
assert policy["aliases"]["testing"] == "tests"
assert policy["aliases"]["test:e2e"] == "test:puppeteer"
def test_requested_changes_option_is_visible_to_the_failure_step():
workflow = yaml.safe_load(AI_REVIEW_PATH.read_text())
job = workflow["jobs"]["comprehensive-review"]
step = next(
step
for step in job["steps"]
if step.get("name") == "Fail Workflow if Requested"
)
condition = step["if"]
option = "FAIL_ON_REQUESTED_CHANGES"
# A flag declared on a sibling step is not visible here. Both a direct
# repository variable and a workflow/job/this-step env binding are valid.
references = re.findall(rf"\b(env|vars)\.{option}\b", condition)
assert references, "failure must remain an explicit configured opt-in"
available_env = {
**workflow.get("env", {}),
**job.get("env", {}),
**step.get("env", {}),
}
if "env" in references:
assert option in available_env, "failure flag is scoped to another step"
assert "steps.ai-review.outputs.DECISION" in condition