Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

ci: classify docs-only pull requests

Christopher Kevincf80ca68be53
diff --git a/scripts/classify_ci_changes.py b/scripts/classify_ci_changes.py
new file mode 100644
index 00000000..5141da8f
--- /dev/null
+++ b/scripts/classify_ci_changes.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Classify pull-request changes for fail-closed CI routing."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+DOC_PREFIXES = (b"docs/", b"fern/")
+KNOWN_STATUSES = frozenset(b"ACDMRTUXB")
+
+
+def is_docs_only(paths: Sequence[bytes]) -> bool:
+ """Return whether every changed path belongs to published documentation."""
+ return bool(paths) and all(path.startswith(DOC_PREFIXES) for path in paths)
+
+
+def parse_name_status_z(payload: bytes) -> list[bytes]:
+ """Parse ``git diff --name-status -z`` without losing rename sources."""
+ if not payload:
+ return []
+ if not payload.endswith(b"\0"):
+ raise ValueError("incomplete Git status record: missing NUL terminator")
+
+ fields = payload.split(b"\0")
+ fields.pop()
+ paths: list[bytes] = []
+ index = 0
+ while index < len(fields):
+ status = fields[index]
+ index += 1
+ if not status or status[0] not in KNOWN_STATUSES:
+ raise ValueError(f"unrecognized Git status record: {status!r}")
+
+ code = status[:1]
+ if code in {b"R", b"C"}:
+ if len(status) == 1 or not status[1:].isdigit():
+ raise ValueError(f"unrecognized Git status record: {status!r}")
+ path_count = 2
+ else:
+ if len(status) != 1:
+ raise ValueError(f"unrecognized Git status record: {status!r}")
+ path_count = 1
+
+ if index + path_count > len(fields):
+ raise ValueError(f"incomplete Git status record: {status!r}")
+ record_paths = fields[index : index + path_count]
+ if any(not path for path in record_paths):
+ raise ValueError(f"incomplete Git status record: {status!r}")
+ paths.extend(record_paths)
+ index += path_count
+ return paths
+
+
+def _validate_revision(value: str) -> str:
+ if re.fullmatch(r"[0-9a-fA-F]{40,64}", value) is None:
+ raise ValueError(f"invalid Git revision: {value!r}")
+ return value
+
+
+def changed_paths(repo: Path, base: str, head: str) -> list[bytes]:
+ """Return every path changed from the merge base through ``head``."""
+ base = _validate_revision(base)
+ head = _validate_revision(head)
+ result = subprocess.run(
+ [
+ "git",
+ "-C",
+ str(repo),
+ "diff",
+ "--name-status",
+ "-z",
+ "--find-renames",
+ "--find-copies",
+ f"{base}...{head}",
+ "--",
+ ],
+ check=True,
+ capture_output=True,
+ )
+ return parse_name_status_z(result.stdout)
+
+
+def _write_result(docs_only: bool) -> None:
+ line = f"docs_only={'true' if docs_only else 'false'}"
+ output_path = os.environ.get("GITHUB_OUTPUT")
+ if output_path:
+ with Path(output_path).open("a", encoding="utf-8") as output:
+ output.write(f"{line}\n")
+ print(line)
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--repo", type=Path, default=Path.cwd(), help="Git repository to inspect")
+ parser.add_argument("--base", required=True, help="pull-request base commit SHA")
+ parser.add_argument("--head", required=True, help="pull-request head commit SHA")
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ """Classify one pull-request diff and emit a GitHub Actions output."""
+ args = _parser().parse_args(argv)
+ try:
+ paths = changed_paths(args.repo, args.base, args.head)
+ if not paths:
+ raise ValueError("no changed paths found")
+ docs_only = is_docs_only(paths)
+ except (OSError, subprocess.CalledProcessError, ValueError) as error:
+ print(f"change classification failed; falling back to full CI: {error}", file=sys.stderr)
+ docs_only = False
+
+ _write_result(docs_only)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/test_ci_change_classifier.py b/tests/test_ci_change_classifier.py
new file mode 100644
index 00000000..08a7ebb7
--- /dev/null
+++ b/tests/test_ci_change_classifier.py
@@ -0,0 +1,198 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+from scripts.classify_ci_changes import is_docs_only, main, parse_name_status_z
+
+
+def _git(repo: Path, *args: str) -> str:
+ result = subprocess.run(
+ ["git", "-C", str(repo), *args],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ return result.stdout.strip()
+
+
+def _commit(repo: Path, message: str) -> str:
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "--quiet", "-m", message)
+ return _git(repo, "rev-parse", "HEAD")
+
+
+def git_repo(tmp_path: Path) -> tuple[Path, str]:
+ _git(tmp_path, "init", "--quiet")
+ _git(tmp_path, "config", "user.email", "[email protected]")
+ _git(tmp_path, "config", "user.name", "CI Test")
+ (tmp_path / "docs").mkdir()
+ (tmp_path / "docs" / "index.mdx").write_text("initial\n", encoding="utf-8")
+ (tmp_path / "README.md").write_text("initial\n", encoding="utf-8")
+ return tmp_path, _commit(tmp_path, "initial")
+
+
+ ("paths", "expected"),
+ [
+ ([b"docs/index.mdx"], True),
+ ([b"fern/docs.yml", b"docs/assets/logo.svg"], True),
+ ([b"README.md"], False),
+ ([b"src/skillevaluator/tier3/reference_skills/demo/SKILL.md"], False),
+ ([b"tests/golden/benchmark_tier1.md"], False),
+ ([b"docs/index.mdx", b"src/skillevaluator/cli.py"], False),
+ ([], False),
+ ],
+)
+def test_is_docs_only(paths: list[bytes], expected: bool) -> None:
+ assert is_docs_only(paths) is expected
+
+
+ ("payload", "expected"),
+ [
+ (b"M\0docs/index.mdx\0", [b"docs/index.mdx"]),
+ (b"D\0docs/old.mdx\0", [b"docs/old.mdx"]),
+ (
+ b"R100\0docs/old.mdx\0src/new.py\0",
+ [b"docs/old.mdx", b"src/new.py"],
+ ),
+ (
+ b"C075\0docs/source.mdx\0fern/copied.yml\0",
+ [b"docs/source.mdx", b"fern/copied.yml"],
+ ),
+ (b"", []),
+ ],
+)
+def test_parse_name_status_z_returns_every_changed_path(payload: bytes, expected: list[bytes]) -> None:
+ assert parse_name_status_z(payload) == expected
+
+
+ "payload",
+ [
+ b"R100\0docs/old.mdx\0",
+ b"M\0",
+ b"Z\0docs/index.mdx\0",
+ b"\0docs/index.mdx\0",
+ b"M\0docs/index.mdx",
+ ],
+)
+def test_parse_name_status_z_rejects_malformed_records(payload: bytes) -> None:
+ with pytest.raises(ValueError, match="Git status record"):
+ parse_name_status_z(payload)
+
+
+def _classify(
+ repo: Path,
+ base: str,
+ head: str,
+ output: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> int:
+ monkeypatch.setenv("GITHUB_OUTPUT", str(output))
+ return main(["--repo", str(repo), "--base", base, "--head", head])
+
+
+def test_main_classifies_a_real_docs_only_diff(
+ git_repo: tuple[Path, str],
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ repo, base = git_repo
+ (repo / "docs" / "index.mdx").write_text("updated\n", encoding="utf-8")
+ head = _commit(repo, "docs")
+ output = tmp_path / "github-output"
+
+ assert _classify(repo, base, head, output, monkeypatch) == 0
+
+ assert capsys.readouterr().out == "docs_only=true\n"
+ assert output.read_text(encoding="utf-8") == "docs_only=true\n"
+
+
+def test_main_classifies_a_real_mixed_diff_as_full_ci(
+ git_repo: tuple[Path, str],
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ repo, base = git_repo
+ (repo / "docs" / "index.mdx").write_text("updated\n", encoding="utf-8")
+ (repo / "source.py").write_text("print('changed')\n", encoding="utf-8")
+ head = _commit(repo, "mixed")
+ output = tmp_path / "github-output"
+
+ assert _classify(repo, base, head, output, monkeypatch) == 0
+
+ assert capsys.readouterr().out == "docs_only=false\n"
+ assert output.read_text(encoding="utf-8") == "docs_only=false\n"
+
+
+def test_main_treats_a_deleted_docs_file_as_docs_only(
+ git_repo: tuple[Path, str],
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ repo, base = git_repo
+ (repo / "docs" / "index.mdx").unlink()
+ head = _commit(repo, "delete docs")
+
+ assert _classify(repo, base, head, tmp_path / "github-output", monkeypatch) == 0
+ assert (tmp_path / "github-output").read_text(encoding="utf-8") == "docs_only=true\n"
+
+
+def test_main_checks_both_sides_of_a_rename(
+ git_repo: tuple[Path, str],
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ repo, base = git_repo
+ (repo / "src").mkdir()
+ _git(repo, "mv", "docs/index.mdx", "src/index.py")
+ head = _commit(repo, "rename out of docs")
+
+ assert _classify(repo, base, head, tmp_path / "github-output", monkeypatch) == 0
+ assert (tmp_path / "github-output").read_text(encoding="utf-8") == "docs_only=false\n"
+
+
[email protected]("revision", ["0" * 40, "not-a-sha"])
+def test_main_fails_closed_for_invalid_revisions(
+ git_repo: tuple[Path, str],
+ revision: str,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ repo, head = git_repo
+ output = tmp_path / "github-output"
+
+ assert _classify(repo, revision, head, output, monkeypatch) == 0
+
+ captured = capsys.readouterr()
+ assert captured.out == "docs_only=false\n"
+ assert "falling back to full CI" in captured.err
+ assert output.read_text(encoding="utf-8") == "docs_only=false\n"
+
+
+def test_main_fails_closed_for_an_empty_diff(
+ git_repo: tuple[Path, str],
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ repo, head = git_repo
+ output = tmp_path / "github-output"
+ output.write_text("existing=value\n", encoding="utf-8")
+
+ assert _classify(repo, head, head, output, monkeypatch) == 0
+
+ captured = capsys.readouterr()
+ assert captured.out == "docs_only=false\n"
+ assert "no changed paths" in captured.err
+ assert output.read_text(encoding="utf-8") == "existing=value\ndocs_only=false\n"