Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

ci: skip heavy jobs for docs-only pull requests

Christopher Kevincecf0cc6e48e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 620c83cb..ae8cb7a2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,40 +10,101 @@ permissions:
contents: read
jobs:
- test:
- name: Tests (Python ${{ matrix.python-version }})
- strategy:
- fail-fast: false
- matrix:
- python-version: ["3.12", "3.13"]
+ classify-changes:
+ name: Classify changes
+ if: ${{ github.event_name == 'pull_request' }}
runs-on: ubuntu-latest
+ outputs:
+ docs_only: ${{ steps.changes.outputs.docs_only }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
- python-version: ${{ matrix.python-version }}
- - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+ fetch-depth: 0
+ - name: Classify pull-request paths
+ id: changes
+ env:
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: python scripts/classify_ci_changes.py --base "$BASE_SHA" --head "$HEAD_SHA"
+
+ test-python-312:
+ name: Tests (Python 3.12)
+ needs: classify-changes
+ if: ${{ always() }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ - name: Set up Python
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+ uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+ with:
+ python-version: "3.12"
+ - name: Set up uv
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+ uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
with:
version: "0.11.27"
- - run: uv sync --all-extras --locked --python "${{ matrix.python-version }}"
+ - name: Install dependencies
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+ run: uv sync --all-extras --locked --python 3.12
- name: Scan OSS source boundary
- if: matrix.python-version == '3.12'
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
run: python scripts/check_oss_boundary.py --root . --allowlist config/oss_boundary_allowlist.json
- - if: matrix.python-version == '3.12'
+ - name: Lint
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
run: uv run ruff check .
- name: Run tests with coverage
- if: matrix.python-version == '3.12'
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
run: >-
uv run pytest -q
--cov=skillevaluator
--cov-report=term-missing
--cov-report=xml:coverage.xml
+ - name: Set up Node.js for docs
+ if: ${{ needs.classify-changes.outputs.docs_only == 'true' }}
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version: "22"
+ - name: Validate Fern documentation
+ if: ${{ needs.classify-changes.outputs.docs_only == 'true' }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ FERN_VERSION="$(node -p "require('./fern/fern.config.json').version")"
+ if [[ ! "$FERN_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ echo "::error file=fern/fern.config.json::Invalid Fern CLI version: $FERN_VERSION"
+ exit 1
+ fi
+ npm install --global "fern-api@$FERN_VERSION"
+ fern check
+ {
+ echo "### Documentation-only CI"
+ echo
+ echo "Validated docs and Fern configuration with Fern CLI $FERN_VERSION."
+ echo "The platform test matrix was skipped because only docs/** and fern/** changed."
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ test-python-313:
+ name: Tests (Python 3.13)
+ needs: classify-changes
+ if: ${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+ with:
+ python-version: "3.13"
+ - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+ with:
+ version: "0.11.27"
+ - run: uv sync --all-extras --locked --python 3.13
- name: Run tests
- if: matrix.python-version != '3.12'
run: uv run pytest -q
rhel8-security-install:
name: RHEL 8 security install
+ needs: classify-changes
+ if: ${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}
runs-on: ubuntu-latest
container: rockylinux/rockylinux:8.10
steps:
@@ -78,6 +139,8 @@ jobs:
package:
name: Package
+ needs: classify-changes
+ if: ${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -115,6 +178,8 @@ jobs:
tier3-macos:
name: Tier 3 macOS contract and progress
+ needs: classify-changes
+ if: ${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}
runs-on: macos-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -134,13 +199,34 @@ jobs:
tests/test_tier3_result_display.py
tests/test_harbor_runtime_preflight.py
- tier2-cross-platform:
- name: Tier 2 (${{ matrix.os }})
- strategy:
- fail-fast: false
- matrix:
- os: [macos-latest, windows-latest]
- runs-on: ${{ matrix.os }}
+ tier2-macos:
+ name: Tier 2 (macos-latest)
+ needs: classify-changes
+ if: ${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}
+ runs-on: macos-latest
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+ with:
+ python-version: "3.12"
+ - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+ with:
+ version: "0.11.27"
+ - run: uv sync --all-extras --locked
+ - run: >-
+ uv run pytest -q
+ tests/embedding
+ tests/deduplication
+ tests/validators/test_similarity.py
+ tests/reporting
+ tests/test_cli.py
+ tests/test_oss_packaging.py
+
+ tier2-windows:
+ name: Tier 2 (windows-latest)
+ needs: classify-changes
+ if: ${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}
+ runs-on: windows-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
@@ -161,6 +247,8 @@ jobs:
native-windows-local-mode:
name: Native Windows local mode fails closed
+ needs: classify-changes
+ if: ${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}
runs-on: windows-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
index b392bedd..46647873 100644
--- a/.github/workflows/security.yml
+++ b/.github/workflows/security.yml
@@ -15,6 +15,23 @@ permissions:
# Private/internal repositories can opt in after GitHub Advanced Security is
# enabled by setting ENABLE_GITHUB_ADVANCED_SECURITY=true.
jobs:
+ classify-changes:
+ name: Classify changes
+ if: ${{ github.event_name == 'pull_request' }}
+ runs-on: ubuntu-latest
+ outputs:
+ docs_only: ${{ steps.changes.outputs.docs_only }}
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ fetch-depth: 0
+ - name: Classify pull-request paths
+ id: changes
+ env:
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: python scripts/classify_ci_changes.py --base "$BASE_SHA" --head "$HEAD_SHA"
+
gitleaks:
name: Gitleaks
runs-on: ubuntu-latest
@@ -35,8 +52,11 @@ jobs:
dependency-review:
name: Dependency review
+ needs: classify-changes
if: >-
- ${{ github.event_name == 'pull_request' &&
+ ${{ always() &&
+ needs.classify-changes.outputs.docs_only != 'true' &&
+ github.event_name == 'pull_request' &&
(github.event.repository.private == false || vars.ENABLE_GITHUB_ADVANCED_SECURITY == 'true') }}
runs-on: ubuntu-latest
steps:
@@ -45,7 +65,11 @@ jobs:
codeql:
name: CodeQL
- if: ${{ github.event.repository.private == false || vars.ENABLE_GITHUB_ADVANCED_SECURITY == 'true' }}
+ needs: classify-changes
+ if: >-
+ ${{ always() &&
+ needs.classify-changes.outputs.docs_only != 'true' &&
+ (github.event.repository.private == false || vars.ENABLE_GITHUB_ADVANCED_SECURITY == 'true') }}
runs-on: ubuntu-latest
permissions:
actions: read
diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py
new file mode 100644
index 00000000..7a222f1f
--- /dev/null
+++ b/tests/test_ci_workflows.py
@@ -0,0 +1,153 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOWS = ROOT / ".github" / "workflows"
+
+REQUIRED_CI_JOBS = {
+ "test-python-312": "Tests (Python 3.12)",
+ "test-python-313": "Tests (Python 3.13)",
+ "package": "Package",
+ "rhel8-security-install": "RHEL 8 security install",
+ "tier2-macos": "Tier 2 (macos-latest)",
+ "tier2-windows": "Tier 2 (windows-latest)",
+ "tier3-macos": "Tier 3 macOS contract and progress",
+ "native-windows-local-mode": "Native Windows local mode fails closed",
+}
+HEAVY_CI_JOBS = set(REQUIRED_CI_JOBS) - {"test-python-312"}
+FULL_LANE_IF = "${{ always() && needs.classify-changes.outputs.docs_only != 'true' }}"
+DOCS_ONLY_IF = "${{ needs.classify-changes.outputs.docs_only == 'true' }}"
+NOT_DOCS_ONLY_IF = "${{ needs.classify-changes.outputs.docs_only != 'true' }}"
+
+
+def _load(name: str) -> dict[str, Any]:
+ return yaml.load((WORKFLOWS / name).read_text(encoding="utf-8"), Loader=yaml.BaseLoader)
+
+
+def _assert_no_path_filter(workflow: dict[str, Any], event: str = "pull_request") -> None:
+ trigger = workflow["on"][event]
+ if isinstance(trigger, dict):
+ assert "paths" not in trigger
+ assert "paths-ignore" not in trigger
+
+
+def _runs(job: dict[str, Any]) -> str:
+ return "\n".join(step.get("run", "") for step in job["steps"])
+
+
+def test_ci_preserves_required_contexts_as_explicit_jobs() -> None:
+ ci = _load("ci.yml")
+
+ assert {job_id: ci["jobs"][job_id]["name"] for job_id in REQUIRED_CI_JOBS} == REQUIRED_CI_JOBS
+ assert all("matrix." not in ci["jobs"][job_id]["name"] for job_id in REQUIRED_CI_JOBS)
+ _assert_no_path_filter(ci)
+
+
+def test_ci_classifier_is_pull_request_only_and_exports_docs_only() -> None:
+ classifier = _load("ci.yml")["jobs"]["classify-changes"]
+
+ assert classifier["name"] == "Classify changes"
+ assert classifier["if"] == "${{ github.event_name == 'pull_request' }}"
+ assert classifier["outputs"]["docs_only"] == "${{ steps.changes.outputs.docs_only }}"
+ assert classifier["steps"][0]["with"]["fetch-depth"] == "0"
+ assert classifier["steps"][1]["id"] == "changes"
+ assert "scripts/classify_ci_changes.py" in classifier["steps"][1]["run"]
+ assert classifier["steps"][1]["env"] == {
+ "BASE_SHA": "${{ github.event.pull_request.base.sha }}",
+ "HEAD_SHA": "${{ github.event.pull_request.head.sha }}",
+ }
+
+
+def test_ci_docs_lane_uses_the_required_python_312_context() -> None:
+ job = _load("ci.yml")["jobs"]["test-python-312"]
+
+ assert job["needs"] == "classify-changes"
+ assert job["if"] == "${{ always() }}"
+ assert job["runs-on"] == "ubuntu-latest"
+ assert all(
+ step.get("if") == NOT_DOCS_ONLY_IF
+ for step in job["steps"]
+ if step.get("name")
+ in {
+ "Set up Python",
+ "Set up uv",
+ "Install dependencies",
+ "Scan OSS source boundary",
+ "Lint",
+ "Run tests with coverage",
+ }
+ )
+
+ node_step = next(step for step in job["steps"] if step.get("name") == "Set up Node.js for docs")
+ docs_step = next(step for step in job["steps"] if step.get("name") == "Validate Fern documentation")
+ assert node_step["if"] == DOCS_ONLY_IF
+ assert len(node_step["uses"].split("@", 1)[1]) == 40
+ assert docs_step["if"] == DOCS_ONLY_IF
+ assert "fern/fern.config.json" in docs_step["run"]
+ assert 'npm install --global "fern-api@$FERN_VERSION"' in docs_step["run"]
+ assert "fern check" in docs_step["run"]
+ assert "GITHUB_STEP_SUMMARY" in docs_step["run"]
+
+
+def test_ci_skips_every_other_required_job_only_after_classification() -> None:
+ jobs = _load("ci.yml")["jobs"]
+
+ for job_id in HEAVY_CI_JOBS:
+ assert jobs[job_id]["needs"] == "classify-changes"
+ assert jobs[job_id]["if"] == FULL_LANE_IF
+
+
+def test_full_lane_keeps_the_existing_commands_and_runners() -> None:
+ jobs = _load("ci.yml")["jobs"]
+
+ assert jobs["test-python-313"]["runs-on"] == "ubuntu-latest"
+ assert "uv run pytest -q" in _runs(jobs["test-python-313"])
+ assert jobs["tier2-macos"]["runs-on"] == "macos-latest"
+ assert jobs["tier2-windows"]["runs-on"] == "windows-latest"
+ assert "tests/embedding" in _runs(jobs["tier2-macos"])
+ assert "tests/embedding" in _runs(jobs["tier2-windows"])
+ assert jobs["tier3-macos"]["runs-on"] == "macos-latest"
+ assert "tests/test_tier3_progress.py" in _runs(jobs["tier3-macos"])
+ assert jobs["native-windows-local-mode"]["runs-on"] == "windows-latest"
+ assert "tests/test_harbor_local_mode.py" in _runs(jobs["native-windows-local-mode"])
+ assert jobs["rhel8-security-install"]["container"] == "rockylinux/rockylinux:8.10"
+ assert "uv build --wheel" in _runs(jobs["rhel8-security-install"])
+ assert "twine==6.2.0" in _runs(jobs["package"])
+
+
+def test_security_keeps_gitleaks_always_on_and_skips_only_nonessential_jobs() -> None:
+ security = _load("security.yml")
+ jobs = security["jobs"]
+
+ _assert_no_path_filter(security)
+ assert "if" not in jobs["gitleaks"]
+ assert "needs" not in jobs["gitleaks"]
+ assert jobs["classify-changes"]["if"] == "${{ github.event_name == 'pull_request' }}"
+ assert jobs["classify-changes"]["outputs"]["docs_only"] == "${{ steps.changes.outputs.docs_only }}"
+
+ dependency_if = " ".join(jobs["dependency-review"]["if"].split())
+ codeql_if = " ".join(jobs["codeql"]["if"].split())
+ for job_id in ("dependency-review", "codeql"):
+ assert jobs[job_id]["needs"] == "classify-changes"
+ assert "always()" in jobs[job_id]["if"]
+ assert "needs.classify-changes.outputs.docs_only != 'true'" in jobs[job_id]["if"]
+ assert "github.event_name == 'pull_request'" in dependency_if
+ assert "github.event.repository.private == false" in dependency_if
+ assert "vars.ENABLE_GITHUB_ADVANCED_SECURITY == 'true'" in dependency_if
+ assert "github.event.repository.private == false" in codeql_if
+ assert "vars.ENABLE_GITHUB_ADVANCED_SECURITY == 'true'" in codeql_if
+
+
+def test_dco_stays_unconditional_and_has_no_path_filter() -> None:
+ dco = _load("dco.yml")
+
+ _assert_no_path_filter(dco)
+ assert "if" not in dco["jobs"]["dco"]
+ assert "needs" not in dco["jobs"]["dco"]