Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

ci: run the all-tier skill scan only when skills or its callers change (#441)

Christopher Kevinfff53328887a
diff --git a/.github/workflows/skill-checks-all-tiers.yml b/.github/workflows/skill-checks-all-tiers.yml
index 66c8f37c..09bc951c 100644
--- a/.github/workflows/skill-checks-all-tiers.yml
+++ b/.github/workflows/skill-checks-all-tiers.yml
@@ -1,10 +1,18 @@
# Generated by SkillEvaluator all-tier CI onboarding.
name: Skill checks (all tiers)
'on':
- pull_request: null
+ pull_request:
+ paths:
+ - skills/**
+ - .github/workflows/skill-checks-all-tiers.yml
+ - .github/workflows/skill-feedback-all-tiers.yml
push:
branches:
- main
+ paths:
+ - skills/**
+ - .github/workflows/skill-checks-all-tiers.yml
+ - .github/workflows/skill-feedback-all-tiers.yml
workflow_dispatch: null
permissions:
contents: read
diff --git a/.github/workflows/skill-feedback-all-tiers.yml b/.github/workflows/skill-feedback-all-tiers.yml
index 8656384a..63f7b3be 100644
--- a/.github/workflows/skill-feedback-all-tiers.yml
+++ b/.github/workflows/skill-feedback-all-tiers.yml
@@ -17,4 +17,4 @@ jobs:
tools-repository: NVIDIA/SkillEvaluator
tools-ref: c3b298c2b1a1a89d520bb22009f2668520127931
trusted-workflow-ref: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@c3b298c2b1a1a89d520bb22009f2668520127931
- trusted-caller-blobs: 66c8f37c7987ec680d66b8365fde8557208b5257
+ trusted-caller-blobs: 09bc951c5df415090d7e6c8159d87ab0fd1e98f8
diff --git a/docs/github-actions.mdx b/docs/github-actions.mdx
index 71a69218..cf47dd38 100644
--- a/docs/github-actions.mdx
+++ b/docs/github-actions.mdx
@@ -296,6 +296,10 @@ or let the [CLI](/skills/skillevaluator/cli-onboarding) handle it.
collection each time. No changed skills is a successful **Skipped** result,
not a scanner pass. Changes to shared files outside the skill roots are not
inferred dependencies; use `all` when they should trigger a full scan.
+ The generated all-tier caller (`skillevaluator ci init`) also carries
+ `paths` filters, so a pull request that changes nothing under the selected
+ skill folders or the two caller workflows starts no run and receives no
+ comment.
Selection is capped at 100 skills. Timeouts retain completed results and
mark unfinished work incomplete. The workflow scans up to four skills at
once; `scan-workers` accepts 1–16.
diff --git a/src/skillevaluator/_ci_onboarding_all_tiers.py b/src/skillevaluator/_ci_onboarding_all_tiers.py
index 90a51831..957d78f0 100644
--- a/src/skillevaluator/_ci_onboarding_all_tiers.py
+++ b/src/skillevaluator/_ci_onboarding_all_tiers.py
@@ -368,6 +368,16 @@ def _github_feedback_path(repo: Path) -> str:
return FEEDBACK
+def _github_trigger_paths(roots: list[str], feedback_path: str) -> list[str]:
+ """Path filters so only skill or caller changes start the all-tier scan.
+
+ A pull request outside these paths neither spends runner time nor receives a
+ feedback comment. Caller changes still run so the setup change is exercised.
+ """
+ filters = ["**" if root in {"", "."} else f"{root}/**" for root in roots]
+ return [*dict.fromkeys(filters), SCAN, feedback_path]
+
+
def render(wizard, repo, release, platform, branch, roots, options):
common = {
"tiers": "1,2,3",
@@ -392,9 +402,15 @@ def render(wizard, repo, release, platform, branch, roots, options):
header = "# Generated by SkillEvaluator all-tier CI onboarding.\n"
if platform == "github":
feedback_path = _github_feedback_path(repo)
+ triggers = _github_trigger_paths(roots, feedback_path)
scan = {
"name": "Skill checks (all tiers)",
- "on": {"pull_request": None, "push": {"branches": [branch]}, "workflow_dispatch": None},
+ "on": {
+ # Separate lists: the dumper would otherwise emit a YAML alias.
+ "pull_request": {"paths": list(triggers)},
+ "push": {"branches": [branch], "paths": list(triggers)},
+ "workflow_dispatch": None,
+ },
"permissions": {"contents": "read", "id-token": "write", "pull-requests": "read", "actions": "read"},
"jobs": {
"skills": {
diff --git a/tests/ci/test_repository_skill_ci.py b/tests/ci/test_repository_skill_ci.py
index f5fb2b22..4f89cb10 100644
--- a/tests/ci/test_repository_skill_ci.py
+++ b/tests/ci/test_repository_skill_ci.py
@@ -20,6 +20,11 @@ def test_repository_skill_checks_use_reviewed_full_three_tier_configuration():
scan = workflow(SCAN)
assert set(scan["on"]) == {"pull_request", "push", "workflow_dispatch"}
assert scan["on"]["push"]["branches"] == ["main"]
+ # Unrelated pull requests start no run and receive no feedback comment;
+ # caller changes still run so the setup change is exercised.
+ triggers = ["skills/**", SCAN, FEEDBACK]
+ assert scan["on"]["pull_request"] == {"paths": triggers}
+ assert scan["on"]["push"]["paths"] == triggers
assert scan["permissions"] == {
"contents": "read",
"actions": "read",
diff --git a/tests/test_ci_onboarding_all_tiers.py b/tests/test_ci_onboarding_all_tiers.py
index df4c19df..01e41159 100644
--- a/tests/test_ci_onboarding_all_tiers.py
+++ b/tests/test_ci_onboarding_all_tiers.py
@@ -88,6 +88,14 @@ def test_all_tiers_save_github_is_minimal_pinned_and_idempotent(repository, all_
assert job["with"]["tiers"] == "1,2,3"
assert "tier3-required" not in job["with"]
assert job["with"]["image"] == ALL_RELEASE["image"]
+ triggers = [
+ "skills/**",
+ ".github/workflows/skill-checks-all-tiers.yml",
+ ".github/workflows/feedback-all-tiers.yml",
+ ]
+ assert scan["on"]["pull_request"] == {"paths": triggers}
+ assert scan["on"]["push"] == {"branches": ["trunk"], "paths": triggers}
+ assert "&" not in scan_path.read_text() # no YAML aliases in Actions workflows
feedback = yaml.safe_load((repository / ".github/workflows/feedback-all-tiers.yml").read_text())
assert feedback["jobs"]["feedback"]["with"]["trusted-caller-blobs"]
assert feedback["jobs"]["feedback"]["uses"].endswith("@" + ALL_RELEASE["feedback_ref"])