Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
ci: run the all-tier skill scan only when skills or its callers change (#441)
Christopher Kevinfff53328887a
diff --git a/.github/workflows/skill-checks-all-tiers.yml b/.github/workflows/skill-checks-all-tiers.ymlindex 66c8f37c..09bc951c 100644--- a/.github/workflows/skill-checks-all-tiers.yml+++ b/.github/workflows/skill-checks-all-tiers.yml@@ -1,10 +1,18 @@# Generated by SkillEvaluator all-tier CI onboarding.name: Skill checks (all tiers)'on':- pull_request: null+ pull_request:+ paths:+ - skills/**+ - .github/workflows/skill-checks-all-tiers.yml+ - .github/workflows/skill-feedback-all-tiers.ymlpush:branches:- main+ paths:+ - skills/**+ - .github/workflows/skill-checks-all-tiers.yml+ - .github/workflows/skill-feedback-all-tiers.ymlworkflow_dispatch: nullpermissions:contents: readdiff --git a/.github/workflows/skill-feedback-all-tiers.yml b/.github/workflows/skill-feedback-all-tiers.ymlindex 8656384a..63f7b3be 100644--- a/.github/workflows/skill-feedback-all-tiers.yml+++ b/.github/workflows/skill-feedback-all-tiers.yml@@ -17,4 +17,4 @@ jobs:tools-repository: NVIDIA/SkillEvaluatortools-ref: c3b298c2b1a1a89d520bb22009f2668520127931trusted-workflow-ref: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@c3b298c2b1a1a89d520bb22009f2668520127931- trusted-caller-blobs: 66c8f37c7987ec680d66b8365fde8557208b5257+ trusted-caller-blobs: 09bc951c5df415090d7e6c8159d87ab0fd1e98f8diff --git a/docs/github-actions.mdx b/docs/github-actions.mdxindex 71a69218..cf47dd38 100644--- a/docs/github-actions.mdx+++ b/docs/github-actions.mdx@@ -296,6 +296,10 @@ or let the [CLI](/skills/skillevaluator/cli-onboarding) handle it.collection each time. No changed skills is a successful **Skipped** result,not a scanner pass. Changes to shared files outside the skill roots are notinferred dependencies; use `all` when they should trigger a full scan.+ The generated all-tier caller (`skillevaluator ci init`) also carries+ `paths` filters, so a pull request that changes nothing under the selected+ skill folders or the two caller workflows starts no run and receives no+ comment.Selection is capped at 100 skills. Timeouts retain completed results andmark unfinished work incomplete. The workflow scans up to four skills atonce; `scan-workers` accepts 1–16.diff --git a/src/skillevaluator/_ci_onboarding_all_tiers.py b/src/skillevaluator/_ci_onboarding_all_tiers.pyindex 90a51831..957d78f0 100644--- a/src/skillevaluator/_ci_onboarding_all_tiers.py+++ b/src/skillevaluator/_ci_onboarding_all_tiers.py@@ -368,6 +368,16 @@ def _github_feedback_path(repo: Path) -> str:return FEEDBACK+def _github_trigger_paths(roots: list[str], feedback_path: str) -> list[str]:+ """Path filters so only skill or caller changes start the all-tier scan.++ A pull request outside these paths neither spends runner time nor receives a+ feedback comment. Caller changes still run so the setup change is exercised.+ """+ filters = ["**" if root in {"", "."} else f"{root}/**" for root in roots]+ return [*dict.fromkeys(filters), SCAN, feedback_path]++def render(wizard, repo, release, platform, branch, roots, options):common = {"tiers": "1,2,3",@@ -392,9 +402,15 @@ def render(wizard, repo, release, platform, branch, roots, options):header = "# Generated by SkillEvaluator all-tier CI onboarding.\n"if platform == "github":feedback_path = _github_feedback_path(repo)+ triggers = _github_trigger_paths(roots, feedback_path)scan = {"name": "Skill checks (all tiers)",- "on": {"pull_request": None, "push": {"branches": [branch]}, "workflow_dispatch": None},+ "on": {+ # Separate lists: the dumper would otherwise emit a YAML alias.+ "pull_request": {"paths": list(triggers)},+ "push": {"branches": [branch], "paths": list(triggers)},+ "workflow_dispatch": None,+ },"permissions": {"contents": "read", "id-token": "write", "pull-requests": "read", "actions": "read"},"jobs": {"skills": {diff --git a/tests/ci/test_repository_skill_ci.py b/tests/ci/test_repository_skill_ci.pyindex f5fb2b22..4f89cb10 100644--- a/tests/ci/test_repository_skill_ci.py+++ b/tests/ci/test_repository_skill_ci.py@@ -20,6 +20,11 @@ def test_repository_skill_checks_use_reviewed_full_three_tier_configuration():scan = workflow(SCAN)assert set(scan["on"]) == {"pull_request", "push", "workflow_dispatch"}assert scan["on"]["push"]["branches"] == ["main"]+ # Unrelated pull requests start no run and receive no feedback comment;+ # caller changes still run so the setup change is exercised.+ triggers = ["skills/**", SCAN, FEEDBACK]+ assert scan["on"]["pull_request"] == {"paths": triggers}+ assert scan["on"]["push"]["paths"] == triggersassert scan["permissions"] == {"contents": "read","actions": "read",diff --git a/tests/test_ci_onboarding_all_tiers.py b/tests/test_ci_onboarding_all_tiers.pyindex df4c19df..01e41159 100644--- a/tests/test_ci_onboarding_all_tiers.py+++ b/tests/test_ci_onboarding_all_tiers.py@@ -88,6 +88,14 @@ def test_all_tiers_save_github_is_minimal_pinned_and_idempotent(repository, all_assert job["with"]["tiers"] == "1,2,3"assert "tier3-required" not in job["with"]assert job["with"]["image"] == ALL_RELEASE["image"]+ triggers = [+ "skills/**",+ ".github/workflows/skill-checks-all-tiers.yml",+ ".github/workflows/feedback-all-tiers.yml",+ ]+ assert scan["on"]["pull_request"] == {"paths": triggers}+ assert scan["on"]["push"] == {"branches": ["trunk"], "paths": triggers}+ assert "&" not in scan_path.read_text() # no YAML aliases in Actions workflowsfeedback = yaml.safe_load((repository / ".github/workflows/feedback-all-tiers.yml").read_text())assert feedback["jobs"]["feedback"]["with"]["trusted-caller-blobs"]assert feedback["jobs"]["feedback"]["uses"].endswith("@" + ALL_RELEASE["feedback_ref"])