Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

ci: pin the repository's skill callers to the released evaluator commit (#712)

Christopher Kevind99d30403f33
diff --git a/.github/workflows/skill-checks-all-tiers.yml b/.github/workflows/skill-checks-all-tiers.yml
index b18d38d5..725e24aa 100644
--- a/.github/workflows/skill-checks-all-tiers.yml
+++ b/.github/workflows/skill-checks-all-tiers.yml
@@ -1,4 +1,7 @@
# Generated by SkillEvaluator all-tier CI onboarding.
+# Pin only a reviewed commit that stays reachable from main, such as the
+# workflow_ref of a committed release descriptor. A pull-request branch commit
+# disappears after its squash merge and GitHub then rejects the called workflow.
name: Skill checks (all tiers)
'on':
pull_request:
@@ -21,7 +24,7 @@ permissions:
actions: read
jobs:
skills:
- uses: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@80ef7b626128a4f54e35d24bc4b73eedf637e559
+ uses: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@ba7779ad617c6444c98b6241a75bce92067e9d56
with:
tiers: 1,2,3
paths: skills
@@ -35,8 +38,8 @@ jobs:
agents: codex,claude-code
runner: ubuntu-24.04
tools-repository: NVIDIA/SkillEvaluator
- tools-ref: 80ef7b626128a4f54e35d24bc4b73eedf637e559
- image: ghcr.io/nvidia/skillevaluator-tier1@sha256:9d5483243918937be4f2178f74cf7c622a531dee41e3176906994d52fa14950a
+ tools-ref: ba7779ad617c6444c98b6241a75bce92067e9d56
+ image: ghcr.io/nvidia/skillevaluator-tier1@sha256:e96e3aaad89bf78c4f3f8c621e88005d18bd7c485b29c0d7945864700e5944e6
n-attempts: 1
n-concurrent: 1
timeout-minutes: 300
diff --git a/.github/workflows/skill-feedback-all-tiers.yml b/.github/workflows/skill-feedback-all-tiers.yml
index 46a49af3..a5083f27 100644
--- a/.github/workflows/skill-feedback-all-tiers.yml
+++ b/.github/workflows/skill-feedback-all-tiers.yml
@@ -12,20 +12,20 @@ permissions:
pull-requests: write
jobs:
feedback:
- uses: NVIDIA/SkillEvaluator/.github/workflows/feedback-all-tiers.yml@80ef7b626128a4f54e35d24bc4b73eedf637e559
+ uses: NVIDIA/SkillEvaluator/.github/workflows/feedback-all-tiers.yml@ba7779ad617c6444c98b6241a75bce92067e9d56
with:
tools-repository: NVIDIA/SkillEvaluator
- tools-ref: 80ef7b626128a4f54e35d24bc4b73eedf637e559
- trusted-workflow-ref: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@80ef7b626128a4f54e35d24bc4b73eedf637e559
- trusted-caller-blobs: b18d38d5c488fe9bc4f042371206dde11606f347
+ tools-ref: ba7779ad617c6444c98b6241a75bce92067e9d56
+ trusted-workflow-ref: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@ba7779ad617c6444c98b6241a75bce92067e9d56
+ trusted-caller-blobs: 725e24aac6cbfc65206d7fde934b9813fddf3af5
benchmarks:
- uses: NVIDIA/SkillEvaluator/.github/workflows/publish-benchmarks.yml@80ef7b626128a4f54e35d24bc4b73eedf637e559
+ uses: NVIDIA/SkillEvaluator/.github/workflows/publish-benchmarks.yml@ba7779ad617c6444c98b6241a75bce92067e9d56
permissions:
contents: write
actions: read
pull-requests: write
with:
tools-repository: NVIDIA/SkillEvaluator
- tools-ref: 80ef7b626128a4f54e35d24bc4b73eedf637e559
- trusted-workflow-ref: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@80ef7b626128a4f54e35d24bc4b73eedf637e559
- trusted-caller-blobs: b18d38d5c488fe9bc4f042371206dde11606f347
+ tools-ref: ba7779ad617c6444c98b6241a75bce92067e9d56
+ trusted-workflow-ref: NVIDIA/SkillEvaluator/.github/workflows/evaluate-all-tiers.yml@ba7779ad617c6444c98b6241a75bce92067e9d56
+ trusted-caller-blobs: 725e24aac6cbfc65206d7fde934b9813fddf3af5
diff --git a/tests/ci/test_repository_skill_ci.py b/tests/ci/test_repository_skill_ci.py
index a09108c0..c9cf4b99 100644
--- a/tests/ci/test_repository_skill_ci.py
+++ b/tests/ci/test_repository_skill_ci.py
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
"""Keep this repository's consumer callers compatible with trusted upstream CI."""
+import json
import re
import subprocess
from pathlib import Path
@@ -102,3 +103,30 @@ def test_repository_skill_has_committed_complete_dataset_prerequisites():
for manifest in skills:
digests = dataset_preflight(manifest.parent)
assert "evals/evals.json" in digests
+
+
+def test_repository_callers_pin_a_released_evaluator_revision():
+ """The pinned runtime must stay resolvable after the pull request that set it merges.
+
+ GitHub resolves ``uses: owner/repo/path@<sha>`` only for a commit reachable
+ from a branch or tag. A commit taken from a pull-request branch vanishes once
+ that branch is squash-merged and deleted, and every later run then fails before
+ any job starts with "error parsing called workflow ... workflow was not found".
+ Release descriptors name only reviewed commits on main, so the repository's own
+ callers pin one of those.
+ """
+ descriptors = [
+ ROOT / "src/skillevaluator/ci-release-all-tiers.json",
+ *sorted((ROOT / "ci/bundles").glob("*/release.json")),
+ ]
+ released = set()
+ for descriptor in descriptors:
+ data = json.loads(descriptor.read_text())
+ released.update(
+ str(value)
+ for key, value in data.items()
+ if key.endswith("_ref") and re.fullmatch(r"[0-9a-f]{40}", str(value))
+ )
+ assert released
+ runtime = workflow(SCAN)["jobs"]["skills"]["with"]["tools-ref"]
+ assert runtime in released, f"{runtime} is not a workflow_ref or tools_ref of any committed release descriptor"