Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Guard the Tier 1 image layout and the workflow that builds it. Building the image is left to CI."""
import fnmatch
import re
import shlex
from pathlib import Path
import yaml
ROOT = Path(__file__).parents[2]
DOCKERFILE = ROOT / "ci/Dockerfile.tier1"
def _instructions() -> list[tuple[str, str]]:
"""Dockerfile instructions as (keyword, arguments), with continuations joined and comments dropped."""
text = re.sub(r"\\\n", " ", DOCKERFILE.read_text(encoding="utf-8"))
lines = [line.strip() for line in text.splitlines()]
return [tuple(line.split(None, 1)) for line in lines if line and not line.startswith("#")]
def _stages() -> dict[str, list[tuple[str, str]]]:
"""Instructions per stage. The unnamed final stage is the shipped runtime."""
stages: dict[str, list[tuple[str, str]]] = {}
for keyword, arguments in _instructions():
if keyword == "FROM":
words = arguments.split()
name = words[2] if len(words) == 3 and words[1].upper() == "AS" else "runtime"
stages[name] = []
stages[name].append((keyword, arguments))
assert list(stages) == ["build", "project", "runtime"]
return stages
def _arguments(stage: list[tuple[str, str]], keyword: str) -> list[str]:
return [arguments for name, arguments in stage if name == keyword]
def _workflow() -> dict:
path = ROOT / ".github/workflows/publish-tier1-image.yml"
return yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader)
def _build_steps(job: dict) -> list[dict]:
return [step for step in job["steps"] if step.get("uses", "").startswith("docker/build-push-action@")]
def test_runtime_stage_ships_only_what_scans_run():
runtime = _stages()["runtime"]
assert _arguments(runtime, "COPY") == [
"--from=build /opt/semgrep /opt/semgrep",
"--from=build /opt/skillspector /opt/skillspector",
"--from=build /usr/local/bin/gitleaks /usr/local/bin/gitleaks",
"--from=build /usr/local/share/licenses/gitleaks /usr/local/share/licenses/gitleaks",
"--from=build /opt/skillevaluator/.venv /opt/skillevaluator/.venv",
"--from=project /overlay/ /",
"LICENSE NOTICE THIRD_PARTY_NOTICES.md /opt/skillevaluator/",
"ci /opt/skillevaluator/ci",
]
commands = {word for command in _arguments(runtime, "RUN") for word in shlex.split(command)}
assert {"uv", "curl", "pip"}.isdisjoint(commands)
assert {"git", "ca-certificates"} <= commands
assert _arguments(runtime, "USER") == ["evaluator"]
assert _arguments(runtime, "ENTRYPOINT") == ['["python", "/opt/skillevaluator/ci/run_tier1.py"]']
def test_build_and_runtime_stages_share_one_pinned_base():
bases = {arguments.split()[0] for keyword, arguments in _instructions() if keyword == "FROM"}
assert bases - {"build"} == {
"python:3.12-slim-bookworm@sha256:62ed8e07e544e8555c5d93748d5275015136b7f5c52b5e8679b4de339c3779d3"
}
def test_bytecode_is_compiled_at_build_time_and_never_written_at_runtime():
stages = _stages()
build_environment = " ".join(_arguments(stages["build"], "ENV")).split()
assert {"UV_COMPILE_BYTECODE=1", "PYC_INVALIDATION_MODE=UNCHECKED_HASH"} <= set(build_environment)
runtime = stages["runtime"]
assert "PYTHONDONTWRITEBYTECODE=1" in " ".join(_arguments(runtime, "ENV")).split()
stdlib = [command for command in _arguments(runtime, "RUN") if command.endswith("/usr/local/lib/python3.12")]
assert len(stdlib) == 1
assert "-m compileall" in stdlib[0] and "--invalidation-mode unchecked-hash" in stdlib[0]
def test_source_edits_rebuild_only_the_last_layers():
stages = _stages()
build = stages["build"]
first_source = next(
index
for index, (keyword, arguments) in enumerate(build)
if keyword == "COPY" and not arguments.startswith("ci/requirements/")
)
scanners = [
index
for index, (keyword, arguments) in enumerate(build)
if keyword == "RUN" and re.search(r"/opt/semgrep|/opt/skillspector|gitleaks", arguments)
]
assert len(scanners) == 3 and max(scanners) < first_source
assert build[first_source] == ("COPY", "pyproject.toml uv.lock ./")
assert _arguments(stages["project"], "COPY") == ["README.md LICENSE NOTICE THIRD_PARTY_NOTICES.md ./", "src ./src"]
runtime = stages["runtime"]
overlay = runtime.index(("COPY", "--from=project /overlay/ /"))
checks = next(index for index, (_, arguments) in enumerate(runtime) if "semgrep --version" in arguments)
shared = [index for index, (_, arguments) in enumerate(runtime) if arguments.startswith("--from=build ")]
assert max([*shared, checks]) < overlay
def _dockerignored(path: str) -> bool:
lines = (ROOT / ".dockerignore").read_text(encoding="utf-8").splitlines()
patterns = [line.strip().rstrip("/") for line in lines if line.strip() and not line.startswith("#")]
parts = Path(path).parts
prefixes = ["/".join(parts[:count]) for count in range(1, len(parts) + 1)]
return any(fnmatch.fnmatchcase(prefix, pattern) for pattern in patterns for prefix in prefixes)
def test_build_context_drops_unused_trees_but_keeps_image_inputs():
copied = [
source
for keyword, arguments in _instructions()
if keyword == "COPY" and not arguments.startswith("--from=")
for source in arguments.split()[:-1]
]
# The repository's root Dockerfile also builds the wheel from this context.
package = ["pyproject.toml", "README.md", "src/skillevaluator/reporting/templates/report.html.j2"]
for path in copied + package:
assert (ROOT / path).exists(), path
assert not _dockerignored(path), path
unused = ["tests/ci/test_image_build.py", "docs/github-actions.mdx", "scripts", "fern", ".github", "CHANGELOG.md"]
for path in unused:
assert (ROOT / path).exists(), path
assert _dockerignored(path), path
def test_pull_requests_read_but_never_write_the_layer_cache():
workflow = _workflow()
assert workflow["on"]["push"]["branches"] == ["main"]
assert workflow["on"]["push"]["paths"] == workflow["on"]["pull_request"]["paths"]
(verify,) = _build_steps(workflow["jobs"]["verify"])
assert verify["with"]["cache-from"] == "type=gha,scope=tier1-image"
cache_to = verify["with"]["cache-to"]
assert "github.event_name != 'pull_request'" in cache_to and "github.ref == 'refs/heads/main'" in cache_to
assert "'type=gha,scope=tier1-image,mode=max,ignore-error=true' || ''" in cache_to
(release,) = _build_steps(workflow["jobs"]["publish"])
for step in (verify, release):
assert step["with"]["context"] == "."
assert step["with"]["file"] == "ci/Dockerfile.tier1"
assert step["with"]["load"] == "true" and "push" not in step["with"]
# Privileged events must never execute PR-controlled image/cache code.
for path in (ROOT / ".github/workflows").glob("*.yml"):
document = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader)
triggers = document["on"]
triggers = [triggers] if isinstance(triggers, str) else triggers
assert "pull_request_target" not in triggers, path.name
if "issue_comment" in triggers:
# NVSkills requests pass metadata to an immutable trusted service;
# they must not gain local steps, PR checkouts, or cache writers.
assert path.name == "request-nvskills-ci.yml"
assert set(triggers) == {"issue_comment", "pull_request", "push"}
assert document["on"]["issue_comment"] == {"types": ["created"]}
assert document["permissions"] == {"contents": "read"}
assert set(document["jobs"]) == {"request"}
caller = document["jobs"]["request"]
assert set(caller) == {"if", "permissions", "uses", "secrets"}
assert re.fullmatch(r"NVIDIA/skills/\.github/workflows/team-request\.yml@[0-9a-f]{40}", caller["uses"])
assert caller["permissions"] == {"contents": "read", "pull-requests": "read", "statuses": "read"}
assert caller["secrets"] == {"NVSKILLS_CI_DISPATCH_TOKEN": "${{ secrets.NVSKILLS_CI_DISPATCH_TOKEN }}"}
if "workflow_run" in triggers:
# The consumer listener calls only the immutable trusted publishers;
# it has no local steps, candidate checkout or image build/cache job.
assert path.name == "skill-feedback-all-tiers.yml"
assert set(triggers) == {"workflow_run"}
publishers = {"feedback": "feedback-all-tiers.yml", "benchmarks": "publish-benchmarks.yml"}
assert set(document["jobs"]) == set(publishers)
feedback = document["jobs"]["feedback"]
source = feedback["uses"].rsplit("@", 1)[1]
for name, publisher in publishers.items():
caller = document["jobs"][name]
expected_keys = {"uses", "with", "permissions"} if name == "benchmarks" else {"uses", "with"}
assert set(caller) == expected_keys
assert re.fullmatch(
rf"NVIDIA/SkillEvaluator/\.github/workflows/{re.escape(publisher)}@[0-9a-f]{{40}}",
caller["uses"],
)
assert caller["uses"].rsplit("@", 1)[1] == source
assert caller["with"]["tools-repository"] == "NVIDIA/SkillEvaluator"
assert caller["with"]["tools-ref"] == source
assert caller["with"] == feedback["with"]
assert document["jobs"]["benchmarks"]["permissions"] == {
"contents": "write",
"actions": "read",
"pull-requests": "write",
}
def test_release_builds_reuse_no_cached_layers():
# A cached apt layer would ship the git and ca-certificates of whichever main build first wrote it.
workflow = _workflow()
(verify,) = _build_steps(workflow["jobs"]["verify"])
(release,) = _build_steps(workflow["jobs"]["publish"])
assert "cache-from" not in release["with"]
assert release["with"]["no-cache"] == "true"
# The release still refreshes the cache that later verify builds read.
assert release["with"]["cache-to"] == "type=gha,scope=tier1-image,mode=max,ignore-error=true"
assert verify["with"]["cache-from"] == "type=gha,scope=tier1-image"
def test_release_builds_once_and_pushes_the_image_it_verified():
jobs = _workflow()["jobs"]
publish = jobs["publish"]
# Exactly one of the two building jobs runs for any event.
assert jobs["verify"]["if"] == "${{ !(" + publish["if"] + ") }}"
assert "needs" not in publish
(build,) = _build_steps(publish)
assert build["with"]["tags"] == "${{ env.IMAGE }}:sha-${{ github.sha }}"
assert build["with"]["labels"].splitlines() == [
"org.opencontainers.image.source=https://github.com/NVIDIA/SkillEvaluator",
"org.opencontainers.image.revision=${{ github.sha }}",
"org.opencontainers.image.description=SkillEvaluator complete Tier 1 CI runtime",
]
steps = publish["steps"]
smoke = next(index for index, step in enumerate(steps) if "ci/smoke_image.py" in step.get("run", ""))
release = next(index for index, step in enumerate(steps) if step.get("id") == "release")
assert steps.index(build) < smoke < release
assert '--image "$IMAGE:sha-$GITHUB_SHA"' in steps[smoke]["run"]
assert 'docker push "$IMAGE:sha-$GITHUB_SHA"' in steps[release]["run"]
runs = [step.get("run", "") for job in jobs.values() for step in job["steps"]]
assert not any("docker build" in run for run in runs)