Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Keep the BENCHMARK.md backfill runbook aligned with the public contract."""
from __future__ import annotations
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
def test_rollout_runbook_covers_safe_backfill_and_freshness() -> None:
runbook = (REPO / "docs/benchmark-rollout.mdx").read_text(encoding="utf-8")
assert "--output-dir ./benchmark-backfill/example-skill" in runbook
assert "uv run python scripts/ci/check_public_benchmarks.py" in runbook
assert "assets/benchmark-card-before.png" in runbook
assert "assets/benchmark-card-after.png" in runbook
assert "--require-files" in runbook
assert "Do not reuse a prior live score" in runbook
assert "The generated evaluation date must come from the live run artifact" in runbook
assert "Effectiveness=50% `goal_accuracy` + 50% `behavior_check`" in runbook
assert "A dimension passes at 50%" in runbook
assert "benchmark_policy.tier2_required = false" in runbook
assert "benchmark_policy.tier3_required = false" in runbook
assert "--no-block-on-dedup" in runbook
assert "it is not a publication waiver" in runbook
assert "incomplete split-tier merge" in runbook
assert "publication `PASS`" in runbook
assert "without completed required Tier 1, Tier 2," in runbook
def test_rollout_runbook_documents_the_expected_source_options() -> None:
runbook = " ".join((REPO / "docs/benchmark-rollout.mdx").read_text(encoding="utf-8").split())
for flag in (
"--expected-source-repository",
"--expected-source-revision",
"--expected-evaluator-container-revision",
):
assert flag in runbook
assert "`--expected-*` option also implies `--require-files`" in runbook
# Only the source revision is case-folded; repository names and container
# revisions are compared exactly, so the runbook must not promise more.
assert "Hex digits in the source revision are compared without regard to case." in runbook
assert "Hex digits are compared without regard to case" not in runbook
assert "Repository names and evaluator container revisions must match the card exactly" in runbook
# The gate reads only plain one-line metadata items; anything else counts as missing.
assert "Only a plain one-line list item counts as that line" in runbook
assert (
"An item that holds a link or inline HTML, or continues onto a second line, "
"is treated as missing and reported as not recorded."
) in runbook
def test_ci_runs_benchmark_gate_with_project_dependencies() -> None:
workflow = (REPO / ".github/workflows/ci.yml").read_text(encoding="utf-8")
assert "uv run python scripts/ci/check_public_benchmarks.py --require-files tests/golden" in workflow
def test_tier3_reference_uses_canonical_dimension_mapping() -> None:
reference = (REPO / "docs/tier3-results.mdx").read_text(encoding="utf-8")
assert "| Security | Is it safe to use? | `security` | 1.0 |" in reference
assert "| Correctness | Is the answer correct? | `accuracy` | 1.0 |" in reference
assert "| Discoverability | Was the right skill loaded when needed? | `skill_execution` | 1.0 |" in reference
assert (
"| Effectiveness | Did the skill help complete the task? | `goal_accuracy` + `behavior_check` | 0.5 + 0.5 |"
) in reference
assert "`goal_accuracy` + `behavior_check` + `accuracy`" not in reference
assert "At least one successful agent has every dimension ≥ 0.50" in reference