Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

3.6 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Keep the BENCHMARK.md backfill runbook aligned with the public contract."""

from __future__ import annotations

from pathlib import Path

REPO = Path(__file__).resolve().parents[1]


def test_rollout_runbook_covers_safe_backfill_and_freshness() -> None:
    runbook = (REPO / "docs/benchmark-rollout.mdx").read_text(encoding="utf-8")

    assert "--output-dir ./benchmark-backfill/example-skill" in runbook
    assert "uv run python scripts/ci/check_public_benchmarks.py" in runbook
    assert "assets/benchmark-card-before.png" in runbook
    assert "assets/benchmark-card-after.png" in runbook
    assert "--require-files" in runbook
    assert "Do not reuse a prior live score" in runbook
    assert "The generated evaluation date must come from the live run artifact" in runbook
    assert "Effectiveness=50% `goal_accuracy` + 50% `behavior_check`" in runbook
    assert "A dimension passes at 50%" in runbook
    assert "benchmark_policy.tier2_required = false" in runbook
    assert "benchmark_policy.tier3_required = false" in runbook
    assert "--no-block-on-dedup" in runbook
    assert "it is not a publication waiver" in runbook
    assert "incomplete split-tier merge" in runbook
    assert "publication `PASS`" in runbook
    assert "without completed required Tier 1, Tier 2," in runbook


def test_rollout_runbook_documents_the_expected_source_options() -> None:
    runbook = " ".join((REPO / "docs/benchmark-rollout.mdx").read_text(encoding="utf-8").split())

    for flag in (
        "--expected-source-repository",
        "--expected-source-revision",
        "--expected-evaluator-container-revision",
    ):
        assert flag in runbook
    assert "`--expected-*` option also implies `--require-files`" in runbook
    # Only the source revision is case-folded; repository names and container
    # revisions are compared exactly, so the runbook must not promise more.
    assert "Hex digits in the source revision are compared without regard to case." in runbook
    assert "Hex digits are compared without regard to case" not in runbook
    assert "Repository names and evaluator container revisions must match the card exactly" in runbook
    # The gate reads only plain one-line metadata items; anything else counts as missing.
    assert "Only a plain one-line list item counts as that line" in runbook
    assert (
        "An item that holds a link or inline HTML, or continues onto a second line, "
        "is treated as missing and reported as not recorded."
    ) in runbook


def test_ci_runs_benchmark_gate_with_project_dependencies() -> None:
    workflow = (REPO / ".github/workflows/ci.yml").read_text(encoding="utf-8")

    assert "uv run python scripts/ci/check_public_benchmarks.py --require-files tests/golden" in workflow


def test_tier3_reference_uses_canonical_dimension_mapping() -> None:
    reference = (REPO / "docs/tier3-results.mdx").read_text(encoding="utf-8")

    assert "| Security | Is it safe to use? | `security` | 1.0 |" in reference
    assert "| Correctness | Is the answer correct? | `accuracy` | 1.0 |" in reference
    assert "| Discoverability | Was the right skill loaded when needed? | `skill_execution` | 1.0 |" in reference
    assert (
        "| Effectiveness | Did the skill help complete the task? | `goal_accuracy` + `behavior_check` | 0.5 + 0.5 |"
    ) in reference
    assert "`goal_accuracy` + `behavior_check` + `accuracy`" not in reference
    assert "At least one successful agent has every dimension ≥ 0.50" in reference