Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from io import StringIO
import pytest
from rich.console import Console
from skillevaluator.evaluation.tier3_report import _build_agent, build_agent_eval_payload
from skillevaluator.models import ValidationResult
from skillevaluator.reporting import BenchmarkReporter, CLIReporter, HTMLReporter
from skillevaluator.tier3.evidence_volume import (
evidence_volume,
evidence_volume_for_conditions,
format_evidence_volume,
)
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, DEFAULT_SCORE_POLICY
from skillevaluator.tier3.result_display import render_result
def _condition(*, cases: int, planned: int, scored: int, judge_failures: int = 0) -> dict:
condition = {
"execution_status": "succeeded" if scored == planned else "failed",
"expected_cases": cases,
"scored_cases": cases,
"expected_attempts": planned,
"scored_attempts": scored,
"execution_errors": [],
}
if judge_failures:
condition["judge_failures"] = judge_failures
return condition
def _volume(status: str) -> dict:
if status == "sufficient":
return evidence_volume(
cases=10,
attempts_planned=20,
attempts_scored=20,
execution_failures=0,
judge_failures=0,
by_arm={
"with_skill": {
"cases": 10,
"attempts_planned": 20,
"attempts_scored": 20,
"execution_failures": 0,
"judge_failures": 0,
},
"without_skill": {
"cases": 10,
"attempts_planned": 20,
"attempts_scored": 20,
"execution_failures": 0,
"judge_failures": 0,
},
},
)
return evidence_volume(
cases=3,
attempts_planned=30,
attempts_scored=27,
execution_failures=2,
judge_failures=1,
by_arm={
"with_skill": {
"cases": 3,
"attempts_planned": 30,
"attempts_scored": 27,
"execution_failures": 2,
"judge_failures": 1,
}
},
)
def _agent_eval(volume: dict) -> dict:
conditions = {
"with_skill": _condition(
cases=volume["cases"],
planned=volume["attempts_planned"],
scored=volume["attempts_scored"],
judge_failures=volume["judge_failures"],
),
"without_skill": _condition(cases=10, planned=20, scored=20),
}
return {
"execution_status": "succeeded",
"verdict": "pass",
"best_agent": "test-agent",
"skill_name": "test-skill",
"overall_score": 0.8,
"overall_lift": 0.1,
"agents_run": ["test-agent"],
"agents": {
"test-agent": {
"execution_status": "succeeded",
"with_skill": 0.8,
"baseline": 0.7,
"lift": 0.1,
"expected_attempts": 40,
"scored_attempts": 40,
"metrics_with_skill": ["security"],
"metrics_without_skill": ["security"],
"conditions": {
"with_skill": _condition(cases=10, planned=20, scored=20),
"without_skill": _condition(cases=10, planned=20, scored=20),
},
"evidence_volume": volume,
"dimensions": [],
"evaluators": {},
"pass_at_k": {},
"trials": [],
"cases": [],
}
},
"conditions": conditions,
"evidence_volume": volume,
"dataset_summary": {"total_tasks": 10, "negative_tasks": 1},
"dataset_sufficiency": {
"status": "sufficient" if not volume["limited"] else "provisional",
"coverage_summary": "Dataset size: test",
},
"attempt_policy": {"max_attempts": 2, "stop_on_pass": False},
"verdict_policy": {"attempt_pass_threshold": 0.5},
"evaluators": {},
"dimensions": [],
"evaluator_cards": [],
"cases": [],
"trials": [],
"pass_at_k": {},
"execution_errors": [],
}
def _provisional_agent_eval() -> dict:
conditions = {
"with_skill": _condition(cases=3, planned=30, scored=30),
"without_skill": _condition(cases=3, planned=30, scored=30),
}
payload = build_agent_eval_payload(
"test-skill",
{
"test-agent": {
"with_skill": dict.fromkeys(DEFAULT_METRICS, 0.8),
"without_skill": dict.fromkeys(DEFAULT_METRICS, 0.7),
"conditions": conditions,
"execution_status": "succeeded",
"expected_attempts": 60,
"scored_attempts": 60,
"rewards": [],
"rewards_baseline": [],
"rewards_complete": True,
"rewards_baseline_complete": True,
}
},
attempt_policy={"max_attempts": 10, "stop_on_pass": False},
persisted_dataset_summary={"total_tasks": 3, "negative_tasks": 1, "unclassified_tasks": 0},
)
assert payload is not None
return payload
@pytest.mark.parametrize("status", ("provisional", "sufficient"))
def test_evidence_volume_is_rendered_on_all_report_surfaces(status):
ae = _provisional_agent_eval() if status == "provisional" else _agent_eval(_volume(status))
volume = ae["evidence_volume"]
expected = format_evidence_volume(volume)
assert expected.startswith("Limited evidence:") == (status == "provisional")
cli = StringIO()
CLIReporter._print_agent_eval_tables(ae, Console(file=cli, force_terminal=False))
assert expected in " ".join(cli.getvalue().split())
evaluate_output = render_result(ae)
assert expected in " ".join(evaluate_output.split())
assert "test-agent: " not in evaluate_output
result = ValidationResult(validator_name="AGENT_EVAL", validator_description="Live evaluation")
result.metadata["agent_eval"] = ae
html = HTMLReporter(include_timestamp=False).render_all([result])
assert expected in html
assert ("t3-evidence-volume warning" in html) == (status == "provisional")
benchmark = BenchmarkReporter(include_timestamp=False, report_scope="tier3").render_all([result])
assert f"Evidence volume: {expected}." in benchmark
def test_default_benchmark_publication_scope_renders_evidence_volume_once():
ae = _agent_eval(_volume("sufficient"))
result = ValidationResult(validator_name="AGENT_EVAL", validator_description="Live evaluation")
result.metadata["agent_eval"] = ae
output = BenchmarkReporter(include_timestamp=False).render_all([result])
expected = f"- Evidence volume: {format_evidence_volume(ae['evidence_volume'])}."
assert expected in output
assert output.count(expected) == 1
def test_evidence_volume_case_fallbacks_and_unknown_case_counts():
from_summary = evidence_volume_for_conditions(
{"with_skill": {"expected_attempts": 2, "scored_attempts": 2}},
dataset_summary={"total_tasks": 4},
)
assert from_summary["cases"] == 4
assert from_summary["by_arm"]["with_skill"]["cases"] == 4
from_rewards = evidence_volume_for_conditions(
{"with_skill": {"expected_attempts": 2, "scored_attempts": 2}},
rewards_by_arm={"with_skill": [{"entry_id": "case-1"}, {"entry_id": "case-2"}]},
)
assert from_rewards["cases"] == 2
unknown = evidence_volume_for_conditions({"with_skill": {"expected_attempts": 2, "scored_attempts": 2}})
assert unknown["cases"] is None
assert unknown["limited"] is False
assert "0 cases" not in format_evidence_volume(unknown)
def test_attempt_policy_marks_nonuniform_evidence_in_agent_payload():
conditions = {
"with_skill": _condition(cases=3, planned=5, scored=5),
"without_skill": _condition(cases=3, planned=5, scored=5),
}
for condition in conditions.values():
condition["execution_status"] = "succeeded"
payload = build_agent_eval_payload(
"test-skill",
{
"test-agent": {
"with_skill": dict.fromkeys(DEFAULT_METRICS, 0.8),
"without_skill": dict.fromkeys(DEFAULT_METRICS, 0.7),
"conditions": conditions,
"execution_status": "succeeded",
"rewards": [],
"rewards_baseline": [],
"rewards_complete": True,
"rewards_baseline_complete": True,
}
},
attempt_policy={"max_attempts": 2, "stop_on_pass": True},
persisted_dataset_summary={"total_tasks": 3, "negative_tasks": 1, "unclassified_tasks": 0},
)
assert payload is not None
evidence = payload["evidence_volume"]
assert evidence["uniform_attempts"] is False
assert evidence["attempts_per_case"] == 2
assert "Based on 3 cases, 5 attempts (up to 2 per case)" in evidence["display"]
def test_any_incomplete_non_skipped_arm_marks_evidence_limited():
volume = evidence_volume_for_conditions(
{
"with_skill": _condition(cases=10, planned=10, scored=10),
"without_skill": _condition(cases=10, planned=10, scored=9),
}
)
assert volume["limited"] is True
assert format_evidence_volume(volume).startswith("Limited evidence:")
def test_failure_case_ids_do_not_inflate_known_case_count():
volume = evidence_volume_for_conditions(
{
"with_skill": {
**_condition(cases=1, planned=2, scored=1),
"failure_cases": 8,
}
},
trial_failures={
"with_skill": [
{"trial": "case-001__0IgxLd6", "reason": "execution failure"},
{"trial": "case-002__0IgxLd6", "reason": "execution failure"},
]
},
)
assert volume["cases"] == 1
def test_nonuniform_attempts_use_per_case_maximum():
volume = evidence_volume(
cases=3,
attempts_planned=5,
attempts_scored=5,
execution_failures=0,
judge_failures=0,
uniform_attempts=False,
attempts_per_case=2,
)
assert "Based on 3 cases, 5 attempts (up to 2 per case)" in format_evidence_volume(volume)
def test_html_evidence_volume_survives_bounded_large_payload():
ae = _provisional_agent_eval()
ae["trials"] = [{"trial_id": f"case-{index}", "output": "x" * 300} for index in range(1200)]
expected = format_evidence_volume(ae["evidence_volume"])
result = ValidationResult(validator_name="AGENT_EVAL", validator_description="Live evaluation")
result.metadata["agent_eval"] = ae
html = HTMLReporter(include_timestamp=False).render_all([result])
assert expected in html
def test_result_display_labels_each_agent_evidence_line():
result = {
"execution_status": "succeeded",
"attempt_policy": {"max_attempts": 2, "stop_on_pass": False},
"agents": {
"codex": {"conditions": {"with_skill": _condition(cases=10, planned=20, scored=18)}},
"opencode": {"conditions": {"with_skill": _condition(cases=10, planned=20, scored=20)}},
},
}
output = render_result(result)
assert "codex: Limited evidence:" in output
assert "opencode: Based on" in output
def test_evidence_volume_counts_cases_once_and_separates_failures_by_arm():
conditions = {
"with_skill": _condition(cases=3, planned=30, scored=27),
"without_skill": _condition(cases=3, planned=30, scored=29),
}
volume = evidence_volume_for_conditions(
conditions,
trial_failures={
"with_skill": [{"trial": "case-1__attempt-10", "reason": "Required judge evaluation failed: accuracy"}]
},
)
assert volume["cases"] == 3
assert volume["attempts_planned"] == 30
assert volume["attempts_scored"] == 27
assert volume["execution_failures"] == 2
assert volume["judge_failures"] == 1
assert volume["by_arm"]["without_skill"] == {
"cases": 3,
"attempts_planned": 30,
"attempts_scored": 29,
"execution_failures": 1,
"judge_failures": 0,
}
def test_evidence_volume_formats_singular_and_complete_coverage():
value = evidence_volume(
cases=1,
attempts_planned=1,
attempts_scored=1,
execution_failures=0,
judge_failures=0,
)
assert format_evidence_volume(value) == (
"Limited evidence: Based on 1 case \u00d7 1 attempt = 1 scored attempt (0 execution failures, 0 judge failures)"
)
def test_agent_payload_includes_evidence_volume_with_per_arm_counts():
conditions = {
"with_skill": _condition(cases=3, planned=30, scored=27),
"without_skill": _condition(cases=3, planned=30, scored=29),
}
conditions["with_skill"]["judge_failures"] = 1
agent = _build_agent(
"test-agent",
{
"with_skill": dict.fromkeys(DEFAULT_METRICS, 0.8),
"without_skill": dict.fromkeys(DEFAULT_METRICS, 0.7),
"conditions": conditions,
"execution_status": "failed",
"expected_attempts": 60,
"scored_attempts": 56,
"rewards": [],
"rewards_baseline": [],
"rewards_complete": True,
"rewards_baseline_complete": True,
},
list(DEFAULT_METRICS),
list(DEFAULT_METRICS),
None,
DEFAULT_SCORE_POLICY,
)
assert agent["evidence_volume"]["cases"] == 3
assert agent["evidence_volume"]["attempts_planned"] == 30
assert agent["evidence_volume"]["attempts_scored"] == 27
assert agent["evidence_volume"]["execution_failures"] == 2
assert agent["evidence_volume"]["judge_failures"] == 1
assert set(agent["evidence_volume"]["by_arm"]) == {"with_skill", "without_skill"}