Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import copy
import hashlib
import io
import json
from pathlib import Path
import pytest
from click.testing import CliRunner
from rich.console import Console
from skillevaluator.cli import cli
from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory
from skillevaluator.provider_config import ProviderConfig
from skillevaluator.reporting.benchmark import BenchmarkReporter
from skillevaluator.reporting.cli import CLIReporter
from skillevaluator.reporting.html import HTMLReporter
from skillevaluator.reporting.markdown import MarkdownReporter
from skillevaluator.tier3.baseline_store import condition_eligibility
from skillevaluator.tier3.harbor import runner, runtime_preflight
from skillevaluator.tier3.result_display import render_evaluation_result
from skillevaluator.tier3.version_comparison import compare_versions, select_prior_version, version_comparison_lines
from test_baseline_store import _job, _write
@pytest.fixture
def evaluation(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
skill = tmp_path / "demo"
_write(skill / "evals/evals.json", [])
provider = ProviderConfig(
provider="openai",
model="test-model",
api_key="test-key",
base_url="https://provider.example/v1",
litellm_model="openai/test-model",
)
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: provider)
monkeypatch.setattr(runner, "load_evals_config", lambda _path: ({"harbor": {"task_source": "evals_json"}}, None))
monkeypatch.setattr(runner, "find_evals_file", lambda _path: skill / "evals/evals.json")
monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: [])
monkeypatch.setattr(
runtime_preflight,
"probe_model",
lambda selected: runtime_preflight.ModelProbeResult(True, selected.provider, selected.model, "available"),
)
monkeypatch.setattr(runner, "render_agent_eval_html_report", lambda *_args, **_kwargs: tmp_path / "report.html")
def emit(_skill, target, **_kwargs):
task = target / "case-001"
task.mkdir(parents=True)
(task / "task.toml").write_text(
'schema_version = "1.3"\n[task]\nname = "case-001"\n[metadata]\nentry_id = "case-001"\n'
)
return [task]
monkeypatch.setattr(runner, "generate_harbor_tasks", emit)
active = {}
launches = []
def launch(**kwargs):
launches.append(kwargs)
for condition, suffix, score in (("with_skill", "with", active["score"]), ("baseline", "without", 0.5)):
if kwargs[condition] is not None:
_job(kwargs["jobs_dir"] / f"demo-opencode-{suffix}", score, grading_mode=active["grading"])
return []
monkeypatch.setattr(runner, "_run_agent_pair", launch)
def evaluate(version="a", score=0.8, **options):
active.update(score=score, grading=options.get("grading_mode", "default"))
(skill / "SKILL.md").write_text(f"---\nname: demo\ndescription: Arithmetic note\n---\nVersion {version}\n")
return runner.run_harbor_eval(
skill,
["opencode"],
output_dir=tmp_path / "results/demo",
env_mode="docker",
agent_runtime_preflight=False,
agent_models={"opencode": "openai/test-model"},
evaluated_source={"repository": "example/skill", "commit": version * 40},
**options,
)
evaluate.skill = skill
evaluate.launches = launches
return evaluate
@pytest.mark.parametrize("grading_mode", ["default", "custom_only"])
def test_live_and_readback_share_regression_math(evaluation, grading_mode):
prior = evaluation(grading_mode=grading_mode)
assert "error" not in prior, prior
current = evaluation("b", 0.2, grading_mode=grading_mode, compare_to=prior["run_id"], reuse_baseline="latest")
assert "error" not in current, current
comparison = current["prior_version_comparison"]
assert comparison["status"] == "available", comparison
assert comparison["regression"] is True
assert comparison["agents"]["opencode"]["delta"] == -0.6
assert comparison["agents"]["opencode"]["current_overall"] == 0.2
assert comparison["agents"]["opencode"]["prior_overall"] == 0.8
if grading_mode == "default":
assert current["agents"]["opencode"]["lift"]["overall"]["delta"] == -0.3
assert comparison["agents"]["opencode"]["metrics"]["accuracy"]["delta"] == -0.6
assert current["baseline"]["mode"] == "reused"
assert evaluation.launches[-1]["baseline"] is None
readback = compare_versions(Path(current["run_dir"]), Path(prior["run_dir"]))
assert readback == {key: value for key, value in comparison.items() if key != "rejected_candidates"}
def test_explicit_incompatible_refuses_before_execution(evaluation):
prior = evaluation()
current = evaluation("b", 0.2, compare_to=prior["run_id"], n_attempts=2)
assert "attempt policy changed" in str(current["error"])
assert len(evaluation.launches) == 1
def test_latest_requires_different_skill_and_can_be_unavailable(evaluation):
prior = evaluation()
same = evaluation(compare_to="latest")
unavailable = same["prior_version_comparison"]
assert unavailable["status"] == "unavailable"
assert "unchanged" in str(unavailable["rejected_candidates"])
assert "regression" not in unavailable
different = evaluation("b", 0.2, compare_to="latest")
assert different["prior_version_comparison"]["status"] == "available"
assert different["prior_version_comparison"]["source_run_id"] in {same["run_id"], prior["run_id"]}
def test_identical_explicit_source_is_flagged(evaluation):
prior = evaluation()
same = evaluation(compare_to=prior["run_id"])
comparison = same["prior_version_comparison"]
assert comparison["status"] == "available"
assert comparison["identical_skill"] is True
assert comparison["regression"] is False
assert "not a version change" in " ".join(version_comparison_lines(comparison))
def test_revision_fallback_is_used_when_publication_digest_is_missing(evaluation):
prior = evaluation("a", 0.8)
current = evaluation("b", 0.2)
for run in (prior, current):
result_path = Path(run["run_dir"]) / "result.json"
result = json.loads(result_path.read_text())
result["publication_target"]["skill_digest"] = None
result["comparison_identity"]["skill"]["publication_target"]["skill_digest"] = None
_write(result_path, result)
comparison = compare_versions(Path(current["run_dir"]), Path(prior["run_dir"]))
assert comparison["status"] == "available"
assert comparison["identical_skill"] is False
assert f"rev {'a' * 40}" in version_comparison_lines(comparison)[0]
@pytest.mark.parametrize("arm", ["prior", "current"])
@pytest.mark.parametrize("damage", ["missing", "corrupt", "unscored", "old", "partial", "failed"])
def test_missing_or_unscored_is_not_a_regression(evaluation, damage, arm):
prior, current = evaluation(), evaluation("b", 0.2)
damaged = prior if arm == "prior" else current
reward = next((Path(damaged["run_dir"]) / "opencode/with-skill/trials").glob("*/reward.json"))
row = json.loads(reward.read_text())
if damage == "missing":
reward.unlink()
elif damage == "corrupt":
reward.write_text("{")
elif damage == "failed":
damaged["agents"]["opencode"]["conditions"]["with_skill"]["execution_status"] = "failed"
_write(Path(damaged["run_dir"]) / "result.json", damaged)
else:
if damage == "unscored":
row["evaluation_status"] = "failed"
elif damage == "old":
row.pop("baseline_evidence")
else:
row["entry_id"] = "another-case"
_write(reward, row)
comparison = compare_versions(Path(current["run_dir"]), Path(prior["run_dir"]))
assert comparison["status"] == "unavailable"
assert comparison["reasons"]
assert comparison["agents"] == {}
assert "regression" not in comparison
def test_reused_no_skill_run_can_supply_prior_with_skill(evaluation):
original = evaluation()
prior = evaluation("b", 0.7, reuse_baseline=original["run_id"])
current = evaluation("c", 0.2, compare_to=prior["run_id"])
assert current["prior_version_comparison"]["status"] == "available"
def test_posthoc_is_read_only_and_prints_table_or_json(evaluation):
prior, current = evaluation(), evaluation("b", 0.2)
root = Path(current["run_dir"]).parent
def snapshot():
return {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob("*") if p.is_file()}
before = snapshot()
args = [
"tier3",
"compare-versions",
str(evaluation.skill),
"--current",
current["run_id"],
"--prior",
prior["run_id"],
"--results-dir",
str(root.parent),
]
result = CliRunner().invoke(cli, [*args, "--json"])
assert result.exit_code == 0, result.output
assert json.loads(result.output)["regression"] is True
table = CliRunner().invoke(cli, args)
assert table.exit_code == 0, table.output
assert "Regression" in table.output and "-0.6000" in table.output
invalid_args = list(args)
invalid_args[6] = "../outside"
invalid = CliRunner().invoke(cli, [*invalid_args, "--json"])
assert invalid.exit_code != 0
assert snapshot() == before
assert len(evaluation.launches) == 2
def test_all_report_formats_keep_version_delta_separate(evaluation):
prior = evaluation()
current = evaluation("b", 0.2, compare_to=prior["run_id"])
result = agent_eval_result_from_directory(
evaluation.skill, Path(current["run_dir"]), engine_result=current, use_llm_judge=False
)
assert result is not None
for reporter in (
BenchmarkReporter(skill_name="demo", report_scope="tier3"),
MarkdownReporter(),
HTMLReporter(),
CLIReporter(),
):
rendered = reporter.render_all([result])
assert "Versus prior version" in rendered
assert "regression" in rendered
stream = io.StringIO()
render_evaluation_result(current, console=Console(file=stream, width=200), verbose=False)
assert "Versus prior version" in stream.getvalue() and "regression" in stream.getvalue()
@pytest.mark.parametrize("changed", ["model", "environment", "judge", "cases", "agents"])
def test_version_compatibility_uses_shared_identity(evaluation, changed):
prior = evaluation()
identity = copy.deepcopy(prior["comparison_identity"])
if changed == "model":
identity["config"]["agents"]["opencode"]["model"] = "another-model"
elif changed == "environment":
identity["config"]["harbor"]["environment"] = "local"
elif changed == "judge":
identity["config"]["judge"]["model"] = "another-judge"
elif changed == "cases":
identity["expected_case_ids"] = ["another-case"]
else:
identity["config"]["agents"] = {}
with pytest.raises(ValueError, match="Cannot compare to"):
select_prior_version(
Path(prior["run_dir"]).parent,
prior["run_id"],
identity,
current_run_dir=Path(prior["run_dir"]).parent / "new",
)
def test_latest_rejects_incompatible_candidates(evaluation):
prior = evaluation()
identity = dict(prior["comparison_identity"])
identity["evaluator_version"] = "another-version"
selection = select_prior_version(
Path(prior["run_dir"]).parent, "latest", identity, current_run_dir=Path(prior["run_dir"]).parent / "new"
)
assert selection["status"] == "unavailable"
assert "evaluator version changed" in str(selection["rejected_candidates"])
def test_latest_never_selects_a_run_newer_than_the_current_run(evaluation):
current = evaluation("a", 0.8)
evaluation("b", 0.2)
selection = select_prior_version(
Path(current["run_dir"]).parent,
"latest",
current["comparison_identity"],
current_run_dir=Path(current["run_dir"]),
)
assert selection["status"] == "unavailable"
assert selection["reasons"] == ["No compatible completed run of a different skill revision"]
def test_with_skill_eligibility_reasons_name_the_with_skill_arm(evaluation):
run = evaluation()
summary_path = Path(run["run_dir"]) / "opencode/with-skill/summary.json"
summary = json.loads(summary_path.read_text())
summary["execution_status"] = "failed"
_write(summary_path, summary)
reasons, _ = condition_eligibility(Path(run["run_dir"]), ["opencode"], run["comparison_identity"], "with_skill")
assert "opencode: with-skill summary is incomplete or failed" in reasons
assert not any("baseline" in reason for reason in reasons)
def test_malformed_display_scores_are_unavailable():
assert "unavailable" in version_comparison_lines({"status": "available", "agents": {"agent": {}}})[0]