Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Public behavior of direct tier workflows and their expert commands."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from click.testing import CliRunner
from skillevaluator.cli import cli
from skillevaluator.cli_help import render_help
from skillevaluator.models.result import ValidationResult
@pytest.fixture(autouse=True)
def clean_provider_environment(monkeypatch):
import os
for name in os.environ:
if name.startswith(("SKILL_EVAL_", "SKILLSPECTOR_")) or name in {
"OPENAI_API_KEY",
"NVIDIA_API_KEY",
"ANTHROPIC_API_KEY",
"LLM_JUDGE_MODEL",
}:
monkeypatch.delenv(name)
@pytest.fixture
def skill(tmp_path):
path = tmp_path / "sample"
path.mkdir()
(path / "SKILL.md").write_text(
"---\nname: sample\ndescription: Use when asked to summarize a local text file.\n"
"metadata:\n author: Example Author <[email protected]>\n---\n"
"# Summarize text\n\nRead the supplied file and return a concise summary.\n",
)
return path
@pytest.fixture(autouse=True)
def isolate_scanner_preparation(monkeypatch):
# Workflow composition tests do not install tools. Ordering tests below
# replace this stub with explicit spies for both rejected and valid inputs.
monkeypatch.setattr("skillevaluator.cli._offer_scanner_install", lambda *_args: None)
def _result(name):
result = ValidationResult(validator_name=name)
result.add_success("checked", "Completed this check")
return [result]
def _report(output_dir):
files = list(output_dir.glob("*.json"))
assert len(files) == 1
return json.loads(files[0].read_text())
@pytest.mark.parametrize("tier", ["tier1", "tier2", "tier3"])
def test_bare_tier_displays_help_without_running(tier):
result = CliRunner().invoke(cli, [tier])
assert result.exit_code == 0, result.output
assert "Usage:" in result.output
assert "PATH" in result.output
assert "Commands:" in result.output
@pytest.mark.parametrize("tier", ["tier1", "tier2", "tier3"])
def test_unknown_subcommand_and_removed_run_are_errors(tier):
for name in ("misspelled-command", "run"):
result = CliRunner().invoke(cli, [tier, name])
assert result.exit_code == 2
assert "No such command" in result.output
def test_direct_tier1_runs_only_requested_static_checks(skill, tmp_path):
output = tmp_path / "reports"
result = CliRunner().invoke(
cli,
["tier1", str(skill), "--checks", "schema", "--no-llm", "-r", "json", "-o", str(output)],
)
assert result.exit_code == 0, result.output
payload = _report(output)
assert len(payload["results"]) == 1
assert "Schema" in payload["results"][0]["validator"]
assert payload["results"][0]["workflow"]["tier"] == 1
def test_direct_tier1_default_reports_stay_outside_publication_target(skill, tmp_path, monkeypatch):
from skillevaluator.publication_identity import publication_target_from_path
from skillevaluator.tier3.output_provenance import is_generated_output_root
initial_target = publication_target_from_path(skill)
assert initial_target is not None
monkeypatch.chdir(skill)
result = CliRunner().invoke(cli, ["tier1", ".", "--checks", "schema", "--no-llm", "-r", "json"])
# The minimal fixture fails schema; the report location is what matters here.
assert result.exception is None or isinstance(result.exception, SystemExit), result.output
relocated = skill.with_name(f"{skill.name}-reports")
assert _report(relocated)["results"][0]["workflow"]["tier"] == 1
assert is_generated_output_root(relocated)
assert not (skill / "reports").exists()
assert publication_target_from_path(skill) == initial_target
def test_direct_tier1_options_before_path_and_failed_findings(skill, tmp_path):
(skill / "SKILL.md").write_text("Missing required frontmatter\n")
result = CliRunner().invoke(
cli,
["tier1", "--checks", "schema", "--no-llm", str(skill), "-r", "json", "-o", str(tmp_path / "reports")],
)
assert result.exit_code == 1, result.output
assert _report(tmp_path / "reports")["results"][0]["passed"] is False
def test_default_tier1_includes_dependency_and_configured_llm_stages(skill, monkeypatch):
from skillevaluator.tier1 import commands
calls = []
def validation(path, **kwargs):
calls.append(("static", path, kwargs))
return _result("Schema Validation")
def rubric(path, **kwargs):
calls.append(("rubric", path, kwargs))
return _result("Rubric Evaluation")
monkeypatch.setattr(commands, "run_validation", validation)
monkeypatch.setattr(commands, "run_rubric_eval", rubric)
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
result = CliRunner().invoke(cli, ["tier1", str(skill)])
assert result.exit_code == 0, result.output
assert [call[0] for call in calls] == ["static", "rubric"]
assert "dependency" in calls[0][2]["checks"].split(",")
assert calls[0][2]["use_llm"] is True
assert calls[0][2]["llm_verify"] is True
def test_keyless_tier1_reports_unrun_llm_scope(skill, tmp_path):
result = CliRunner().invoke(
cli,
["tier1", str(skill), "--checks", "schema", "-r", "json", "-o", str(tmp_path / "reports")],
)
assert result.exit_code == 0, result.output
workflow = _report(tmp_path / "reports")["results"][0]["workflow"]
assert workflow["stages"]["llm"]["status"] == "skipped"
assert "LLM" in result.output and "skipped" in result.output
@pytest.mark.parametrize("tier,options", [("tier1", ["--llm"]), ("tier2", [])])
def test_required_provider_failure_is_a_configuration_error(tier, options, skill, tmp_path, monkeypatch):
preparations = []
monkeypatch.setattr("skillevaluator.cli._offer_scanner_install", lambda *_args: preparations.append("prepare"))
result = CliRunner().invoke(cli, [tier, str(skill), *options, "-o", str(tmp_path / "reports")])
assert result.exit_code == 1
assert result.stdout == ""
assert "provider" in result.output.lower()
assert "\n\nFor NVIDIA Build, set:\n export " in result.stderr
assert ("--no-llm" in result.stderr) is (tier == "tier1")
assert "No such command" not in result.output
assert not (tmp_path / "reports").exists()
assert preparations == []
@pytest.mark.parametrize("case", ["unknown_checks", "empty_checks", "missing_manifest"])
def test_tier1_rejects_local_configuration_before_scanner_preparation(case, skill, tmp_path, monkeypatch):
preparations = []
monkeypatch.setattr("skillevaluator.cli._offer_scanner_install", lambda *_args: preparations.append("prepare"))
options = ["--no-llm"]
if case == "unknown_checks":
options += ["--checks", "schema,unknown"]
elif case == "empty_checks":
options += ["--checks", " , "]
else:
(skill / "SKILL.md").unlink()
result = CliRunner().invoke(cli, ["tier1", str(skill), *options, "-o", str(tmp_path / "reports")])
assert result.exit_code == 2, result.output
assert preparations == []
assert not (tmp_path / "reports").exists()
def test_tier1_rejects_missing_llm_extra_before_scanner_preparation(skill, tmp_path, monkeypatch):
import sys
preparations = []
monkeypatch.setattr("skillevaluator.cli._offer_scanner_install", lambda *_args: preparations.append("prepare"))
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
monkeypatch.setitem(sys.modules, "openai", None)
result = CliRunner().invoke(cli, ["tier1", str(skill), "--llm", "-o", str(tmp_path / "reports")])
assert result.exit_code == 1, result.output
assert "Install skillevaluator[llm]" in result.stderr
assert result.stdout == ""
assert preparations == []
assert not (tmp_path / "reports").exists()
@pytest.mark.parametrize("mode", ["keyless", "no_llm", "configured_llm"])
def test_tier1_prepares_scanners_once_after_preflight_and_before_checks(mode, skill, tmp_path, monkeypatch):
from skillevaluator.tier1 import commands
stages = []
monkeypatch.setattr(
"skillevaluator.cli._offer_scanner_install",
lambda checks, skip: stages.append(("prepare", checks, skip)),
)
def validation(_path, **kwargs):
stages.append(("validation", kwargs["checks"], kwargs["skip"]))
return _result("Schema Validation")
def rubric(_path, **_kwargs):
stages.append(("rubric", None, None))
return _result("Rubric Evaluation")
monkeypatch.setattr(commands, "run_validation", validation)
monkeypatch.setattr(commands, "run_rubric_eval", rubric)
options = []
if mode == "no_llm":
options = ["--no-llm"]
elif mode == "configured_llm":
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
options = ["--llm"]
reports = tmp_path / "reports"
result = CliRunner().invoke(
cli,
[
"tier1",
str(skill),
"--checks",
"schema,security",
"--skip",
"security",
*options,
"-r",
"json",
"-o",
str(reports),
],
)
assert result.exit_code == 0, result.output
expected = [("prepare", "schema,security", "security"), ("validation", "schema,security", "security")]
if mode == "configured_llm":
expected.append(("rubric", None, None))
assert stages == expected
assert _report(reports)["results"][0]["workflow"]["tier"] == 1
def test_tier2_combines_intra_and_catalog_comparison_once(skill, tmp_path, monkeypatch):
from skillevaluator.embedding.client import EmbeddingClient
from skillevaluator.embedding.registry import EmbeddingRegistry
from skillevaluator.tier2 import commands
calls = []
catalog = tmp_path / "catalog.json"
def intra(path, **kwargs):
calls.append(("intra", kwargs))
return _result("Context Deduplication")
def inter(path, **kwargs):
calls.append(("inter", kwargs))
return _result("Similarity Check")
monkeypatch.setattr(commands, "run_context_optimization_check", intra)
monkeypatch.setattr(commands, "run_similarity_check", inter)
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
client = EmbeddingClient()
monkeypatch.setattr(client, "embed", lambda _texts: [[1.0, 0.0]])
registry = EmbeddingRegistry(client)
registry.build_from_directory(skill.parent, "skill")
registry.save_catalog(catalog)
result = CliRunner().invoke(cli, ["tier2", str(skill), "--catalog", str(catalog)])
assert result.exit_code == 0, result.output
assert [call[0] for call in calls] == ["intra", "inter"]
assert calls[1][1]["catalog"] == catalog
def test_tier2_invalid_catalog_fails_before_model_work(skill, tmp_path, monkeypatch):
from skillevaluator.tier2 import commands
calls = []
monkeypatch.setattr(commands, "run_context_optimization_check", lambda *_a, **_kw: calls.append("intra") or [])
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
catalog = tmp_path / "bad.json"
catalog.write_text("[]")
result = CliRunner().invoke(cli, ["tier2", str(skill), "--catalog", str(catalog)])
assert result.exit_code != 0
assert not calls
assert "Catalog" in result.output
def test_tier1_static_failure_does_not_become_a_rubric_failure(skill, tmp_path, monkeypatch):
from skillevaluator.tier1 import commands
failure = ValidationResult(validator_name="Schema Validation")
failure.add_error("Invalid schema")
monkeypatch.setattr(commands, "run_validation", lambda *_a, **_kw: [failure])
monkeypatch.setattr(commands, "run_rubric_eval", lambda *_a, **_kw: _result("Rubric Evaluation"))
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
result = CliRunner().invoke(cli, ["tier1", str(skill), "-r", "json", "-o", str(tmp_path / "reports")])
assert result.exit_code == 1
stages = _report(tmp_path / "reports")["workflow"]["stages"]
assert stages["validation"]["status"] == "failed"
assert stages["rubric"]["status"] == "passed"
assert stages["llm"]["status"] == "enabled"
def test_tier2_without_catalog_does_not_manufacture_a_similarity_pass(skill, tmp_path, monkeypatch):
from skillevaluator.tier2 import commands
monkeypatch.setattr(commands, "run_context_optimization_check", lambda *_a, **_kw: _result("Context Deduplication"))
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
result = CliRunner().invoke(
cli,
["tier2", str(skill), "-r", "json", "-o", str(tmp_path / "reports")],
)
assert result.exit_code == 0, result.output
payload = _report(tmp_path / "reports")
assert len(payload["results"]) == 1
assert payload["results"][0]["workflow"]["stages"]["catalog_comparison"]["status"] == "skipped"
assert "--catalog" in result.output
@pytest.mark.parametrize(
("tier", "validator"),
[("tier1", "Schema Validation"), ("tier2", "Context Deduplication")],
)
@pytest.mark.parametrize(
("directory", "target"),
[("parent", "sample"), ("parent", "sample/"), ("skill", ".")],
)
def test_direct_tier_json_names_the_clean_skill_and_records_the_target_as_given(
tier, validator, directory, target, skill, tmp_path, monkeypatch
):
"""A clean result names no file and has no quality score, so the JSON
names the skill from the target path, which records the path as typed.
``.`` has no folder name of its own, so the folder it opens names the skill."""
from skillevaluator.tier1 import commands as tier1_commands
from skillevaluator.tier2 import commands as tier2_commands
options = []
if tier == "tier1":
monkeypatch.setattr(tier1_commands, "run_validation", lambda *_a, **_kw: _result(validator))
options = ["--no-llm"]
else:
monkeypatch.setattr(tier2_commands, "run_context_optimization_check", lambda *_a, **_kw: _result(validator))
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
monkeypatch.chdir({"parent": skill.parent, "skill": skill}[directory])
result = CliRunner().invoke(cli, [tier, target, *options, "-r", "json", "-o", str(tmp_path / "reports")])
assert result.exit_code == 0, result.output
payload = _report(tmp_path / "reports")
assert payload["skills"] == [{"name": "sample", "passed": True, "issue_count": 0}]
assert [item["name"] for contributor in payload["contributors"] for item in contributor["items"]] == ["sample"]
assert payload["target_path"] == str(Path(target))
number = tier.removeprefix("tier")
assert payload["gating"]["tiers"] == {
number: {
"blocking": True,
"validators": [validator],
"validator_policies": [{"validator": validator, "blocking": True}],
}
}
def test_tier_name_collision_requires_explicit_path(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
path = tmp_path / "quality-check"
path.mkdir()
(path / "SKILL.md").write_text(
"---\nname: quality-check\ndescription: Use when asked to check quality.\n"
"metadata:\n author: Example Author <[email protected]>\n---\n# Quality check\n",
)
result = CliRunner().invoke(cli, ["tier1", "./quality-check", "--checks", "schema", "--no-llm"])
assert result.exit_code == 0, result.output
expert = CliRunner().invoke(cli, ["tier1", "quality-check", "--help"])
assert expert.exit_code == 0
assert "--min-score" in expert.output
def test_tier1_validate_alias_stays_inside_tier1(skill, monkeypatch):
from skillevaluator.tier1 import commands
monkeypatch.setattr(commands, "run_validation", lambda *_a, **_kw: _result("Schema Validation"))
result = CliRunner().invoke(cli, ["tier1", "validate", str(skill), "--no-llm"])
assert result.exit_code == 0, result.output
assert "Tier 2" not in result.output
@pytest.mark.parametrize("tier,flag", [("tier1", "--no-llm"), ("tier2", "--catalog"), ("tier3", "--agents")])
def test_tier_help_shows_default_workflow_options(tier, flag):
result = CliRunner().invoke(cli, [tier, "--help"])
assert result.exit_code == 0
assert flag in result.output
@pytest.mark.parametrize("width", [80, 120])
def test_tier3_help_keeps_option_descriptions_visible(width):
import click
command = cli.commands["tier3"]
output = render_help(command.workflow, click.Context(command.workflow), width=width)
normalized = " ".join(output.split())
assert "Run an extra agent-only execution of" in normalized
assert "first staged task before the full" in normalized
assert "matrix [default: disabled]" in normalized
assert "Per-agent model override" in normalized
def test_tier2_rejects_linked_target_before_provider_resolution(skill, tmp_path):
link = tmp_path / "linked-skill"
link.symlink_to(skill, target_is_directory=True)
result = CliRunner().invoke(cli, ["tier2", str(link)])
assert result.exit_code == 2
assert "symlink" in result.output
def test_tier2_rejects_catalog_report_collision_before_reading_or_overwriting(skill, tmp_path):
catalog = tmp_path / "skillevaluator-tier2.json"
catalog.write_text("keep this source unchanged")
result = CliRunner().invoke(cli, ["tier2", str(skill), "--catalog", str(catalog), "-o", str(tmp_path)])
assert result.exit_code == 2
assert "conflicts" in result.output
assert catalog.read_text() == "keep this source unchanged"
def test_full_body_without_catalog_is_rejected(skill):
result = CliRunner().invoke(cli, ["tier2", str(skill), "--full-body"])
assert result.exit_code == 2
assert "--full-body requires --catalog" in result.output
def test_tier1_invalid_provider_is_not_silently_treated_as_keyless(skill, monkeypatch):
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "misspelled-provider")
result = CliRunner().invoke(cli, ["tier1", str(skill), "--checks", "schema"])
assert result.exit_code != 0
assert "provider" in result.output.lower()
static = CliRunner().invoke(cli, ["tier1", str(skill), "--checks", "schema", "--no-llm", "-r", "cli"])
assert static.exit_code == 0, static.output
def test_missing_target_and_unknown_workflow_flag_are_errors(tmp_path):
for tier in ("tier1", "tier2", "tier3"):
missing = CliRunner().invoke(cli, [tier, str(tmp_path / "missing")])
assert missing.exit_code == 2
assert "does not exist" in missing.output
bad_flag = CliRunner().invoke(cli, [tier, "--unknown-option"])
assert bad_flag.exit_code == 2
assert "No such option" in bad_flag.output