Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Tier 2 command wrappers preserve useful failure diagnostics."""
from __future__ import annotations
import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from click.testing import CliRunner
from skillevaluator.cli import cli
from skillevaluator.models.result import Finding, Severity, ValidationResult
from skillevaluator.tier2 import commands
@pytest.mark.parametrize("option", ["max_entries", "max_scalar_comparisons"])
@pytest.mark.parametrize("value", [0, -1, True, False, 1.5, "private-input", None])
def test_similarity_check_rejects_invalid_limit_options_before_construction(
tmp_path: Path, monkeypatch, option: str, value: object
) -> None:
def unexpected_construction(**_kwargs):
pytest.fail("Invalid limit options must be rejected before constructing the validator")
monkeypatch.setattr(commands, "SimilarityValidator", unexpected_construction)
with pytest.raises(ValueError, match=option) as failure:
commands.run_similarity_check(tmp_path, **{option: value})
assert f"--{option.replace('_', '-')}" in str(failure.value)
assert "private-input" not in str(failure.value)
assert "provider" not in str(failure.value)
def test_similarity_check_rejects_entry_limit_above_hard_cap(tmp_path: Path) -> None:
with pytest.raises(ValueError, match=r"max_entries.*--max-entries"):
commands.run_similarity_check(tmp_path, max_entries=5_001)
@pytest.mark.parametrize("during_construction", [False, True])
@pytest.mark.parametrize("error_type", [RuntimeError, ValueError])
def test_similarity_check_wraps_unexpected_validator_exceptions(
tmp_path: Path,
monkeypatch,
during_construction: bool,
error_type: type[Exception],
) -> None:
class BrokenSimilarityValidator:
def __init__(self, **_kwargs):
if during_construction:
raise error_type("provider body contains private-token")
def validate(self, _content_path: Path):
raise error_type("provider body contains private-token")
monkeypatch.setattr(commands, "SimilarityValidator", BrokenSimilarityValidator)
(result,) = commands.run_similarity_check(tmp_path, catalog=tmp_path / "catalog.json")
assert result.validator_name == "Similarity Check"
assert result.validator_description == "Tier 2 check"
assert result.passed is False
assert result.status == "incomplete"
assert result.findings == []
assert "unexpected error" in result.errors[0]
assert "private-token" not in result.errors[0]
@pytest.mark.parametrize("during_construction", [False, True])
def test_context_check_preserves_incomplete_contract_for_unexpected_errors(
tmp_path: Path, monkeypatch, during_construction: bool
) -> None:
class BrokenValidator:
def __init__(self, **_kwargs):
if during_construction:
raise RuntimeError("private provider details")
def validate(self, _path):
raise RuntimeError("private provider details")
monkeypatch.setattr(commands, "IntraSkillValidator", BrokenValidator)
(result,) = commands.run_context_optimization_check(tmp_path)
assert result.status == "incomplete"
assert not result.passed
assert result.findings == []
assert "private provider details" not in result.errors[0]
def test_detected_duplicate_remains_failed_not_incomplete(tmp_path: Path, monkeypatch) -> None:
result = ValidationResult(validator_name="Context Deduplication")
result.add_finding(Finding("DUPLICATE", Severity.HIGH, "duplicate", "Repeated instructions", "SKILL.md"))
class DuplicateValidator:
def __init__(self, **_kwargs):
pass
def validate(self, _path):
return result
monkeypatch.setattr(commands, "IntraSkillValidator", DuplicateValidator)
(actual,) = commands.run_context_optimization_check(tmp_path)
assert actual.status == "failed"
assert actual.findings[0].category == "DUPLICATE"
def test_work_limit_is_incomplete_in_cli_and_reports(tmp_path: Path, monkeypatch) -> None:
"""A skill over a Tier 2 work limit reads INCOMPLETE, never a failed duplicate scan."""
from skillevaluator.deduplication.intra_skill import intra_skill_validator
from skillevaluator.reporting import BenchmarkReporter
monkeypatch.setattr(intra_skill_validator, "CONTENT_DEDUP_MAX_CHUNKS", 1)
skill = tmp_path / "skill"
skill.mkdir()
(skill / "SKILL.md").write_text("## Section A\n" + "a" * 200 + "\n## Section B\n" + "b" * 200)
reports = tmp_path / "reports"
for purpose in ("LLM", "EMBEDDING"):
# The limit trips before any provider request; port 9 is never contacted.
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_PROVIDER", "openai-compatible")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_BASE_URL", "http://127.0.0.1:9/v1")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_API_KEY", "local-test-key")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_MODEL", "local-test-model")
invocation = CliRunner().invoke(cli, ["tier2", str(skill), "-r", "cli,json,markdown", "-o", str(reports)])
assert invocation.exit_code == 1, invocation.output
assert "context-deduplication did not complete" in invocation.output
payload = json.loads((reports / "skillevaluator-tier2.json").read_text())
assert payload["overall_status"] == "incomplete"
result = payload["results"][0]
assert result["status"] == "incomplete"
assert result["incomplete_scans"] == ["context-deduplication"]
assert result["findings"] == []
assert result["summary"]["critical_count"] == 0
assert result["work_limit"] == {"check_name": "chunk_count_limit", "file_path": "SKILL.md", "actual": 2, "limit": 1}
markdown = (reports / "skillevaluator-tier2.md").read_text()
assert "**Status:** ⚠️ INCOMPLETE" in markdown
assert "Tier 2 scan incomplete: the skill produced more than 1 content chunks" in markdown
benchmark = BenchmarkReporter(include_timestamp=False).render_all(commands.run_context_optimization_check(skill))
assert (
"| Tier 2 | Semantic deduplication | **INCOMPLETE** | Missing trustworthy evidence from context-deduplication |"
in benchmark
)
assert "**FAILED**" not in benchmark
def test_http_410_is_incomplete_in_real_cli_and_reports(tmp_path: Path, monkeypatch) -> None:
"""Exercise the actual SDK against a local HTTP service, without external calls."""
requests = []
class RetiredModelHandler(BaseHTTPRequestHandler):
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append((self.path, body))
response = json.dumps(
{"error": {"message": "Retired model: provider-private-token", "type": "model_retired"}}
).encode()
self.send_response(410)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def log_message(self, _format, *_args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), RetiredModelHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
skill = tmp_path / "skill"
skill.mkdir()
(skill / "SKILL.md").write_text("## Section A\n" + "a" * 200 + "\n## Section B\n" + "b" * 200)
reports = tmp_path / "reports"
for purpose in ("LLM", "EMBEDDING"):
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_PROVIDER", "openai-compatible")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_BASE_URL", f"http://127.0.0.1:{server.server_port}/v1")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_API_KEY", "local-test-key")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_MODEL", "local-test-model")
try:
invocation = CliRunner().invoke(cli, ["tier2", str(skill), "-o", str(reports)])
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
assert invocation.exit_code == 1, invocation.output
assert len(requests) == 1
assert requests[0][0] == "/v1/embeddings"
assert requests[0][1]["model"] == "local-test-model"
assert "INCOMPLETE" in invocation.output
payload = json.loads((reports / "skillevaluator-tier2.json").read_text())
html = (reports / "skillevaluator-tier2.html").read_text()
assert payload["overall_status"] == "incomplete"
assert payload["overall_passed"] is False
assert payload["results"][0]["status"] == "incomplete"
assert payload["results"][0]["findings"] == []
assert "Incomplete" in html
for output in (invocation.output, json.dumps(payload), html):
assert "provider-private-token" not in output
@pytest.mark.parametrize("chat_status", [200, 400, 410, 429])
def test_chat_http_diagnostics_after_successful_embedding(tmp_path: Path, monkeypatch, chat_status: int) -> None:
"""Run collection, SDK requests, clustering, analysis and reports over loopback HTTP."""
requests = []
class ChatHandler(BaseHTTPRequestHandler):
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append((self.path, body))
if self.path == "/v1/embeddings":
status = 200
payload = {
"data": [
{"index": index, "embedding": [float(dimension == index // 2) for dimension in range(6)]}
for index in range(len(body["input"]))
]
}
elif self.path == "/v1/chat/completions":
status = chat_status
if status == 200:
payload = {
"id": "local-completion",
"object": "chat.completion",
"created": 0,
"model": body["model"],
"choices": [
{
"index": 0,
"finish_reason": "stop",
"message": {
"role": "assistant",
"content": json.dumps(
{
"verdict": "RELATED_BUT_DISTINCT",
"confidence": 0.9,
"reasoning": "Separate tasks",
"suggestion": "Keep both",
}
),
},
}
],
}
else:
payload = {
"error": {
"message": "provider-private-token with private input content",
"code": "unsupported_parameter" if status == 400 else "model_retired",
"param": "temperature",
"type": "provider-private-token",
}
}
else:
status, payload = 404, {}
response = json.dumps(payload).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response)))
self.send_header("x-should-retry", "false")
self.end_headers()
self.wfile.write(response)
def log_message(self, _format, *_args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), ChatHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
skill = tmp_path / "skill"
skill.mkdir()
(skill / "SKILL.md").write_text(
"\n".join(f"## Section {letter}\n{letter.lower() * 200}" for letter in "ABCDEFGHIJKL")
)
reports = tmp_path / "reports"
for purpose in ("LLM", "EMBEDDING"):
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_PROVIDER", "openai-compatible")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_BASE_URL", f"http://127.0.0.1:{server.server_port}/v1")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_API_KEY", "local-test-key")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_MODEL", f"local-{purpose.lower()}-model")
monkeypatch.setenv("SKILL_EVAL_LLM_MAX_RETRIES", "2")
monkeypatch.setenv("SKILL_EVAL_LLM_RETRY_BASE_DELAY", "0")
monkeypatch.setenv("SKILL_EVAL_LLM_RETRY_MAX_DELAY", "0")
try:
invocation = CliRunner().invoke(cli, ["tier2", str(skill), "-o", str(reports)])
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
assert requests[0][0] == "/v1/embeddings"
chat_requests = [body for path, body in requests if path == "/v1/chat/completions"]
assert len(chat_requests) == 6 * (3 if chat_status == 429 else 1)
assert all(body["model"] == "local-llm-model" and body["stream"] is False for body in chat_requests)
payload = json.loads((reports / "skillevaluator-tier2.json").read_text())
html = (reports / "skillevaluator-tier2.html").read_text()
result = payload["results"][0]
assert result["findings"] == []
analysis = result["llm_analysis"]
assert analysis["provider"] == "openai-compatible"
assert analysis["model"] == "local-llm-model"
assert analysis["clusters_total"] == 6
if chat_status == 200:
assert invocation.exit_code == 0, invocation.output
assert payload["overall_status"] == "passed"
assert analysis["clusters_completed"] == 6
assert analysis["clusters_failed"] == 0
assert analysis["failures"] == []
else:
assert invocation.exit_code == 1, invocation.output
assert payload["overall_status"] == "incomplete"
assert result["status"] == "incomplete"
assert analysis["clusters_failed"] == 6
assert analysis["clusters_completed"] == 0
errors = result["legacy"]["errors"]
assert len(errors) == len(analysis["failures"]) == 1
assert analysis["failures"][0]["count"] == 6
for output in (invocation.output, json.dumps(payload), html):
assert f"HTTP {chat_status}" in output
assert "6 of 6 content clusters" in output
assert "local-llm-model" in output
assert "provider-private-token" not in output
assert "private input content" not in output
assert "local-test-key" not in output
if chat_status == 400:
assert "code=unsupported_parameter" in errors[0]
assert "param=temperature" in errors[0]
elif chat_status == 410:
assert "SKILL_EVAL_LLM_MODEL" in errors[0]
elif chat_status == 429:
assert "quota" in errors[0]
def test_chat_refusal_is_advisory_in_real_cli_and_reports(tmp_path: Path, monkeypatch) -> None:
"""A content-filtered chat reply is an advisory finding, not a blocking incomplete scan."""
requests = []
class RefusingChatHandler(BaseHTTPRequestHandler):
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append((self.path, body))
if self.path == "/v1/embeddings":
payload = {
"data": [
{"index": index, "embedding": [float(dimension == index // 2) for dimension in range(4)]}
for index in range(len(body["input"]))
]
}
else:
payload = {
"id": "local-completion",
"object": "chat.completion",
"created": 0,
"model": body["model"],
"choices": [
{
"index": 0,
"finish_reason": "content_filter",
"message": {"role": "assistant", "content": "provider-private-token"},
}
],
}
response = json.dumps(payload).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def log_message(self, _format, *_args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), RefusingChatHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
skill = tmp_path / "skill"
skill.mkdir()
(skill / "SKILL.md").write_text("\n".join(f"## Section {letter}\n{letter.lower() * 200}" for letter in "ABCD"))
reports = tmp_path / "reports"
for purpose in ("LLM", "EMBEDDING"):
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_PROVIDER", "openai-compatible")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_BASE_URL", f"http://127.0.0.1:{server.server_port}/v1")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_API_KEY", "local-test-key")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_MODEL", f"local-{purpose.lower()}-model")
try:
invocation = CliRunner().invoke(cli, ["tier2", str(skill), "-o", str(reports)])
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
assert invocation.exit_code == 0, invocation.output
# A refusal is not transient, so the same request is not retried.
assert len([path for path, _body in requests if path == "/v1/chat/completions"]) == 2
payload = json.loads((reports / "skillevaluator-tier2.json").read_text())
result = payload["results"][0]
assert payload["overall_status"] == "passed"
assert result["status"] == "passed"
assert [(finding["check_name"], finding["severity"]) for finding in result["findings"]] == [
("llm_refused", "medium"),
("llm_refused", "medium"),
]
analysis = result["llm_analysis"]
assert (analysis["clusters_completed"], analysis["clusters_refused"], analysis["clusters_failed"]) == (0, 2, 0)
assert all(finding["metadata"]["llm_model"] == "local-llm-model" for finding in result["findings"])
html = (reports / "skillevaluator-tier2.html").read_text()
for output in (invocation.output, json.dumps(payload), html):
assert "provider-private-token" not in output
assert "local-test-key" not in output