Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Execution diagnostics remain visible alongside genuine content findings."""
from __future__ import annotations
import json
import threading
from dataclasses import asdict
from html.parser import HTMLParser
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from click.testing import CliRunner
from skillevaluator.cli import cli
from skillevaluator.models import Finding, Severity, ValidationResult
class _VisibleHTMLText(HTMLParser):
"""Exclude diagnostics present only in the report's embedded JSON."""
def __init__(self) -> None:
super().__init__()
self.hidden = 0
self.parts: list[str] = []
def handle_starttag(self, tag, _attrs) -> None:
if tag in {"script", "style"}:
self.hidden += 1
def handle_endtag(self, tag) -> None:
if tag in {"script", "style"}:
self.hidden -= 1
def handle_data(self, data) -> None:
if not self.hidden:
self.parts.append(data)
@pytest.mark.parametrize("outcome", ["mixed", "errors_only", "finding_only"])
def test_tier2_cli_reports_preserve_analysis_errors_with_findings(tmp_path: Path, monkeypatch, outcome: str) -> None:
"""Exercise collection, clustering, SDK requests and all reports over loopback HTTP."""
requests: list[str] = []
class AnalysisHandler(BaseHTTPRequestHandler):
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append(self.path)
if self.path == "/v1/embeddings":
payload = {
"data": [
{"index": index, "embedding": [float(index // 2 == dimension) for dimension in range(2)]}
for index in range(len(body["input"]))
]
}
elif self.path == "/v1/chat/completions":
first_cluster = "Section A" in body["messages"][-1]["content"]
verdict = "DUPLICATE" if first_cluster else "RELATED_BUT_DISTINCT"
if outcome == "errors_only" or (outcome == "mixed" and not first_cluster):
verdict = "INVALID"
payload = {
"id": "local-completion",
"object": "chat.completion",
"created": 0,
"model": body["model"],
"choices": [
{
"index": 0,
"finish_reason": "stop",
"message": {
"role": "assistant",
"content": json.dumps(
{
"verdict": verdict,
"confidence": 0.9,
"reasoning": "Repeated instructions",
"suggestion": "Consolidate the repeated sections",
}
),
},
}
],
}
else:
self.send_error(404)
return
response = json.dumps(payload).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def log_message(self, _format, *_args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), AnalysisHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
skill = tmp_path / "skill"
skill.mkdir()
(skill / "SKILL.md").write_text(
"\n".join(f"## Section {letter}\n{letter.lower() * 200}" for letter in "ABCD"), encoding="utf-8"
)
reports = tmp_path / "reports"
for purpose in ("LLM", "EMBEDDING"):
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_PROVIDER", "openai-compatible")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_BASE_URL", f"http://127.0.0.1:{server.server_port}/v1")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_API_KEY", "local-test-key")
monkeypatch.setenv(f"SKILL_EVAL_{purpose}_MODEL", "local-test-model")
try:
invocation = CliRunner().invoke(cli, ["tier2", str(skill), "-r", "cli,json,html,markdown", "-o", str(reports)])
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
assert invocation.exit_code == 1, invocation.output
assert requests.count("/v1/embeddings") == 1
assert requests.count("/v1/chat/completions") == 2
payload = json.loads((reports / "skillevaluator-tier2.json").read_text(encoding="utf-8"))
result = payload["results"][0]
failed_clusters = {"mixed": 1, "errors_only": 2, "finding_only": 0}[outcome]
# A confirmed policy finding outranks a separate incomplete analyzer.
assert result["status"] == ("incomplete" if outcome == "errors_only" else "failed")
assert result["llm_analysis"]["clusters_total"] == 2
assert result["llm_analysis"]["clusters_failed"] == failed_clusters
assert result["llm_analysis"]["clusters_completed"] == 2 - failed_clusters
# Provider failures remain execution evidence rather than duplicate findings.
assert [finding["check_name"] for finding in result["findings"]] == (
[] if outcome == "errors_only" else ["duplicate"]
)
# Analysis errors count as issues beside any finding, so an errors-only run
# never reports zero issues.
assert payload["message_counts"]["independent_errors"] == int(failed_clusters > 0)
# The skipped catalog comparison is a workflow-scope note in every case. It
# stays a raw warning but records scope, not an issue without a finding.
assert any(warning.startswith("Workflow scope: ") for warning in result["legacy"]["warnings"])
assert payload["message_counts"]["independent_warnings"] == 0
markdown_lines = (reports / "skillevaluator-tier2.md").read_text(encoding="utf-8").splitlines()
finding_text = "1 unique finding (1 high) / 1 occurrence / 1 raw record"
assert [line for line in markdown_lines if line.startswith("| Total Issues |")] == [
{
"mixed": f"| Total Issues | {finding_text}; 1 error without a finding |",
"errors_only": "| Total Issues | 1 error without a finding |",
"finding_only": f"| Total Issues | {finding_text} |",
}[outcome]
]
html = _VisibleHTMLText()
html.feed((reports / "skillevaluator-tier2.html").read_text(encoding="utf-8"))
# Earlier logger output must not mask omissions from the actual CLI report.
marker = "SkillEvaluator Validation Results"
assert marker in invocation.output
outputs = {
"cli": invocation.output[invocation.output.index(marker) :],
"html": " ".join(html.parts),
"markdown": (reports / "skillevaluator-tier2.md").read_text(encoding="utf-8"),
}
outputs = {name: " ".join(text.split()) for name, text in outputs.items()}
diagnostic = "LLM analysis did not complete"
expected_diagnostics = int(failed_clusters > 0)
assert {name: text.count(diagnostic) for name, text in outputs.items()} == dict.fromkeys(
outputs, expected_diagnostics
)
if failed_clusters:
for text in outputs.values():
assert f"{failed_clusters} of 2 content clusters" in text
assert "structured-output support" in text
duplicate_message = "Duplicate content found within SKILL.md:"
# Markdown intentionally repeats the finding in its reconciliation table,
# its findings table, and its details.
expected_findings = {"cli": 1, "html": 1, "markdown": 3}
assert {name: text.count(duplicate_message) for name, text in outputs.items()} == {
name: count if outcome != "errors_only" else 0 for name, count in expected_findings.items()
}
@pytest.mark.parametrize("prefixed", [False, True])
def test_additional_errors_excludes_only_existing_finding_messages_without_mutation(prefixed: bool) -> None:
from skillevaluator.reporting.base import additional_errors
result = ValidationResult(validator_name="Context Deduplication")
result.add_finding(Finding("DUPLICATE", Severity.HIGH, "duplicate", "Repeated instructions", "SKILL.md"))
diagnostic = "LLM analysis failed while reviewing Repeated instructions"
result.add_error(diagnostic)
if prefixed:
merged = ValidationResult(validator_name="Context Deduplication")
merged.merge_with_prefix(result, "sample-skill")
result = merged
diagnostic = f"[sample-skill] {diagnostic}"
before = asdict(result)
assert additional_errors(result) == [diagnostic]
assert asdict(result) == before
@pytest.mark.parametrize("prefixed", [False, True])
def test_additional_warnings_excludes_only_existing_finding_messages_without_mutation(prefixed: bool) -> None:
from skillevaluator.reporting.base import additional_warnings
result = ValidationResult(validator_name="Code Risk")
result.add_finding(Finding("CODE", Severity.LOW, "style", "Prefer explicit encoding", "run.py", line_number=4))
diagnostic = "bandit not installed."
result.add_warning(diagnostic)
if prefixed:
merged = ValidationResult(validator_name="Code Risk")
merged.merge_with_prefix(result, "sample-skill")
result = merged
diagnostic = f"[sample-skill] {diagnostic}"
before = asdict(result)
assert len(result.warnings) == 2
assert additional_warnings(result) == [diagnostic]
assert asdict(result) == before