Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Verification rationale stays visible without changing finding gates or exports."""
from __future__ import annotations
import copy
import json
import re
from html.parser import HTMLParser
from pathlib import Path
from unittest.mock import patch
import pytest
from skillevaluator.models import Finding, Severity, ValidationResult
from skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter, MarkdownReporter
from skillevaluator.reporting.base import passes_required_gate
from skillevaluator.validators.policy import ValidationPolicy, apply_policy
from skillevaluator.validators.security import SecurityValidator
_SGR = re.compile(r"\x1b\[[0-9;]*m")
_CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]")
class _Text(HTMLParser):
def __init__(self) -> None:
super().__init__()
self.parts: list[str] = []
def handle_data(self, data: str) -> None:
self.parts.append(data)
def _text(fragment: str) -> str:
parser = _Text()
parser.feed(fragment)
return " ".join(" ".join(parser.parts).split())
def _finding(**overrides) -> Finding:
fields = {
"category": "SECURITY",
"severity": Severity.HIGH,
"check_name": "example-check",
"message": "Review the artifact operation",
"file_path": "SKILL.md",
"line_number": 10,
"line_content": "build artifact",
"suggestion": "Review the operation and its context.",
}
return Finding(**(fields | overrides))
def _result(finding: Finding) -> ValidationResult:
result = ValidationResult(validator_name="SECURITY", validator_description="Security review")
result.add_finding(finding)
return result
def _render(format_name: str, result: ValidationResult) -> tuple[str, str]:
if format_name == "html":
output = HTMLReporter(include_timestamp=False, explicit_skill_name="example-skill").render(result)
blocks = re.findall(r'<div class="llm-verification">(.*?)</div>', output, re.DOTALL)
return output, _text(" ".join(blocks))
if format_name == "markdown":
output = MarkdownReporter(include_timestamp=False, include_details=False).render(result)
return output, _text(output)
output = CLIReporter().render(result)
# Rich adds SGR styling in terminal/color environments. Preserve every other
# control byte for the safety assertion instead of stripping arbitrary ANSI.
plain = _SGR.sub("", output)
assert _CONTROL.search(plain) is None, repr(plain)
return output, " ".join(plain.split())
@pytest.mark.parametrize("format_name", ["html", "markdown", "cli"])
@pytest.mark.parametrize(
("verdict", "confidence", "severity", "passed", "verdict_label", "effect"),
[
("false_positive", "medium", Severity.HIGH, False, "False positive", "Reported severity: HIGH"),
("uncertain", "medium", Severity.HIGH, False, "Uncertain", "Reported severity: HIGH"),
("true_positive", "high", Severity.HIGH, False, "True positive", "Reported severity: HIGH"),
(
"false_positive",
"high",
Severity.INFO,
True,
"False positive",
"Verifier recorded a downgrade to INFO; reported severity: INFO",
),
],
)
def test_verified_finding_shows_reasoning_and_actual_gate(
tmp_path: Path,
format_name: str,
verdict: str,
confidence: str,
severity: Severity,
passed: bool,
verdict_label: str,
effect: str,
) -> None:
"""Dropping metadata in a reporter or lowering the confidence threshold breaks this."""
finding = _finding()
result = _result(finding)
reasoning = "The reference points to generated build artifacts."
with patch("skillevaluator.inference.FindingVerifier") as verifier:
verifier.return_value.verify.return_value = {
0: {"verdict": verdict, "confidence": confidence, "rationale": reasoning}
}
SecurityValidator(verify_llm=True)._verify_findings_with_llm(result, tmp_path)
before = copy.deepcopy(result)
exported_before = JSONReporter(include_timestamp=False).render(result)
output, visible = _render(format_name, result)
assert "LLM verification:" in visible
assert verdict_label in visible
assert f"{confidence} confidence" in visible
assert reasoning in visible
assert effect in visible
if not passed:
assert "only high-confidence false positives are downgraded to INFO" in visible
assert finding.severity == severity
assert result.passed is passed
assert passes_required_gate(result) is passed
assert result == before
assert JSONReporter(include_timestamp=False).render(result) == exported_before
assert json.loads(exported_before)["findings"][0]["metadata"]["llm_reasoning"] == reasoning
assert finding.message in output
if format_name == "markdown":
assert finding.suggestion in MarkdownReporter(include_timestamp=False).render(result)
else:
assert finding.suggestion in output
if format_name == "html":
# Both the skill and validator views show the explanation, including advisory INFO cards.
assert output.count('<div class="llm-verification">') == 2
@pytest.mark.parametrize("format_name", ["html", "markdown", "cli"])
@pytest.mark.parametrize(
"metadata",
[
{},
{"llm_reasoning": "Unverified metadata"},
{"llm_verdict": ["false_positive"], "llm_confidence": "medium", "llm_reasoning": "Invalid verdict"},
{"llm_verdict": "unexpected", "llm_confidence": "medium", "llm_reasoning": "Invalid verdict"},
{"llm_verdict": "false_positive", "llm_confidence": {}, "llm_reasoning": "Invalid confidence"},
{"llm_verdict": "false_positive", "llm_confidence": "certain", "llm_reasoning": "Invalid confidence"},
{"llm_verdict": "false_positive", "llm_confidence": "medium", "llm_reasoning": {"text": "Invalid reason"}},
],
)
def test_missing_or_invalid_verification_metadata_adds_no_verdict(format_name: str, metadata: dict) -> None:
result = _result(_finding(metadata=metadata))
output, visible = _render(format_name, result)
assert "LLM verification:" not in visible
assert '<div class="llm-verification">' not in output
assert "Reported severity" not in visible
assert not result.passed
@pytest.mark.parametrize("format_name", ["html", "markdown", "cli"])
def test_verification_reasoning_is_inert_and_bounded(format_name: str) -> None:
prefix = '<script>alert("review")</script> [link=https://example.test]text[/link] | @team'
reason = prefix + "\x1b[31m\n" + "x" * 10_000 + "UNBOUNDED_TAIL"
finding = _finding(metadata={"llm_verdict": "false_positive", "llm_confidence": "medium", "llm_reasoning": reason})
output, visible = _render(format_name, _result(finding))
assert "LLM verification:" in visible
assert '<script>alert("review")</script>' in visible
assert "UNBOUNDED_TAIL" not in visible
assert "[reasoning truncated]" in visible
assert len(visible) < 12_000
assert "\x1b" not in visible
assert finding.metadata["llm_reasoning"] == reason
if format_name == "html":
blocks = re.findall(r'<div class="llm-verification">(.*?)</div>', output, re.DOTALL)
assert blocks
assert all("<script>" not in block and "<script>" in block for block in blocks)
elif format_name == "markdown":
assert "[link=" in output
assert "|" in output
assert "@team" in output
assert len([line for line in output.splitlines() if line.startswith("|")]) == 3
else:
assert "[link=https://example.test]text[/link]" in visible
def test_html_groups_same_verification_decision_with_distinct_reasoning() -> None:
"""Per-occurrence prose must not turn one repeated issue into many skill cards."""
result = ValidationResult(validator_name="SECURITY", validator_description="Security review")
reasons = ["First occurrence is an artifact.", "Second occurrence is generated.", "Third occurrence is a pointer."]
for line_number, reasoning in zip((10, 20, 30), reasons, strict=True):
result.add_finding(
_finding(
line_number=line_number,
metadata={"llm_verdict": "false_positive", "llm_confidence": "medium", "llm_reasoning": reasoning},
)
)
output, _ = _render("html", result)
skill_view, validator_view = output.split('id="results-by-skill"', 1)[1].split('id="results-by-validator"', 1)
skill_text = _text(skill_view)
validator_blocks = re.findall(r'<div class="llm-verification">(.*?)</div>', validator_view, re.DOTALL)
validator_text = _text(" ".join(validator_blocks))
assert skill_view.count('<div class="llm-verification">') == 1
assert "3 occurrences" in skill_text
assert "Lines: 10, 20, 30" in skill_text
assert reasons[0] in skill_text
assert all(reason not in skill_text for reason in reasons[1:])
assert len(validator_blocks) == 3
assert all(reason in validator_text for reason in reasons)
exported = json.loads(JSONReporter(include_timestamp=False).render(result))
assert [finding["metadata"]["llm_reasoning"] for finding in exported["findings"]] == reasons
assert [finding["line_number"] for finding in exported["findings"]] == [10, 20, 30]
assert not passes_required_gate(result)
def test_html_keeps_different_verification_decisions_separate() -> None:
result = _result(
_finding(
metadata={
"llm_verdict": "false_positive",
"llm_confidence": "medium",
"llm_reasoning": "First occurrence uses an artifact reference.",
}
)
)
result.add_finding(
_finding(
line_number=20,
metadata={
"llm_verdict": "uncertain",
"llm_confidence": "medium",
"llm_reasoning": "Second occurrence lacks enough context.",
},
)
)
output, visible = _render("html", result)
assert output.count('<div class="llm-verification">') == 4
assert "First occurrence uses an artifact reference." in visible
assert "Second occurrence lacks enough context." in visible
assert "False positive" in visible
assert "Uncertain" in visible
@pytest.mark.parametrize("format_name", ["html", "markdown", "cli"])
def test_verifier_and_reporter_share_downgrade_policy_with_coverage_exemption(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, format_name: str
) -> None:
"""A changed rule must drive both actual downgrade behavior and its explanation."""
from skillevaluator import finding_verification_policy
monkeypatch.setattr(finding_verification_policy, "DOWNGRADE_CONFIDENCE", "medium")
monkeypatch.setattr(finding_verification_policy, "DOWNGRADE_SEVERITY", Severity.LOW)
coverage = _finding(
check_name="skillspector_reference_caveats", severity=Severity.MEDIUM, message="Coverage remains advisory"
)
result = _result(coverage)
result.add_finding(_finding(message="Possible artifact reference"))
result.add_finding(_finding(message="Operation needs more context"))
with patch("skillevaluator.inference.FindingVerifier") as verifier:
verifier.return_value.verify.return_value = {
0: {"verdict": "false_positive", "confidence": "medium", "rationale": "Artifact reference."},
1: {"verdict": "uncertain", "confidence": "medium", "rationale": "More context needed."},
}
SecurityValidator(verify_llm=True)._verify_findings_with_llm(result, tmp_path)
assert result.findings[1].severity == Severity.LOW
assert result.findings[2].severity == Severity.HIGH
assert coverage.severity == Severity.MEDIUM
assert "llm_verdict" not in coverage.metadata
assert not result.is_incomplete
before = copy.deepcopy(result)
_, visible = _render(format_name, result)
assert "Verifier recorded a downgrade to LOW; reported severity: LOW" in visible
assert "only medium-confidence false positives are downgraded to LOW by verification" in visible
assert not passes_required_gate(result)
assert result == before
@pytest.mark.parametrize("format_name", ["html", "markdown", "cli"])
def test_policy_can_restore_blocking_severity_after_verifier_downgrade(tmp_path: Path, format_name: str) -> None:
"""Policy remapping must not hide the verifier's recorded downgrade or the final gate."""
finding = _finding()
result = _result(finding)
with patch("skillevaluator.inference.FindingVerifier") as verifier:
verifier.return_value.verify.return_value = {
0: {"verdict": "false_positive", "confidence": "high", "rationale": "A documented placeholder."}
}
SecurityValidator(verify_llm=True)._verify_findings_with_llm(result, tmp_path)
assert result.passed
apply_policy([result], ValidationPolicy(severity_overrides={"SECURITY.example-check": Severity.HIGH}))
before = copy.deepcopy(result)
_, visible = _render(format_name, result)
assert "Verifier recorded a downgrade to INFO; reported severity: HIGH" in visible
assert "no verification downgrade recorded" not in visible
assert "high confidence" in visible
assert finding.metadata["downgraded"] is True
assert finding.severity == Severity.HIGH
assert not passes_required_gate(result)
assert result == before
@pytest.mark.parametrize("force_color", [False, True])
def test_cli_verification_is_visible_with_or_without_color(monkeypatch: pytest.MonkeyPatch, force_color: bool) -> None:
"""Rich's own styling must not obscure semantic assertions or permit terminal controls."""
monkeypatch.setenv("TERM", "xterm-256color" if force_color else "dumb")
monkeypatch.setenv("COLUMNS", "200")
monkeypatch.delenv("TTY_COMPATIBLE", raising=False)
if force_color:
monkeypatch.setenv("FORCE_COLOR", "1")
monkeypatch.delenv("NO_COLOR", raising=False)
else:
monkeypatch.setenv("NO_COLOR", "1")
monkeypatch.delenv("FORCE_COLOR", raising=False)
hostile_sgr = "\x1b[38;2;1;2;3m"
reasoning = (
'<script>alert("review")</script> [link=https://example.test]text[/link] '
f"{hostile_sgr}\x1b[2J\x1b]8;;https://example.test\x07click\x1b]8;;\x07"
)
result = _result(
_finding(metadata={"llm_verdict": "false_positive", "llm_confidence": "medium", "llm_reasoning": reasoning})
)
output, visible = _render("cli", result)
# Check unmodified output: normalizing SGR would otherwise hide a leaked input style.
assert hostile_sgr not in output
assert ("\x1b[" in output) is force_color
assert "False positive (medium confidence)" in visible
assert "only high-confidence false positives are downgraded to INFO" in visible
assert '<script>alert("review")</script>' in visible
assert "[link=https://example.test]text[/link]" in visible
assert "[38;2;1;2;3m" in visible
assert "\x1b[2J" not in output
assert "\x1b]8;" not in output
assert not passes_required_gate(result)