Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Completeness evidence survives validation, merging, and report rendering."""
import json
from pathlib import Path
import pytest
from rich.text import Text
from skillevaluator.models.result import Finding, Severity, ValidationResult
from skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter, MarkdownReporter
from skillevaluator.reporting.retained import retained_results
from skillevaluator.skillspector_diagnostics import (
completeness_record,
merge_completeness,
sanitize_completeness,
)
from skillevaluator.validators.policy import apply_policy, default_policy, load_policy_file
from skillevaluator.validators.security import SecurityValidator
def partial_result() -> ValidationResult:
payload = json.loads((Path(__file__).parents[1] / "fixtures" / "skillspector-2.13.3-safe-no-llm.json").read_text())
payload["risk_assessment"]["recommendation"] = "CAUTION"
payload["analysis_completeness"].update(
is_complete=False,
status="partial",
ledger_exceptions=[
{
"fatal": False,
"path": "SKILL.md",
"start_line": 8,
"phase": "reference_resolution",
"reason_code": "reference_missing",
"analyzers": [],
}
],
)
result = ValidationResult(validator_name="Security Scan")
assert SecurityValidator._validate_skillspector_report(payload, 0, False, "skillspector", result)
return result
@pytest.mark.parametrize("terminal", ["dumb", "xterm-256color"])
def test_partial_evidence_is_retained_without_turning_it_into_a_pass(monkeypatch, terminal):
monkeypatch.setenv("TERM", terminal)
monkeypatch.delenv("NO_COLOR", raising=False)
result = partial_result()
apply_policy([result], default_policy())
assert result.status == "incomplete"
assert not result.passed
assert result.findings == [] # Coverage diagnostics are not security defects.
for reporter in (CLIReporter(), HTMLReporter(), MarkdownReporter(), JSONReporter()):
output = reporter.render(result)
if isinstance(reporter, CLIReporter):
output = Text.from_ansi(output).plain
assert "reference_missing" in output
assert "SKILL.md:8" in output
data = json.loads(JSONReporter().render(result))
row = data["skillspector_completeness"]["records"][0]["ledger_exceptions"][0]
assert row["path"] == "SKILL.md"
assert row["start_line"] == 8
assert row["phase"] == "reference_resolution"
def test_merge_preserves_both_stages_and_skill_identity():
static, llm = partial_result(), partial_result()
llm.metadata["skillspector_completeness"]["records"][0]["stage"] = "skillspector-llm"
static.merge(llm)
folder = ValidationResult()
folder.merge_with_prefix(static, "contact-skill")
records = json.loads(JSONReporter().render(folder))["skillspector_completeness"]["records"]
assert [record["stage"] for record in records] == ["skillspector", "skillspector-llm"]
assert {record["skill"] for record in records} == {"contact-skill"}
assert folder.is_incomplete
def test_policy_recalculation_keeps_completeness_error_and_retained_report_evidence(tmp_path):
result = partial_result()
result.add_structured_finding(
Finding(
category="PII",
check_name="ownership_emails",
severity=Severity.HIGH,
message="contact",
file_path="SKILL.md",
)
)
policy_path = tmp_path / "policy.yaml"
policy_path.write_text("severity_overrides:\n PII.ownership_emails: info\n")
apply_policy([result], load_policy_file(policy_path))
assert result.status == "incomplete"
assert any("reference_missing" in error for error in result.errors)
retained = retained_results(json.loads(JSONReporter().render_all([result])))[0]
assert retained.metadata["skillspector_completeness"] == result.metadata["skillspector_completeness"]
assert "reference_missing" in HTMLReporter().render(retained)
def test_diagnostic_text_and_lists_are_bounded_redacted_and_idempotent():
secret = "ghp_" + "a" * 36
raw = {
"ledger_exceptions": [
{
"path": f"references/{secret}/file.md\n\x1b[31mextra",
"phase": "static",
"reason_code": "opaque_content",
"analyzers": ["analyzer"] * 12,
"start_line": True,
"unexpected": "must not persist",
}
]
* 25,
"limitations": ["password=fictional-password " + "x" * 800] * 10,
}
record = completeness_record(raw, "skillspector")
assert record["omitted_exceptions"] == 5
assert record["omitted_limitations"] == 2
assert record["ledger_exceptions"][0]["omitted_analyzers"] == 4
assert "start_line" not in record["ledger_exceptions"][0]
encoded = json.dumps(record)
assert secret not in encoded and "fictional-password" not in encoded
assert "must not persist" not in encoded and "\\u001b" not in encoded and "\\n" not in encoded
wrapped = {"records": [record], "omitted_records": 0}
assert sanitize_completeness(wrapped) == wrapped
merged = merge_completeness(wrapped, {"records": [record] * 70})
assert len(merged["records"]) == 64
assert merged["omitted_records"] == 7
assert sanitize_completeness(merged) == merged
def test_parser_diagnostic_is_projected_without_source_offsets_and_survives_merge():
raw = {
"ledger_exceptions": [
{
"path": "scripts/admission.mjs",
"phase": "static",
"reason_code": "static_parse_limit",
"analyzers": ["static_patterns_tool_misuse"],
"parser_diagnostic": {
"cause": "runtime_dependent",
"check": "command_string_runtime",
"start_line": 74,
"end_line": 170,
"unchecked_scope": "remaining_input",
"full_input_checked": False,
"source_start_offset": 3958,
"source_end_offset": 8650,
"location_precision": "source_span",
"observed_is_lower_bound": False,
},
}
]
}
record = completeness_record(raw, "skillspector")
diagnostic = record["ledger_exceptions"][0]["parser_diagnostic"]
assert diagnostic == {
"cause": "runtime_dependent",
"check": "command_string_runtime",
"start_line": 74,
"end_line": 170,
"unchecked_scope": "remaining_input",
"full_input_checked": False,
}
encoded = json.dumps(record)
assert "source_start_offset" not in encoded
assert "3958" not in encoded
assert "location_precision" not in encoded
wrapped = {"records": [record], "omitted_records": 0}
assert sanitize_completeness(wrapped) == wrapped
merged = merge_completeness({}, wrapped, prefix="spawn-helper")
assert merged["records"][0]["skill"] == "spawn-helper"
assert merged["records"][0]["ledger_exceptions"][0]["parser_diagnostic"] == diagnostic
assert sanitize_completeness(merged) == merged
record["ledger_exceptions"][0]["parser_diagnostic"]["source_start_offset"] = 3958
saved = sanitize_completeness({"records": [record]})
reprojected = saved["records"][0]["ledger_exceptions"][0]["parser_diagnostic"]
assert "source_start_offset" not in reprojected
@pytest.mark.parametrize(
"diagnostic",
[
"not-a-dict",
True,
{"cause": "runtime_dependent", "start_line": "74", "end_line": 170},
{"cause": True, "check": False, "start_line": -1, "end_line": 10, "full_input_checked": "no"},
{"cause": "runtime_dependent", "start_line": 170, "end_line": 74},
],
ids=["string", "bool", "string-line", "bool-fields", "end-before-start"],
)
def test_malformed_parser_diagnostic_is_dropped_or_stripped_to_safe_fields(diagnostic: object):
record = completeness_record(
{"ledger_exceptions": [{"path": "scripts/helper.mjs", "parser_diagnostic": diagnostic}]},
"skillspector",
)
projected = record["ledger_exceptions"][0].get("parser_diagnostic")
if projected is None:
return
assert "start_line" not in projected
assert "end_line" not in projected
assert projected.get("full_input_checked") is not True
text_keys = {key for key in projected if key not in {"start_line", "end_line", "full_input_checked"}}
assert all(isinstance(projected[key], str) for key in text_keys)
def test_saved_diagnostic_metadata_is_revalidated_at_json_boundary():
result = partial_result()
record = result.metadata["skillspector_completeness"]["records"][0]
record["unexpected_secret"] = "not part of contract"
record["ledger_exceptions"][0]["path"] = "password=fictional-password"
output = JSONReporter().render(result)
assert "fictional-password" not in output
assert "unexpected_secret" not in output
@pytest.mark.parametrize("stage", [[], {}, None, True, 3, "unsupported"])
def test_malformed_saved_stage_cannot_crash_json_or_retained_reports(stage):
result = partial_result()
result.metadata["skillspector_completeness"]["records"][0]["stage"] = stage
data = json.loads(JSONReporter().render_all([result]))
assert data["results"][0]["skillspector_completeness"]["records"][0]["stage"] == "unknown"
# Exercise the independent retained-report input boundary as well.
data["results"][0]["skillspector_completeness"]["records"][0]["stage"] = stage
retained = retained_results(data)[0]
assert retained.is_incomplete
assert retained.metadata["skillspector_completeness"]["records"][0]["stage"] == "unknown"
assert "reference_missing" in HTMLReporter().render(retained)
def test_limitation_only_partial_report_keeps_its_reason_in_the_error():
payload = json.loads((Path(__file__).parents[1] / "fixtures" / "skillspector-2.13.3-safe-no-llm.json").read_text())
payload["risk_assessment"]["recommendation"] = "CAUTION"
payload["analysis_completeness"].update(
is_complete=False,
status="partial",
ledger_exceptions=[],
limitations=["Binary assets were not inspected"],
)
result = ValidationResult(validator_name="Security Scan")
SecurityValidator._validate_skillspector_report(payload, 0, False, "skillspector", result)
assert result.status == "incomplete"
assert any("Limitation: Binary assets were not inspected" in error for error in result.errors)