Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Failed Tier 3 source checks are listed in every report, not only in JSON."""
from __future__ import annotations
import json
import logging
import re
from pathlib import Path
import pytest
from click.testing import CliRunner
from jinja2 import make_logging_undefined
from markupsafe import escape
from skillevaluator.cli import cli
from skillevaluator.constants import SOURCE_VALIDATION_CLI_LIMIT
from skillevaluator.evaluation.tier3_report import advisory_skip_result, dataset_required_result
from skillevaluator.models import Finding, ValidationResult
from skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter
from skillevaluator.reporting.base import (
source_validation_errors,
source_validation_full_report,
source_validation_preview,
)
from skillevaluator.reporting.console_ui import summarize_tier3
from skillevaluator.reporting.html import _sanitize_tier3_display_payload
from skillevaluator.tier3.evals_spec import CheckResult
_MESSAGE = "Missing required fields: question"
_HTML_REPORT = "Full Insights in the HTML report"
_JSON_REPORT = "tier3.provenance.source_validation in the JSON report"
_OVERFLOW = f"see {_HTML_REPORT}."
_JSON_OVERFLOW = f"see {_JSON_REPORT}."
# Format combinations a run can emit: with and without the CLI report, and
# with HTML, JSON, both, or neither written as a file.
_FORMAT_SETS = [
(),
("cli",),
("cli", "markdown", "sarif"),
("cli", "json"),
("cli", "html"),
("cli", "json", "html"),
("markdown",),
("json",),
("html", "json"),
]
def _written_report(report_formats: tuple[str, ...]) -> str | None:
"""The written report that lists every source error, or None when the run writes none."""
if "html" in report_formats:
return _HTML_REPORT
if "json" in report_formats:
return _JSON_REPORT
return None
def _overflow_note(hidden: int, report: str | None) -> str:
return f"{hidden} more {'error' if hidden == 1 else 'errors'}: see {report}."
def _plain(output: str) -> str:
"""Strip ANSI styling and Rich's line wrapping."""
return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", output).split())
def _source_result(tmp_path: Path, checks: list[CheckResult], *, blocking: bool = True) -> ValidationResult:
return dataset_required_result(
tmp_path / "evals" / "evals.json",
checks,
blocking=blocking,
skill_name="example-skill",
source_kind="evals_json",
)
def _entry_errors(count: int, message: str = _MESSAGE) -> list[CheckResult]:
return [CheckResult(f"entry case-{index}", "error", message) for index in range(1, count + 1)]
def _html_section(html: str, section_id: str) -> str:
start = html.index(f'id="{section_id}"')
return html[start : html.index("</section>", start)]
@pytest.mark.parametrize("blocking", [False, True])
def test_dataset_required_result_records_only_failed_checks(tmp_path: Path, blocking: bool) -> None:
result = _source_result(
tmp_path,
[
CheckResult("evals/evals.json", "ok", "Tier 3 eval dataset found."),
CheckResult("entry case-1", "error", _MESSAGE),
CheckResult("entry case-1", "warning", "Missing recommended fields: expected_skill"),
CheckResult("evals/config.yml", "missing", "Required config is absent."),
],
blocking=blocking,
)
expected = [
{"path": "entry case-1", "status": "error", "message": _MESSAGE},
{"path": "evals/config.yml", "status": "missing", "message": "Required config is absent."},
]
assert result.findings[0].metadata["source_validation"] == expected
assert result.metadata["agent_eval"]["provenance"]["source_validation"] == expected
json_report = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
assert json_report["tier3"]["provenance"]["source_validation"] == expected
def test_source_errors_stay_out_of_the_finding_text(tmp_path: Path) -> None:
# Report grouping and SARIF results key on the finding text, so fixing one
# entry at a time must not turn the remaining errors into a "new" finding.
before = _source_result(tmp_path, _entry_errors(13))
after = _source_result(tmp_path, _entry_errors(13)[3:])
suggestion = f"Create or fix the Tier 3 source under {tmp_path / 'evals'} and rerun validation."
assert before.findings[0].suggestion == after.findings[0].suggestion == suggestion
assert before.findings[0].to_legacy_string() == after.findings[0].to_legacy_string()
assert before.metadata["agent_eval"]["suggestions"] == after.metadata["agent_eval"]["suggestions"] == [suggestion]
assert len(after.findings[0].metadata["source_validation"]) == 10
@pytest.mark.parametrize("report_formats", [fmts for fmts in _FORMAT_SETS if "cli" in fmts])
@pytest.mark.parametrize("error_count", [0, 1, 5, 6, 13, 100])
def test_cli_report_previews_source_errors_in_order(
tmp_path: Path,
error_count: int,
report_formats: tuple[str, ...],
) -> None:
result = _source_result(tmp_path, _entry_errors(error_count))
output = _plain(CLIReporter(report_formats=report_formats).render(result))
report = _written_report(report_formats)
# Without an HTML or JSON report the terminal is the only place the errors
# appear, so it lists every one of them and names no report.
shown = min(error_count, SOURCE_VALIDATION_CLI_LIMIT) if report else error_count
assert re.findall(r"entry case-\d+:", output) == [f"entry case-{index}:" for index in range(1, shown + 1)]
assert "Create or fix the Tier 3 source" in output
assert ("Source validation:" in output) == (error_count > 0)
if error_count:
assert output.index("Source validation:") > output.index("Create or fix the Tier 3 source")
hidden = error_count - shown
if hidden:
assert _overflow_note(hidden, report) in output
else:
assert "more error" not in output
for other_report in {_HTML_REPORT, _JSON_REPORT} - {report}:
assert other_report not in output
def test_cli_reporter_without_report_formats_lists_every_source_error(tmp_path: Path) -> None:
# A reporter that is not told about any written report must not point at one.
output = _plain(CLIReporter().render(_source_result(tmp_path, _entry_errors(13))))
assert re.findall(r"entry case-\d+:", output) == [f"entry case-{index}:" for index in range(1, 14)]
assert "more error" not in output
assert _HTML_REPORT not in output
assert _JSON_REPORT not in output
@pytest.mark.parametrize("report_formats", _FORMAT_SETS)
@pytest.mark.parametrize("error_count", [0, 5, 6, 13])
def test_source_validation_preview_names_only_written_reports(
error_count: int,
report_formats: tuple[str, ...],
) -> None:
errors = [{"path": f"entry case-{index}", "status": "error", "message": _MESSAGE} for index in range(error_count)]
report = _written_report(report_formats)
shown, note = source_validation_preview(errors, report_formats)
assert source_validation_full_report(report_formats) == report
if report and error_count > SOURCE_VALIDATION_CLI_LIMIT:
assert shown == errors[:SOURCE_VALIDATION_CLI_LIMIT]
assert note == _overflow_note(error_count - SOURCE_VALIDATION_CLI_LIMIT, report)
else:
assert shown == errors
assert note is None
def test_cli_report_prints_source_errors_as_inert_text(tmp_path: Path) -> None:
result = _source_result(
tmp_path,
[
CheckResult(
"entry [bold]x[/bold]\x1b[2J\x1b]8;;https://evil.test/\x07", "error", "Expected <field>\n& more."
),
CheckResult("", "error", "Cannot parse dataset: [/x] at C:\\evals\\"),
],
)
text = re.sub(r"\x1b\[[0-9;]*m", "", CLIReporter().render(result))
# Rich's own styling aside, no escape sequence from the eval file reaches the terminal.
assert re.search(r"[\x00-\x09\x0b-\x1f\x7f-\x9f]", text) is None, repr(text)
output = " ".join(text.split())
assert "Source validation: entry [bold]x[/bold]: Expected <field> & more." in output
assert "more. Cannot parse dataset: [/x] at C:\\evals\\" in output
assert "evil.test" not in output
def test_cli_report_ignores_malformed_source_validation_metadata() -> None:
result = ValidationResult(validator_name="AGENT_EVAL")
result.add_finding(
Finding(
category="AGENT_EVAL",
severity="high",
check_name="eval_dataset_required",
message="No valid Tier 3 source.",
file_path="evals/evals.json",
suggestion="Keep the existing recommendation.",
metadata={
"source_validation": [
"entry text-only",
{"path": "entry ok", "status": "ok", "message": "Loaded."},
{"path": "entry warn", "status": "warning", "message": "Optional field."},
{"path": 7, "status": "error", "message": "Non-string path."},
{"path": "entry list", "status": ["error"], "message": "List status."},
{"path": "entry dict", "status": {"error": True}, "message": "Mapping status."},
{"path": "entry blank", "status": "error", "message": " "},
{"path": "entry kept", "status": "error", "message": "Invalid entry."},
]
},
)
)
output = _plain(CLIReporter().render(result))
assert "Keep the existing recommendation." in output
assert "Source validation: entry kept: Invalid entry." in output
for dropped in (
"text-only",
"Loaded.",
"Optional field.",
"Non-string path.",
"List status.",
"Mapping status.",
"entry blank",
):
assert dropped not in output
@pytest.mark.parametrize("metadata", [{}, {"source_validation": []}, {"source_validation": "entry x: bad"}])
def test_cli_report_omits_source_section_without_errors(metadata: dict) -> None:
result = ValidationResult(validator_name="SCHEMA")
result.add_finding(
Finding(
category="SCHEMA",
severity="high",
check_name="schema-check",
message="A validation finding.",
file_path="SKILL.md",
suggestion="Keep the existing recommendation.",
metadata=metadata,
)
)
output = _plain(CLIReporter().render(result))
assert "Keep the existing recommendation." in output
assert "Source validation:" not in output
@pytest.mark.parametrize("blocking", [False, True])
@pytest.mark.parametrize("error_count", [0, 1, 5, 6, 13, 100])
@pytest.mark.parametrize("report_formats", _FORMAT_SETS)
def test_compact_tier3_panel_counts_and_previews_source_errors(
tmp_path: Path,
blocking: bool,
error_count: int,
report_formats: tuple[str, ...],
) -> None:
result = _source_result(tmp_path, _entry_errors(error_count), blocking=blocking)
finding = result.findings[0]
original_suggestion = finding.suggestion
original_metadata = json.dumps(result.metadata, sort_keys=True)
ran, _passed, rows, skip_reason = summarize_tier3(result, report_formats=report_formats)
assert ran is True
assert skip_reason == ""
report = _written_report(report_formats)
# The full CLI report lists the errors itself; otherwise the panel lists all
# of them unless an HTML or JSON report written by the run keeps the rest.
if "cli" in report_formats:
shown = 0
elif report:
shown = min(error_count, SOURCE_VALIDATION_CLI_LIMIT)
else:
shown = error_count
assert [row.segments[0][0] for row in rows if row.label == "source"] == [
f"entry case-{index}: {_MESSAGE}" for index in range(1, shown + 1)
]
counts = [row.segments[0][0] for row in rows if row.label == "validation"]
truncated = "cli" not in report_formats and shown < error_count
if error_count:
expected = f"{error_count} source validation {'error' if error_count == 1 else 'errors'}"
if truncated:
expected += f" (showing first {SOURCE_VALIDATION_CLI_LIMIT})"
assert counts == [expected]
else:
assert counts == []
rendered = " ".join(text for row in rows for text, _style in row.segments)
if truncated:
assert _overflow_note(error_count - SOURCE_VALIDATION_CLI_LIMIT, report) in rendered
else:
assert "more error" not in rendered
assert finding.suggestion == original_suggestion
assert json.dumps(result.metadata, sort_keys=True) == original_metadata
def test_compact_tier3_panel_projects_source_errors_to_one_safe_line(tmp_path: Path) -> None:
result = _source_result(
tmp_path,
[CheckResult("entry \x1b[2Jcase\x07-1", "error", "Expected\na value [bold]here[/bold]. " + "x" * 300)],
)
_ran, _passed, rows, _skip_reason = summarize_tier3(result)
(source_row,) = [row for row in rows if row.label == "source"]
text = source_row.segments[0][0]
assert text.startswith("entry [2Jcase -1: Expected a value [bold]here[/bold]. x")
assert text.endswith("…")
assert len(text) <= 110
assert not any(ord(character) < 32 for character in text)
@pytest.mark.parametrize("error_count", [0, 1, 2, 13, 100])
def test_html_report_links_source_error_scope_and_preserves_full_insights(tmp_path: Path, error_count: int) -> None:
result = _source_result(tmp_path, _entry_errors(error_count))
html = HTMLReporter(include_timestamp=False).render_all([result])
preview = _html_section(html, "tier3-next-steps-preview")
insights = _html_section(html, "tier3-page-suggestions")
assert 'href="#tier3-page-suggestions">View insights</a>' in preview
assert "Create or fix the Tier 3 source" in preview
assert "Create or fix the Tier 3 source" in insights
assert "entry case-" not in preview
assert "Source validation error:" not in preview
if error_count:
assert f"Source Validation Errors ({error_count})" in insights
items = re.findall(r"<li>(entry case-\d+): ([^<]*)</li>", insights)
assert items == [(f"entry case-{index}", _MESSAGE) for index in range(1, error_count + 1)]
else:
assert 'id="tier3-source-validation-errors"' not in insights
def test_html_report_escapes_source_errors(tmp_path: Path) -> None:
result = _source_result(tmp_path, [CheckResult("entry <b>&", "error", "Expected <script>alert(1)</script>.")])
html = HTMLReporter(include_timestamp=False).render_all([result])
preview = _html_section(html, "tier3-next-steps-preview")
insights = _html_section(html, "tier3-page-suggestions")
error_text = escape("entry <b>&: Expected <script>alert(1)</script>.")
assert error_text not in preview
assert 'href="#tier3-page-suggestions"' in preview
assert f"<li>{error_text}</li>" in insights
assert "<script>alert(1)" not in html
@pytest.mark.parametrize("has_source_errors", [False, True])
def test_html_source_errors_keep_existing_insights(tmp_path: Path, has_source_errors: bool) -> None:
result = _source_result(tmp_path, _entry_errors(1 if has_source_errors else 0))
payload = result.metadata["agent_eval"]
payload["conclusions"] = [{"title": "Coverage gap", "message": "Error handling needs more coverage."}]
payload["recommendations"] = [{"title": "Improve coverage", "message": "Add an error-handling test."}]
insights = _html_section(HTMLReporter(include_timestamp=False).render_all([result]), "tier3-page-suggestions")
assert "<summary>Conclusions</summary>" in insights
assert "Error handling needs more coverage." in insights
assert "<summary>Recommendations</summary>" in insights
assert "Add an error-handling test." in insights
assert ('id="tier3-source-validation-errors"' in insights) == has_source_errors
@pytest.mark.parametrize("has_source_errors", [False, True])
def test_html_empty_insights_message_requires_no_source_errors(tmp_path: Path, has_source_errors: bool) -> None:
payload = _source_result(tmp_path, _entry_errors(1 if has_source_errors else 0)).metadata["agent_eval"]
payload["suggestions"] = []
# A bare payload carries no finding diagnostics that would become recommendations.
result = ValidationResult(validator_name="AGENT_EVAL")
result.passed = False
result.metadata["agent_eval"] = payload
insights = _html_section(HTMLReporter(include_timestamp=False).render_all([result]), "tier3-page-suggestions")
assert ("No evaluation insights were generated." in insights) == (not has_source_errors)
assert ('id="tier3-source-validation-errors"' in insights) == has_source_errors
def test_skipped_tier3_html_defines_source_errors(caplog: pytest.LogCaptureFixture) -> None:
reporter = HTMLReporter(include_timestamp=False, expected_skill_name="example-skill")
reporter._env.undefined = make_logging_undefined(logger=logging.getLogger(__name__))
with caplog.at_level(logging.WARNING):
html = reporter.render_all([advisory_skip_result("Harbor runtime unavailable", skill_name="example-skill")])
assert "Harbor runtime unavailable" in html
assert 'id="tier3-source-validation-errors"' not in html
assert "t3_source_errors" not in caplog.text
def test_html_payload_keeps_only_well_formed_source_errors() -> None:
payload = _sanitize_tier3_display_payload(
{
"provenance": {
"source_validation": [
{"path": "entry a", "status": "error", "message": "Invalid entry.", "extra": "dropped"},
{"path": "entry b", "status": "ok", "message": "Loaded."},
{"path": None, "status": "error", "message": "Bad path."},
{"path": "entry d", "status": ["error"], "message": "Unhashable status."},
["entry c", "error", "Not a mapping."],
{"message": "No path or status recorded."},
]
}
}
)
assert payload["provenance"]["source_validation"] == [
{"path": "entry a", "status": "error", "message": "Invalid entry."},
{"path": "", "status": "error", "message": "No path or status recorded."},
]
assert source_validation_errors({"source_validation": {"path": "entry a"}}) == []
def _write_invalid_dataset(skill: Path, count: int) -> None:
(skill / "evals").mkdir()
(skill / "evals" / "evals.json").write_text(
json.dumps([{"id": f"case-{index}", "expected_behavior": ["Done."]} for index in range(1, count + 1)]),
encoding="utf-8",
)
def _invoke_validate(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
extra_args: list[str],
*,
blocking: bool,
) -> tuple[str, Path]:
"""Run ``validate`` on a skill with 13 invalid eval entries; return its plain output and report directory."""
from skillevaluator import cli as cli_module
from skillevaluator.evaluation import EvaluationService
# A forced terminal would animate the pipeline view and repeat each frame.
for name in ("FORCE_COLOR", "TTY_COMPATIBLE"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("COLUMNS", "120")
skill = tmp_path / "example-skill"
skill.mkdir()
(skill / "SKILL.md").write_text("---\nname: example-skill\ndescription: test\n---\n", encoding="utf-8")
_write_invalid_dataset(skill, 13)
tier1 = ValidationResult(validator_name="SCHEMA")
tier1.add_success("schema", "ok")
monkeypatch.setattr(cli_module, "run_validation", lambda *_args, **_kwargs: [tier1])
monkeypatch.setattr(
EvaluationService,
"evaluate",
lambda *_args, **_kwargs: pytest.fail("source preflight must run before live evaluation"),
)
output_dir = tmp_path / "reports"
args = ["validate", "--no-autopilot", str(skill), "--no-tier2", "--tier3", "--output-dir", str(output_dir)]
args.extend(extra_args)
if blocking:
args.append("--block-on-agent-eval")
command = CliRunner().invoke(cli, args)
assert command.exit_code == (1 if blocking else 0), command.output
return _plain(command.output), output_dir
@pytest.mark.parametrize("blocking", [False, True])
@pytest.mark.parametrize("report_formats", [None, "cli", "json", "cli,json", "cli,json,html", "json,html"])
def test_validate_shows_source_errors_once(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
blocking: bool,
report_formats: str | None,
) -> None:
extra_args = ["--report", report_formats] if report_formats else []
output, output_dir = _invoke_validate(monkeypatch, tmp_path, extra_args, blocking=blocking)
# The quiet default writes HTML and JSON; an explicit -r is honoured exactly.
emitted = tuple(report_formats.split(",")) if report_formats else ("html", "json")
full_cli_report = "cli" in emitted
report = _written_report(emitted)
shown = SOURCE_VALIDATION_CLI_LIMIT if report else 13
assert "13 source validation errors" in output
assert ("(showing first 5)" in output) == (not full_cli_report and report is not None)
assert ("Source validation:" in output) == full_cli_report
assert re.findall(r"entry case-\d+:", output) == [f"entry case-{index}:" for index in range(1, shown + 1)]
if report:
assert output.count(_overflow_note(13 - shown, report)) == 1
else:
assert "more error" not in output
for other_report in {_HTML_REPORT, _JSON_REPORT} - {report}:
assert other_report not in output
json_reports = list(output_dir.glob("*.json"))
assert len(json_reports) == ("json" in emitted)
if json_reports:
json_report = json.loads(json_reports[0].read_text(encoding="utf-8"))
assert len(json_report["tier3"]["provenance"]["source_validation"]) == 13
html_reports = list(output_dir.glob("*.html"))
assert len(html_reports) == ("html" in emitted)
if html_reports:
html = html_reports[0].read_text(encoding="utf-8")
assert "Source Validation Errors (13)" in html
assert f"<li>entry case-13: {_MESSAGE}</li>" in html
@pytest.mark.parametrize("blocking", [False, True])
def test_verbose_validate_without_file_reports_lists_every_source_error(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
blocking: bool,
) -> None:
# --verbose prints the full CLI report and, without -r, writes no HTML or
# JSON report, so every error must be in the terminal output.
output, output_dir = _invoke_validate(monkeypatch, tmp_path, ["--verbose"], blocking=blocking)
assert "Source validation:" in output
assert re.findall(r"entry case-\d+:", output) == [f"entry case-{index}:" for index in range(1, 14)]
assert "more error" not in output
assert _HTML_REPORT not in output
assert _JSON_REPORT not in output
assert list(output_dir.glob("*.json")) == []
assert list(output_dir.glob("*.html")) == []