Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-License-Identifier: Apache-2.0
"""Roundtrip and trust-boundary tests for retained native HTML rendering."""
import json
from pathlib import Path
from unittest.mock import patch
import pytest
from skillevaluator.models import Finding, SuccessDetail, ValidationResult
from skillevaluator.reporting.html import HTMLReporter
from skillevaluator.reporting.json_reporter import JSONReporter
from skillevaluator.reporting.retained import RetainedReportError, render_retained_report, retained_results
@pytest.fixture
def payload():
result = ValidationResult(validator_name="Schema & Repository Governance", validator_description="Schema checks")
result.add_finding(
Finding(
category="SCHEMA",
severity="high",
check_name="name_consistency",
message="Directory name differs from 'name'",
file_path="/untrusted/skill/SKILL.md",
suggestion="Match the directory and frontmatter names",
)
)
result.success_details.append(SuccessDetail(check_name="manifest_exists", message="Found SKILL.md"))
result.metadata.update(
{
"gating": {"tier": 1, "blocking": True},
"workflow": {"tier": 1, "stages": {"llm": {"status": "skipped", "reason": "Disabled"}}},
}
)
quality = ValidationResult(validator_name="QUALITY", validator_description="Quality dimensions")
quality.metadata["quality_scores"] = {
"skill_name": "skill",
"overall_score": 71.8,
"grade": "C",
"dimensions": {"correctness": {"score": 71.8, "weight": 1.0, "issues_count": 0, "issues": []}},
"metrics": {},
}
return json.loads(JSONReporter(include_timestamp=False).render_all([result, quality]))
def test_roundtrip_preserves_verdict_findings_quality_and_unknown_line(payload):
rows = retained_results(payload)
exported = json.loads(JSONReporter(include_timestamp=False).render_all(rows))
assert exported["overall_passed"] is False
assert exported["results"] == payload["results"]
assert exported["quality_summary"] == payload["quality_summary"]
assert rows[0].findings[0].line_number is None
html = render_retained_report(payload, skill_name="skill")
assert "71.8" in html and "name_consistency" in html
assert "Match the directory and frontmatter names" in html
def test_roundtrip_preserves_upstream_quality_configuration_provenance(payload):
provenance = {
"source": "file",
"file": "/workspace/quality.yaml",
"digest": "sha256:" + "a" * 64,
"min_score": 80,
}
payload["quality_config"] = provenance
exported = json.loads(JSONReporter(include_timestamp=False).render_all(retained_results(payload)))
assert exported["quality_config"] == provenance
assert exported["results"] == payload["results"]
def test_render_explicit_identity_never_inspects_artifact_paths(payload):
payload["results"][0]["findings"][0]["file_path"] = "/private/secret/../../etc/SKILL.md"
# Package template loading remains permitted, but artifact path discovery must not occur.
with patch.object(Path, "exists", side_effect=AssertionError("Artifact path IO")):
rendered = render_retained_report(payload, skill_name="skill")
assert "name_consistency" in rendered
def test_incomplete_remains_incomplete():
result = ValidationResult(validator_name="Security", metadata={"incomplete_scans": ["semgrep"]})
result.add_error("Scanner execution did not complete")
payload = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
rows = retained_results(payload)
assert rows[0].is_incomplete
assert not json.loads(JSONReporter(include_timestamp=False).render_all(rows))["overall_passed"]
assert "incomplete" in render_retained_report(payload, skill_name="skill").lower()
@pytest.mark.parametrize("same_validator", [False, True])
def test_confirmed_failure_precedes_incompleteness_in_retained_roundtrip(payload, same_validator):
rows = retained_results(payload)
result = rows[0] if same_validator else rows[1]
result.metadata["incomplete_scans"] = ["semgrep"]
result.add_error("Scanner execution did not complete")
emitted = json.loads(JSONReporter(include_timestamp=False).render_all(rows))
assert emitted["overall_status"] == "failed"
assert emitted["incomplete_scans"] == ["semgrep"]
reconstructed = retained_results(emitted)
assert json.loads(JSONReporter(include_timestamp=False).render_all(reconstructed))["results"] == emitted["results"]
@pytest.mark.parametrize("status", ["passed", "incomplete"])
def test_retained_evidence_rejects_status_that_hides_confirmed_failure(payload, status):
payload["results"][0]["incomplete_scans"] = ["semgrep"]
payload["results"][0]["status"] = status
with pytest.raises(RetainedReportError, match="Result status contradicts evidence"):
retained_results(payload)
def test_confirmed_legacy_failure_survives_retained_roundtrip():
result = ValidationResult(validator_name="Code Integrity & Hygiene")
result.add_error("Banned dependency found in one audited member")
result.metadata["confirmed_failure"] = True
result.mark_scan_incomplete("another-member")
emitted = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
assert emitted["results"][0]["confirmed_failure"] is True
reconstructed = retained_results(emitted)
assert reconstructed[0].metadata["confirmed_failure"] is True
assert json.loads(JSONReporter(include_timestamp=False).render_all(reconstructed))["overall_status"] == "failed"
@pytest.mark.parametrize("value", [1, "true", [], None])
def test_retained_confirmed_failure_marker_must_be_boolean(payload, value):
payload["results"][0]["confirmed_failure"] = value
with pytest.raises(RetainedReportError, match="confirmed_failure must be boolean"):
retained_results(payload)
def test_retained_confirmed_failure_marker_requires_failing_diagnostics():
result = ValidationResult(validator_name="Security", passed=False)
emitted = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
emitted["results"][0]["confirmed_failure"] = True
with pytest.raises(RetainedReportError, match="Confirmed failure requires failing diagnostics"):
retained_results(emitted)
def test_retained_finding_failure_cannot_claim_a_passing_result(payload):
payload["results"][0]["passed"] = True
payload["overall_passed"] = True
payload["overall_status"] = "passed"
with pytest.raises(RetainedReportError, match="Result verdict contradicts evidence"):
retained_results(payload)
def test_advisory_skip_keeps_required_gate_and_explicit_tier1_boundary():
from skillevaluator.evaluation.tier3_report import advisory_skip_result
from skillevaluator.reporting.base import is_cleanly_skipped, passes_required_gate
result = advisory_skip_result("Live evaluation runtime unavailable", skill_name="skill")
assert result.passed is False
assert is_cleanly_skipped(result) and passes_required_gate(result)
emitted = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
assert emitted["overall_passed"] is True
assert emitted["results"][0]["status"] == "skipped"
with pytest.raises(RetainedReportError, match="Only retained Tier 1 reports are supported"):
retained_results(emitted, allowed_tiers={1})
assert retained_results(emitted)[0].passed is False
@pytest.mark.parametrize(
"mutate",
[
lambda p: p["results"][0].update(passed="false"),
lambda p: p["results"][0]["findings"][0].update(severity="invented"),
lambda p: p["results"][0]["summary"].update(errors=True),
lambda p: p["results"][0].update(incomplete_scans="semgrep"),
lambda p: p["results"][0].update(workflow=[]),
lambda p: p["results"][0].update(gating={"tier": "1", "blocking": True}),
lambda p: p["results"][1]["quality"].update(overall_score="71.8"),
lambda p: p.update(overall_passed=True),
lambda p: p.update(results=[]),
],
)
def test_malformed_evidence_rejected(payload, mutate):
mutate(payload)
with pytest.raises(RetainedReportError):
retained_results(payload)
def test_hostile_text_safe_and_context_urls_rejected(payload):
payload["results"][0]["findings"][0]["message"] = '</script><script>alert("x")</script>'
rendered = render_retained_report(payload, skill_name="skill", context={"commit": "<img src=x onerror=alert(1)>"})
assert '</script><script>alert("x")</script>' not in rendered
assert "\\u003c/script\\u003e" in rendered
for url in (
"javascript:alert(1)",
"http://example.com",
"https://user:[email protected]",
"https://example.com\\evil",
):
with pytest.raises(ValueError):
render_retained_report(payload, skill_name="skill", context={"run_url": url})
def test_no_quality_identity_is_explicit(payload):
payload["results"] = payload["results"][:1]
payload["total_validators"] = 1
html = render_retained_report(payload, skill_name="explicit-name")
assert "explicit-name" in html
assert "manifest_exists" in html
def test_ci_context_allowlist_and_https():
assert HTMLReporter(ci_context={"run_url": "", "report_url": ""}).ci_context == {}
reporter = HTMLReporter(
ci_context={"run_url": "https://github.com/a/b/actions/runs/1", "commit": "a" * 40, "unrecognized": "ignored"}
)
assert reporter.ci_context == {"run_url": "https://github.com/a/b/actions/runs/1", "commit": "a" * 40}
def test_fixed_relative_ci_links_only():
reporter = HTMLReporter(
ci_context={"report_index_url": "../index.html", "canonical_json_url": "skillevaluator-tier1.json"}
)
assert reporter.ci_context["report_index_url"] == "../index.html"
for key, value in (("run_url", "../index.html"), ("canonical_json_url", "../../secret")):
with pytest.raises(ValueError):
HTMLReporter(ci_context={key: value})
def test_validation_without_copies_matches_the_copying_rehydration(payload):
payload["results"][0]["findings"][0]["metadata"] = {"rule": {"nested": ["evidence"]}}
copied = retained_results(payload)
shared = retained_results(payload, copy_metadata=False)
assert shared == copied
finding_metadata = payload["results"][0]["findings"][0]["metadata"]
assert copied[0].findings[0].metadata is not finding_metadata
assert shared[0].findings[0].metadata is finding_metadata
payload["results"][0]["passed"] = "true"
with pytest.raises(RetainedReportError, match="passed must be boolean"):
retained_results(payload, copy_metadata=False)
def test_reporters_share_one_environment_and_compile_each_template_once(payload, monkeypatch):
from skillevaluator.reporting import html as html_module
loads = []
original = html_module.PackageLoader.get_source
def counting(self, environment, template):
loads.append(template)
return original(self, environment, template)
monkeypatch.setattr(html_module.PackageLoader, "get_source", counting)
HTMLReporter._create_environment.cache_clear()
results = retained_results(payload)
plain = HTMLReporter(include_timestamp=False)
ci = HTMLReporter(include_timestamp=False, title="Other", ci_context={"report_index_url": "../index.html"})
assert plain._env is ci._env
first = plain.render_all(results)
ci.render_all(results)
assert plain.render_all(results) == first
# Pages built outside report.html.j2, such as the CI index, reuse the same compiled logo.
assert html_module.render_logo("one") != html_module.render_logo("two")
assert sorted(loads) == [
"dataset_diagnostics.html.j2",
"logo.svg.j2",
"native_evidence.html.j2",
"plugin_evidence.html.j2",
"report.html.j2",
"report_ui.css.j2",
"report_ui.js.j2",
"telemetry.html.j2",
]
def test_shared_environment_renders_identically_from_many_threads(payload):
from concurrent.futures import ThreadPoolExecutor
HTMLReporter._create_environment.cache_clear()
def render(index):
context = {"report_index_url": "../index.html"} if index % 2 else None
return render_retained_report(payload, skill_name="skill", context=context)
expected = [render(0), render(1)]
HTMLReporter._create_environment.cache_clear()
with ThreadPoolExecutor(max_workers=8) as pool:
documents = list(pool.map(render, range(32)))
assert all(document == expected[index % 2] for index, document in enumerate(documents))
def test_only_standalone_reports_embed_the_export_json(payload):
standalone = render_retained_report(payload, skill_name="skill")
embedded = standalone.split('<script id="report-data" type="application/json">', 1)[1].split("</script>", 1)[0]
assert json.loads(embedded)["results"][0]["validator_name"] == payload["results"][0]["validator"]
assert 'onclick="exportJSON()"' in standalone
hosted = render_retained_report(
payload, skill_name="skill", context={"canonical_json_url": "skillevaluator-tier1.json"}
)
assert 'id="report-data"' not in hosted
assert 'href="skillevaluator-tier1.json" download' in hosted
assert 'onclick="exportJSON()"' not in hosted
def test_not_applicable_and_incomplete_presentation_preserves_gate():
lint = ValidationResult(validator_name="SCRIPT_LINT", validator_description="Scripts")
lint.success_details.append(SuccessDetail(check_name="lint", message="No scripts/ or tools/ directory found"))
reporter = HTMLReporter(explicit_skill_name="skill")
data = reporter._reorganize_single_skill([lint], "skill")
assert data["skill"]["validators"]["SCRIPT_LINT"]["status"] == "not_applicable"
assert lint.passed is True
assert reporter._env.filters["execution_status"](lint) == "not_applicable"
lint.metadata["incomplete_scans"] = ["scanner"]
lint.errors.append("Scanner unavailable")
lint.warnings.append("Scanner timeout after 30 seconds")
data = reporter._reorganize_single_skill([lint], "skill")
assert data["skill"]["status"] == "incomplete"
assert data["skill"]["validators"]["SCRIPT_LINT"]["errors"] == ["Scanner unavailable"]
assert data["skill"]["validators"]["SCRIPT_LINT"]["warnings"] == ["Scanner timeout after 30 seconds"]
assert data["skill"]["validators"]["SCRIPT_LINT"]["incomplete_scans"] == ["scanner"]
assert reporter._env.filters["execution_status"](lint) == "incomplete"
assert data["skill"]["validators"]["SCRIPT_LINT"]["status"] == "incomplete"
@pytest.mark.parametrize(
"check,message",
[
("code_file_discovery", "No code files found - code risk analysis is not applicable"),
("dependency_file_discovery", "No dependency files found - vulnerability audit is not applicable"),
("version_optional", "Version is optional"),
],
)
def test_explicit_discovery_nonexecution_is_not_presented_as_pass(check, message):
result = ValidationResult(validator_name="Discovery")
result.success_details.append(SuccessDetail(check_name=check, message=message))
reporter = HTMLReporter(explicit_skill_name="skill")
assert reporter._env.filters["execution_status"](result) == "not_applicable"
data = reporter._reorganize_single_skill([result], "skill")
assert data["skill"]["validators"]["Discovery"]["details"][0]["status"] == "not_applicable"
assert result.passed is True
def test_explicit_retained_paths_preserve_canonical_finding_location(payload):
results = retained_results(payload)
data = HTMLReporter(explicit_skill_name="skill")._reorganize_single_skill(results, "skill")
finding = data["skill"]["validators"]["Schema & Repository Governance"]["findings"][0]
assert finding["file_path"] == "/untrusted/skill/SKILL.md"
assert finding["line_number"] is None
def test_display_path_preserves_canonical_publication_identity_and_bound_waivers():
from skillevaluator.publication_evidence import stamp_publication_evidence, tier1_required_checks
results = []
for check in tier1_required_checks(skill_like=True):
result = ValidationResult(validator_name=check)
result.add_success(check, "Scanned")
result.metadata.update(
publication_target={
"skill_name": "demo",
"skill_digest": "sha256:" + "c" * 64,
"skill_digest_algorithm": "skill-evaluator-source-tree/2",
},
benchmark_policy={"tier2_required": False, "tier3_required": False},
)
stamp_publication_evidence([result], tier=1, check_id=check)
results.append(result)
payload = json.loads(JSONReporter(include_timestamp=False, expected_skill_name="demo").render_all(results))
assert payload["publication"]["status"] == "pass"
rendered = render_retained_report(payload, skill_name="skills/demo", expected_skill_name="demo")
assert 'data-publication-status="pass"' in rendered
assert "skills/demo" in rendered
mismatched = render_retained_report(payload, skill_name="skills/demo", expected_skill_name="different-skill")
assert 'data-publication-status="incomplete"' in mismatched
def test_skillspector_version_survives_json_and_retained_html_roundtrip():
result = ValidationResult(validator_name="Security")
result.metadata["skillspector_version"] = "2.12.0"
payload = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
assert payload["results"][0]["skillspector_version"] == "2.12.0"
assert retained_results(payload)[0].metadata["skillspector_version"] == "2.12.0"
assert "SkillSpector <strong>v2.12.0" in render_retained_report(payload, skill_name="demo")
_SKILLSPECTOR_2_12_LLM_PROVENANCE = Path(__file__).parents[1] / "fixtures" / "skillspector-2.12.0-llm-provenance.json"
def test_skillspector_llm_provenance_survives_json_and_retained_html_roundtrip():
provenance = json.loads(_SKILLSPECTOR_2_12_LLM_PROVENANCE.read_text(encoding="utf-8"))
result = ValidationResult(validator_name="Security")
result.metadata.update({"skillspector_llm_provenance": provenance, "skillspector_provider_id": "nv_build"})
payload = json.loads(JSONReporter(include_timestamp=False).render_all([result]))
metadata = retained_results(payload)[0].metadata
rendered = render_retained_report(payload, skill_name="demo")
assert metadata["skillspector_llm_provenance"] == provenance
assert metadata["skillspector_provider_id"] == "nv_build"
assert "Tier 1 LLM Provenance" in rendered
assert "<td>nv_build</td>" in rendered
def test_retained_report_omits_non_canonical_llm_provenance():
provenance = json.loads(_SKILLSPECTOR_2_12_LLM_PROVENANCE.read_text(encoding="utf-8"))
provenance["analyzers"][0]["model"] = "https://gateway.example.test/v1"
payload = json.loads(
JSONReporter(include_timestamp=False).render_all([ValidationResult(validator_name="Security")])
)
payload["results"][0].update(
{"skillspector_llm_provenance": provenance, "skillspector_provider_id": "private-gateway"}
)
metadata = retained_results(payload)[0].metadata
rendered = render_retained_report(payload, skill_name="demo")
assert "skillspector_llm_provenance" not in metadata
assert "skillspector_provider_id" not in metadata
assert "Tier 1 LLM Provenance" not in rendered
assert "gateway.example.test" not in rendered
@pytest.mark.parametrize(
"check,message,metadata,expected",
[
("code_file_discovery", "No code files found - code risk analysis is not applicable", {}, "NOT APPLICABLE"),
("lint", "No scripts/ or tools/ directory found", {}, "NOT APPLICABLE"),
("random-check", "Some rule is not applicable, other rules ran", {}, "PASS"),
("new-check", "No applicable target", {"not_applicable": True}, "NOT APPLICABLE"),
("lint", "No scripts/ or tools/ directory found", {"not_applicable": False}, "PASS"),
],
)
def test_html_and_ci_use_the_same_not_applicable_evidence(check, message, metadata, expected):
from ci.report_views import _status
result = ValidationResult(validator_name="Discovery")
result.success_details.append(SuccessDetail(check_name=check, message=message, metadata=metadata))
recorded = json.loads(JSONReporter(include_timestamp=False).render(result))
html_status = HTMLReporter()._env.filters["execution_status"](result)
assert html_status.upper().replace("_", " ") == expected
assert _status(recorded) == expected