Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-License-Identifier: Apache-2.0
"""Concise reporting keeps unavailable measurements and evidence truthful."""
import json
import re
from copy import deepcopy
from html import unescape
from html.parser import HTMLParser
import pytest
from tests.reporting.browser_navigation_fixtures import _agent_result
from skillevaluator.models import ValidationResult
from skillevaluator.reporting import HTMLReporter
def _text(markup):
return " ".join(unescape(re.sub(r"<[^>]*>", " ", markup)).split())
def _reliability(markup):
return _text(re.search(r'<table class="summary-table pass-at-k-table".*?</table>', markup, re.DOTALL)[0])
def _render(result):
return HTMLReporter(include_timestamp=False).render_all([result])
class _DisclosureText(HTMLParser):
def __init__(self):
super().__init__()
self.closed = []
self.text = []
def handle_starttag(self, tag, attrs):
if tag == "details":
self.closed.append("open" not in dict(attrs))
def handle_endtag(self, tag):
if tag == "details":
self.closed.pop()
def handle_data(self, data):
if not any(self.closed):
self.text.append(data)
def test_comparison_keeps_measurements_visible_and_recorded_reasoning_expandable():
result = _agent_result("demo", ["agent"])
explanation = "Detailed recorded evaluator reasoning remains available for inspection."
for dim in result.metadata["agent_eval"]["dimensions"]:
dim["reasoning_bullets"] = [explanation]
markup = _render(result)
table = re.search(r'<table[^>]*id="tier3-dimension-table".*?</table>', markup, re.DOTALL)[0]
visible = _DisclosureText()
visible.feed(table)
text = " ".join(visible.text)
assert explanation not in text
assert explanation in table
for label in ("Security", "Correctness", "Discoverability", "Effectiveness", "Overall"):
assert label in text
assert "0.80" in text and "0.40" in text and "+0.40" in text
def test_attempt_coverage_does_not_invent_pass_rate_or_pass_counts():
result = _agent_result("demo", ["agent"])
payload = result.metadata["agent_eval"]
agent = payload["agents"]["agent"]
agent["pass_at_k"] = {
"with_skill": {"attempts_used": 16, "max_attempts_possible": 18},
"without_skill": {"attempts_used": 12, "max_attempts_possible": 18},
"lift": {},
}
payload["pass_at_k"] = deepcopy(agent["pass_at_k"])
original = deepcopy(payload)
markup = _render(result)
table = _reliability(markup)
assert "0%" not in table
assert "0 of 0" not in table
assert "0 of 16" not in table
assert "16 scored" in table and "12 scored" in table
assert "N/A" in table
exported = re.search(r'<script[^>]*id="tier3-full"[^>]*>(.*?)</script>', markup, re.DOTALL)[1]
assert json.loads(exported) == original
assert payload == original
def test_recorded_zero_reliability_is_preserved():
result = _agent_result("demo", ["agent"])
payload = result.metadata["agent_eval"]
branch = {
"rate": 0.0,
"passed_cases": 0,
"total_cases": 2,
"attempts_used": 2,
"cases": {name: {"attempts": [{"passed": False, "score": 0.0}]} for name in ["case-1", "case-2"]},
}
payload["agents"]["agent"]["pass_at_k"] = {"with_skill": branch}
payload["pass_at_k"] = {"with_skill": deepcopy(branch)}
table = _reliability(_render(result))
assert "0%" in table
assert "0 of 2" in table
@pytest.mark.parametrize("attempts", [None, [{"passed": True}], [{"passed": "false"}]])
def test_partial_or_untyped_attempt_outcomes_do_not_invent_a_pass_count(attempts):
result = _agent_result("demo", ["agent"])
payload = result.metadata["agent_eval"]
case = {"attempts_used": 16}
if attempts is not None:
case["attempts"] = attempts
branch = {"attempts_used": 16, "cases": {"case-1": case}}
payload["agents"]["agent"]["pass_at_k"] = {"with_skill": branch}
payload["pass_at_k"] = {"with_skill": deepcopy(branch)}
table = _reliability(_render(result))
assert "0 of 16" not in table and "1 of 16" not in table
assert "N/A · 16 scored" in table
def test_agent_does_not_call_a_completed_baseline_absent_when_aggregate_is_missing():
result = _agent_result("demo", ["agent"])
agent = result.metadata["agent_eval"]["agents"]["agent"]
agent["baseline"] = None
agent["lift"] = None
markup = _render(result)
details = markup.split('id="tier3-page-agents"', 1)[1].split('id="tier3-page-dataset"', 1)[0]
strip = details.split('class="t3-kpi-strip"', 1)[1].split('class="t3-dim-strip"', 1)[0]
assert "Overall baseline unavailable" in _text(strip)
assert "Skill Lift compares Overall" not in strip
def test_one_tier2_skip_notice_retains_both_affected_checks():
results = [
ValidationResult(validator_name=name, metadata={"skipped": True, "skip_reason": "No local catalog supplied"})
for name in ["CONTEXT_DEDUP", "CONTEXT_OPTIMIZATION"]
]
markup = HTMLReporter(include_timestamp=False).render_all(results)
within = markup.split('id="tier2-within"', 1)[1].split('id="dedup-inter"', 1)[0]
assert within.count("No local catalog supplied") == 1
assert "CONTEXT_DEDUP" in within and "CONTEXT_OPTIMIZATION" in within
def test_unknown_tier2_scope_keeps_skipped_check_identity_and_reason():
result = ValidationResult(
validator_name="UNKNOWN_CATALOG_CHECK",
metadata={"gating": {"tier": 2}, "skipped": True, "skip_reason": "Unique retained skip explanation"},
)
markup = _render(result)
other = markup.split('id="tier2-other"', 1)[1].split("</section>", 1)[0]
assert "Unique retained skip explanation" in other and "UNKNOWN_CATALOG_CHECK" in other
ids = re.findall(r'\bid="([^"]+)"', markup)
assert len(ids) == len(set(ids))
def test_harbor_display_labels_use_only_recorded_job_associations():
from skillevaluator.reporting.harbor_viewer import harbor_job_display_links
source = {
"harbor_viewer": {
"jobs": [
{"name": "very-long-opaque-job-id", "url": "https://example.test/jobs/1"},
{"name": "another-opaque-job-id", "url": "https://example.test/jobs/2"},
]
},
"agents": {"agent": {"trials": [{"harbor_viewer": {"job_url": "https://example.test/jobs/1"}}]}},
}
before = deepcopy(source)
jobs = harbor_job_display_links(source)
assert [job["display_label"] for job in jobs] == ["agent · With skill", "Job 2"]
assert jobs[0]["name"] == "very-long-opaque-job-id"
assert source == before