Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Reports carry an offline, licensed chart runtime and accessible fallbacks."""
import hashlib
import json
import re
import shutil
import subprocess
from html.parser import HTMLParser
import pytest
from skillevaluator.evaluation.tier3_report import build_agent_eval_payload
from skillevaluator.models import ValidationResult
from skillevaluator.reporting import HTMLReporter
from skillevaluator.reporting import html as html_module
class _Report(HTMLParser):
def __init__(self) -> None:
super().__init__()
self.external: list[dict[str, str | None]] = []
self.elements: dict[str, tuple[str, dict[str, str | None]]] = {}
self.scripts: dict[str, str] = {}
self.script_id: str | None = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attributes = dict(attrs)
element_id = attributes.get("id")
if element_id:
self.elements[element_id] = tag, attributes
if tag == "script":
if attributes.get("src"):
self.external.append(attributes)
self.script_id = element_id
if element_id:
self.scripts[element_id] = ""
def handle_endtag(self, tag: str) -> None:
if tag == "script":
self.script_id = None
def handle_data(self, data: str) -> None:
if self.script_id:
self.scripts[self.script_id] += data
def _tier3_html(*, agent_count: int = 1) -> str:
result = ValidationResult(validator_name="AGENT_EVAL")
result.metadata["agent_eval"] = build_agent_eval_payload(
"example",
{
agent: {
"execution_status": "succeeded",
"expected_attempts": 1,
"scored_attempts": 1,
"with_skill": {"security": 1.0},
"rewards": [
{
"entry_id": "case-1",
"security": 1.0,
"_traj": {"steps": 4, "prompt_tokens": 10, "completion_tokens": 4, "cached_tokens": 2},
}
],
}
for agent in ["agent", "second-agent"][:agent_count]
},
use_llm_judge=False,
)
return HTMLReporter(include_timestamp=False).render_all([result])
def _parse(html: str) -> _Report:
parser = _Report()
parser.feed(html)
return parser
def test_tier3_embeds_verified_chart_runtime_without_network_or_source_map() -> None:
report = _parse(_tier3_html())
assert report.external == []
runtime = report.scripts["tier3-chart-runtime"]
assert "Chart.js v4.5.1" in runtime
assert "@kurkle/color v0.3.2" in runtime
# SRI-verified upstream bytes with the source-map directive removed and one final newline.
assert hashlib.sha256(runtime.encode()).hexdigest() == (
"84d0e233daba702b8f77d669d8c137cad36d441a10f200b6f2d3ab553bdfcf6b"
)
assert "sourceMappingURL" not in runtime
assert "</script" not in runtime.lower()
def test_reports_retain_both_bundled_library_licenses() -> None:
html = _tier3_html()
assert 'id="tier3-chart-notices"' in html
assert "Copyright (c) 2014-2024 Chart.js Contributors" in html
assert "Copyright (c) 2018-2021 Jukka Kurkela" in html
assert html.count("Permission is hereby granted, free of charge") == 2
assert html.count("THE SOFTWARE IS PROVIDED") == 2
@pytest.mark.parametrize("agent_count", [1, 2])
def test_chart_canvases_explain_data_and_keep_visible_failure_fallback(agent_count: int) -> None:
html = _tier3_html(agent_count=agent_count)
report = _parse(html)
canvases = {key: attrs for key, (tag, attrs) in report.elements.items() if tag == "canvas"}
token_chart = "tier3-recorded-token-chart" if "tier3-recorded-token-chart" in canvases else "tier3-token-chart"
assert canvases.keys() == {"tier3-radar", "tier3-metric-bar", token_chart, "tier3-steps-chart"}
for chart_id, attrs in canvases.items():
assert attrs["role"] == "img"
assert attrs["aria-label"]
assert "hidden" in attrs
description = attrs["aria-describedby"]
assert description and description in report.elements
_, fallback = report.elements[chart_id + "-fallback"]
assert "hidden" not in fallback
references = re.findall(r'class="chart-description"[^>]*>[^<]*<a href="#([^"]+)"', html)
assert len(references) == 4
for reference in references:
if reference == "tier3-multi-agent-dimension-table":
assert re.search(
r'id="tier3-multi-agent-dimension-table"[^>]*>\s*<table[^>]*id="tier3-dimension-table"',
html,
)
else:
assert report.elements[reference][0] == "table"
assert references[:2] == ["tier3-dimension-table" if agent_count == 1 else "tier3-multi-agent-dimension-table"] * 2
def test_tier1_does_not_load_or_embed_chart_resources(monkeypatch) -> None:
def forbidden():
raise AssertionError("Tier 1 must not load chart assets")
monkeypatch.setattr(html_module, "_load_chart_assets", forbidden)
html = HTMLReporter(include_timestamp=False).render_all([ValidationResult(validator_name="QUALITY")])
report = _parse(html)
assert "tier3-chart-runtime" not in report.scripts
assert "Chart.js v4.5.1" not in html
assert "tier3-chart-notices" not in html
def test_chart_package_resources_are_cached_across_report_instances() -> None:
html_module._load_chart_assets.cache_clear()
try:
_tier3_html()
_tier3_html()
info = html_module._load_chart_assets.cache_info()
assert info.misses == 1
assert info.hits == 1
finally:
html_module._load_chart_assets.cache_clear()
def test_chart_initialization_failure_keeps_fallback_and_recovers() -> None:
node = shutil.which("node")
if not node:
pytest.skip("Node.js is needed to execute the generated chart controller")
html = _tier3_html()
controller = re.search(r"\(function\(\)\{\s+var tier3Data = .*?\}\)\(\);", html, re.DOTALL)
assert controller
script = r"""
const vm = require('node:vm');
const controller = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
const ids = ['tier3-radar', 'tier3-metric-bar', 'tier3-recorded-token-chart', 'tier3-steps-chart'];
const elements = {};
let visible = true;
ids.forEach(id => {
elements[id] = {
id, hidden: true,
closest: () => null,
parentElement: {getClientRects: () => visible ? [{}] : []}
};
elements[id + '-fallback'] = {hidden: false};
});
const listeners = {};
let theme = 'dark';
const context = {
window: {addEventListener: (name, fn) => {listeners[name] = fn;}},
document: {
documentElement: {getAttribute: () => theme, setAttribute: (name, value) => {theme = value;}},
getElementById: id => elements[id] || null,
addEventListener: (name, fn) => {listeners[name] = fn;}
},
getComputedStyle: () => ({getPropertyValue: () => theme === 'dark' ? '#e2e8f0' : '#0f172a'})
};
vm.createContext(context);
vm.runInContext(controller, context);
// Theme initialization invokes the controller after selecting the report palette.
context.window.rebuildTier3Charts();
const missing = ids.map(id => [elements[id].hidden, elements[id + '-fallback'].hidden]);
context.Chart = class {constructor() {throw new Error('Canvas unavailable');}};
context.window.rebuildTier3Charts();
const failed = ids.map(id => [elements[id].hidden, elements[id + '-fallback'].hidden]);
let released = 0;
// Like Chart.js 4.5.1: a context-less instance is registered with canvas=null,
// so getChart(canvas) cannot find it; only the constructed instance can release it.
context.Chart = class {
constructor() {this.ctx = null;}
destroy() {released++;}
static getChart() {return undefined;}
};
context.window.rebuildTier3Charts();
const missingContext = ids.map(id => [elements[id].hidden, elements[id + '-fallback'].hidden]);
let built = 0, destroyed = 0, resized = 0;
const constructions = [];
const updates = [];
context.Chart = class {
constructor(canvas, config) {
this.ctx = {};
this.id = canvas.id;
this.data = config.data;
this.options = config.options;
built++;
constructions.push({id:canvas.id, type:config.type, color:config.options.plugins.legend.labels.color});
}
resize() {resized++;}
update(mode) {
updates.push({
id:this.id, mode:mode || null,
animation:this.options.animation === false ? false : null,
color:this.options.plugins.legend.labels.color
});
}
destroy() {destroyed++;}
};
visible = false;
context.window.rebuildTier3Charts();
const hiddenBuilt = built;
visible = true;
context.window.rebuildTier3Charts();
const recovered = ids.map(id => [elements[id].hidden, elements[id + '-fallback'].hidden]);
listeners.beforeprint();
// The shared report handler redraws after exposing print containers/disclosures.
context.window.rebuildTier3Charts();
listeners.afterprint();
process.stdout.write(JSON.stringify({missing, failed, missingContext, released, recovered, hiddenBuilt, built, destroyed, resized, constructions, updates, theme}));
"""
result = subprocess.run(
[node, "-e", script], input=json.dumps(controller.group()), capture_output=True, text=True, check=False
)
assert result.returncode == 0, result.stderr
states = json.loads(result.stdout)
assert states["missing"] == [[True, False]] * 4
assert states["failed"] == [[True, False]] * 4
assert states["missingContext"] == [[True, False]] * 4
assert states["released"] == 4
assert states["recovered"] == [[False, True]] * 4
assert states["hiddenBuilt"] == 0
assert states["built"] == 4
assert states["destroyed"] == 0
assert states["resized"] == 8
assert [(row["id"], row["type"]) for row in states["constructions"]] == [
("tier3-radar", "radar"),
("tier3-metric-bar", "bar"),
("tier3-recorded-token-chart", "bar"),
("tier3-steps-chart", "bar"),
]
assert [row["color"] for row in states["constructions"]] == ["#e2e8f0"] * 4
assert [row["animation"] for row in states["updates"][:4]] == [False] * 4
assert [row["mode"] for row in states["updates"][:4]] == ["none"] * 4
assert [row["color"] for row in states["updates"]] == ["#0f172a"] * 4 + ["#e2e8f0"] * 4
assert states["theme"] == "dark"
def test_missing_dimension_scores_keep_agent_cards_and_openable_downloads() -> None:
result = ValidationResult(validator_name="AGENT_EVAL")
result.metadata["agent_eval"] = build_agent_eval_payload(
"example",
{"agent": {"execution_status": "failed", "expected_attempts": 1, "scored_attempts": 0, "with_skill": {}}},
use_llm_judge=False,
)
html = HTMLReporter(include_timestamp=False).render_all([result])
report = _parse(html)
assert "tier3-radar" not in report.elements and "tier3-metric-bar" not in report.elements
assert 'class="t3-agent-score" style="color:var(--text-secondary);">N/A</div>' in html
assert report.elements["tier3-json-downloads"][0] == "details"
# The section sits in the hidden Diagnostics sub-tab; the static handler reveals it.
assert "function openTier3Reference(id)" in html
def test_missing_chart_asset_keeps_the_report_and_visible_fallbacks(monkeypatch) -> None:
class Missing:
def joinpath(self, *_parts):
return self
def read_text(self, **_kwargs):
raise FileNotFoundError("chart.umd.min.js")
html_module._load_chart_assets.cache_clear()
monkeypatch.setattr(html_module.resources, "files", lambda _package: Missing())
try:
html = _tier3_html()
finally:
html_module._load_chart_assets.cache_clear()
report = _parse(html)
assert "tier3-chart-runtime" not in report.scripts
assert "tier3-chart-notices" not in html
_, fallback = report.elements["tier3-radar-fallback"]
assert "hidden" not in fallback
def test_chart_reference_opens_ancestor_evidence_and_reuses_shared_navigation() -> None:
node = shutil.which("node")
if not node:
pytest.skip("Node.js is needed to execute the generated chart reference handler")
html = _tier3_html()
helper = re.search(r"function openTier3Reference\(id\).*?(?=\s*function showTier3SubTab)", html, re.DOTALL)
assert helper
script = r"""
const vm = require('vm');
const state = {lookups:0, focused:false, hash:null, tab:null, routed:[]};
const outer = {tagName:'DETAILS', open:false, parentElement:null};
const inner = {tagName:'DETAILS', open:false, parentElement:outer};
const page = {id:'tier3-page-trials', classList:{contains:()=>false}};
const target = {
tagName:'TABLE', parentElement:inner, closest:()=>page,
setAttribute:(key,value)=>{state[key]=value;},
focus:()=>{state.focused=true;},
scrollIntoView:()=>{}
};
const context = {
document:{getElementById:()=>{state.lookups++; return target;}, querySelectorAll:()=>[{dataset:{tier3Tab:'trials'}}]},
history:{replaceState:(_data,_title,hash)=>{state.hash=hash;}},
showTier3SubTab:(tab)=>{state.tab=tab;}
};
vm.createContext(context);
vm.runInContext(HELPER, context);
const result = context.openTier3Reference('evidence-table');
const standalone = {outer:outer.open,inner:inner.open,focused:state.focused,tabindex:state.tabindex,hash:state.hash,tab:state.tab,result};
const before = state.lookups;
context.openReportDestination = id => {state.routed.push(id); return false;};
const routed = context.openTier3Reference('evidence-table');
process.stdout.write(JSON.stringify({standalone,routed,targets:state.routed,extraLookups:state.lookups-before}));
""".replace("HELPER", json.dumps(helper.group()))
states = json.loads(subprocess.run([node, "-e", script], check=True, capture_output=True, text=True).stdout)
assert states["standalone"] == {
"outer": True,
"inner": True,
"focused": True,
"tabindex": "-1",
"hash": "#evidence-table",
"tab": "trials",
"result": False,
}
assert states["routed"] is False
assert states["targets"] == ["evidence-table"]
assert states["extraLookups"] == 0