Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Recorded telemetry stays scoped, bounded and distinct from missing evidence."""
import copy
import json
import re
from pathlib import Path
import pytest
from jinja2 import Environment, FileSystemLoader, select_autoescape
def _view(payload):
from skillevaluator.reporting.telemetry_view import build_telemetry_view
return build_telemetry_view(payload)
def _render(view):
root = Path(__file__).parents[2] / "src/skillevaluator/reporting/templates"
env = Environment(loader=FileSystemLoader(root), autoescape=select_autoescape(default=True))
return env.get_template("telemetry.html.j2").module.render_telemetry(view)
def _arm(totals, *, status="complete", measured=2, reasons=None):
return {
"totals": totals,
"coverage": {
"expected_attempts": 2,
"observed_attempts": 2,
"measured_attempts": measured,
"partial_attempts": 2 - measured,
"status": status,
"reason_codes": reasons or [],
"llm_timing_sources": ["atif", "step_timestamps"],
},
}
def _payload():
token_scope = {
"with_skill": _arm(
{
"prompt_tokens": 100,
"cached_tokens": 40,
"uncached_prompt_tokens": 60,
"completion_tokens": 20,
"total_tokens": 120,
"efficiency_tokens": 80,
}
),
"baseline": _arm(
{
"prompt_tokens": 200,
"cached_tokens": 20,
"uncached_prompt_tokens": 180,
"completion_tokens": 40,
"total_tokens": 240,
"efficiency_tokens": 220,
}
),
"delta": {
"prompt_tokens": -100,
"cached_tokens": 20,
"uncached_prompt_tokens": -120,
"completion_tokens": -20,
"total_tokens": -120,
"efficiency_tokens": -140,
"total_tokens_change_percent": -50.0,
"status": "available",
"reason_codes": [],
},
}
latency_scope = {
"with_skill": _arm({"llm_s": 10.5, "tool_exec_s": 0, "llm_ms_per_token": 87.5}),
"baseline": _arm(
{"llm_s": 20, "tool_exec_s": None, "llm_ms_per_token": 83.33},
status="partial",
measured=1,
reasons=["missing_timing_evidence"],
),
"delta": {
"llm_s": None,
"tool_exec_s": None,
"llm_ms_per_token": None,
"llm_s_change_percent": None,
"llm_ms_per_token_change_percent": None,
"status": "unavailable",
"reason": "Baseline timing is incomplete.",
"reason_codes": ["baseline_partial"],
},
}
payload = {}
for key, scope in (("token_usage", token_scope), ("latency_metrics", latency_scope)):
agent = copy.deepcopy(scope)
agent["cases"] = [{"entry_id": "case-one", **copy.deepcopy(scope)}]
agent["arm_runtime_ms"] = {
"with_skill": {"duration_ms": 35000},
"without_skill": {"duration_ms": 50000},
}
payload[key] = {
"source": "recorded_native",
"schema_version": "1.0",
"dataset": copy.deepcopy(scope),
"agents": {"agent-a": agent},
}
return payload
def test_native_tables_keep_all_recorded_columns_scopes_conditions_and_reasons():
view = _view(_payload())
html = _render(view)
assert view["has_tokens"] and view["has_latency"]
for label in (
"Dataset",
"agent-a",
"case-one",
"With skill",
"Without skill",
"Change",
"Prompt",
"Cached",
"Uncached prompt",
"Completion",
"Total",
"Efficiency Tokens",
"LLM (s)",
"Tool (s)",
"LLM ms/token",
"LLM timing",
"Arm runtime (ms)",
"120",
"240",
"-120",
"-50.00%",
"10.5",
"87.5",
"35,000",
"50,000",
"native (ATIF)",
"inferred (timestamps)",
"1/2",
"partial",
"missing_timing_evidence",
"baseline_partial",
"Baseline timing is incomplete.",
):
assert label in html
assert 'id="tier3-token-usage"' in html and 'id="tier3-latency-metrics"' in html
assert 'id="tier3-recorded-token-chart"' in html
assert "<script" not in html
def test_native_telemetry_rows_do_not_enter_the_tier1_summary_filter_scope():
html = _render(_view(_payload()))
tables = re.findall(r"<table\b[^>]*>", html)
assert len(tables) == 4
# Tier 1's global .summary-table tbody tr filter hides rows without its
# validator status attributes. These are resource measurements instead.
assert all('class="contributor-table"' in table for table in tables)
assert all("summary-table" not in table for table in tables)
assert 'id="t3-recorded-token-usage-table"' in html
assert 'id="t3-latency-table"' in html
def test_token_chart_uses_both_arms_without_counting_cached_reads_twice():
chart = _view(_payload())["token_chart"]
assert chart == {
"labels": ["agent-a · With skill", "agent-a · Without skill"],
"datasets": [
{"label": "Uncached prompt", "data": [60, 180]},
{"label": "Cached prompt", "data": [40, 20]},
{"label": "Completion", "data": [20, 40]},
],
}
def test_chart_fallback_is_visible_until_the_shared_lifecycle_draws_chart():
html = _render(_view(_payload()))
canvas = re.search(r'<canvas\b[^>]*id="tier3-recorded-token-chart"[^>]*>', html).group()
fallback = re.search(r'<p\b[^>]*id="tier3-recorded-token-chart-fallback"[^>]*>', html).group()
assert " hidden" in canvas
assert " hidden" not in fallback
assert 'id="t3-recorded-token-usage-table"' in html
assert 'id="t3-token-usage-table"' not in html
for paragraph_id in ("tier3-recorded-token-chart-description", "tier3-recorded-token-chart-fallback"):
paragraph = re.search(rf'<p\b[^>]*id="{paragraph_id}"[^>]*>.*?</p>', html).group()
assert 'href="#t3-recorded-token-usage-table"' in paragraph
assert ""#" not in paragraph
description = re.search(r'<p\b[^>]*id="tier3-recorded-token-chart-description"[^>]*>', html).group()
assert 'class="chart-description"' in description
def test_zero_is_recorded_but_unknown_counters_and_unavailable_deltas_stay_na():
payload = _payload()
scope = payload["token_usage"]["dataset"]
scope["with_skill"]["totals"] = {"total_tokens": 0, "completion_tokens": 0}
scope["baseline"]["coverage"] = {}
scope["delta"].update(status="unavailable", reason="Scopes differ.", total_tokens=999)
html = _render(_view(payload))
dataset = html.split("Dataset", 1)[1].split("agent-a", 1)[0]
assert ">0<" in dataset
assert "N/A" in dataset and "?/?" in dataset
assert "Scopes differ." in dataset and "999" not in dataset
def test_partial_token_groups_are_labelled_and_missing_chart_components_remain_unknown():
payload = _payload()
arm = payload["token_usage"]["agents"]["agent-a"]["with_skill"]
arm["coverage"].update(status="partial", measured_attempts=1, reason_codes=["missing_tokens"])
chart = _view(payload)["token_chart"]
assert chart["labels"][0] == "agent-a · With skill (partial)"
del arm["totals"]["uncached_prompt_tokens"]
chart = _view(payload)["token_chart"]
assert chart["datasets"][0]["data"][0] is None
assert "missing_tokens" in _render(_view(payload))
def test_recorded_timing_precision_is_preserved_without_recalculation():
payload = _payload()
payload["latency_metrics"]["dataset"]["with_skill"]["totals"]["llm_ms_per_token"] = 87.123456789
assert "87.123456789" in _render(_view(payload))
def test_native_partial_delta_preserves_absolute_change_when_percent_is_unavailable():
payload = _payload()
delta = payload["token_usage"]["dataset"]["delta"]
delta.update(
status="partial",
total_tokens=120,
total_tokens_change_percent=None,
reason="Baseline is zero; percentage unavailable.",
reason_codes=["baseline_zero"],
)
html = _render(_view(payload))
dataset = html.split("Dataset", 1)[1].split("agent-a", 1)[0]
assert "+120" in dataset and "baseline_zero" in dataset
assert "Baseline is zero; percentage unavailable." in dataset
assert "%" not in dataset
def test_view_preserves_each_family_within_byte_limit_for_escape_heavy_text():
payload = _payload()
for family in ("token_usage", "latency_metrics"):
agent = payload[family]["agents"].pop("agent-a")
for scope in (agent, agent["cases"][0], payload[family]["dataset"]):
for condition in ("with_skill", "baseline"):
scope[condition]["coverage"]["reason_codes"] = ["\U0001f600<>\"'&" * 100] * 100
scope["delta"]["reason"] = "\U0001f600<>\"'&" * 100
agent["cases"] *= 1000
payload[family]["agents"] = {f"agent-{i}" + "\U0001f600<>\"'&" * 100: agent for i in range(1000)}
view = _view(payload)
assert len(json.dumps(view).encode()) <= 256 * 1024
assert len(_render(view).encode()) <= 512 * 1024
assert "Display text shortened" in _render(view)
def test_combined_projection_byte_bound_includes_unicode_agent_labels_and_chart():
payload = _payload()
for family in ("token_usage", "latency_metrics"):
agent = payload[family]["agents"]["agent-a"]
agent["cases"] *= 1000
payload[family]["agents"] = {"\U0001f600" * 240 + str(i): agent for i in range(32)}
view = _view(payload)
assert len(json.dumps(view).encode()) <= 256 * 1024
def test_html_expansion_of_escape_heavy_reasons_stays_bounded():
payload = _payload()
for family in ("token_usage", "latency_metrics"):
agent = payload[family]["agents"]["agent-a"]
for scope in (agent, agent["cases"][0], payload[family]["dataset"]):
for condition in ("with_skill", "baseline"):
scope[condition]["coverage"]["reason_codes"] = ["<" * 240] * 8
scope["delta"]["reason"] = "<" * 240
agent["cases"] *= 1000
payload[family]["agents"] = {f"agent-{i}": agent for i in range(32)}
html = _render(_view(payload))
assert len(html.encode()) <= 512 * 1024
assert "omitted" in html
@pytest.mark.parametrize(
"bad",
[float("nan"), float("inf"), -1, True, "123", {}, 10**10000],
ids=["nan", "infinite", "negative", "boolean", "string", "mapping", "huge-int"],
)
def test_bad_numeric_values_never_become_recorded_measurements(bad):
payload = {
"token_usage": {"dataset": {"with_skill": _arm({"total_tokens": bad})}},
"latency_metrics": {"dataset": {"with_skill": _arm({"llm_s": bad})}},
}
html = _render(_view(payload))
assert "N/A" in html
assert ">nan<" not in html and ">inf<" not in html and ">True<" not in html
assert ">123<" not in html and ">-1<" not in html
def test_untrusted_text_is_escaped_and_projection_has_no_mutable_canonical_aliases():
payload = _payload()
attack = '<img src=x onerror="alert(1)">\ud800'
payload["token_usage"]["agents"][attack] = payload["token_usage"]["agents"].pop("agent-a")
payload["latency_metrics"]["dataset"]["delta"]["reason"] = attack
before = copy.deepcopy(payload)
view = _view(payload)
html = _render(view)
assert "<img" not in html and "<img" in html
assert "\ud800" not in html
json.dumps(view, ensure_ascii=False, allow_nan=False).encode("utf-8")
view["token_chart"]["datasets"][0]["data"].append(12345)
view["token_sections"].clear()
assert payload == before
def test_all_telemetry_text_is_redacted_before_shortening_and_controls_are_removed():
payload = _payload()
secret = "syntheticcredential123"
unsafe = f"\x1b[31mBearer {secret}\x1b[0m\x00\ud800\u202e"
for family in ("token_usage", "latency_metrics"):
usage = payload[family]
usage["agents"][unsafe] = usage["agents"].pop("agent-a")
agent = usage["agents"][unsafe]
usage["source"] = unsafe
agent["cases"][0]["entry_id"] = unsafe
for condition in ("with_skill", "baseline"):
agent[condition]["coverage"]["reason_codes"] = [unsafe]
usage["dataset"]["delta"]["reason"] = "x " * 110 + f" api_key={secret}"
before = copy.deepcopy(payload)
view = _view(payload)
html = _render(view)
projected = json.dumps(view, ensure_ascii=False)
assert secret not in projected and secret not in html
assert "redacted" in projected and "redacted" in html
assert "\x1b" not in projected and "\x00" not in projected and "\ud800" not in projected
assert "\u202e" not in projected
assert payload == before
def test_large_inputs_are_bounded_without_walking_all_cases_or_extra_payloads():
class BoundedCases(list):
def __iter__(self):
raise AssertionError("unbounded list traversal")
payload = _payload()
agent = payload["token_usage"]["agents"]["agent-a"]
agent["cases"] = BoundedCases([{"entry_id": "x" * 10000, **copy.deepcopy(agent["cases"][0])}] * 100000)
payload["token_usage"]["agents"] = {f"agent-{i}": agent for i in range(1000)}
payload["token_usage"]["source"] = "x" * 1000000
payload["irrelevant"] = {"recursive": payload}
view = _view(payload)
assert len(json.dumps(view).encode()) <= 256 * 1024
assert len(_render(view).encode()) <= 512 * 1024
assert "omitted" in _render(view).lower()
assert len(view["token_chart"]["labels"]) <= 64
@pytest.mark.parametrize("payload", [None, [], "oops", {}, {"agents": {"agent-a": {"resources": {"tokens": 0}}}}])
def test_absent_native_telemetry_keeps_public_resources_separate(payload):
view = _view(payload)
assert view == {
"token_sections": [],
"latency_sections": [],
"token_chart": None,
"has_tokens": False,
"has_latency": False,
}
assert _render(view).strip() == ""