Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

feat(tier3): write deductions as plain-language reasons with evidence (#399)

Christopher Kevin9feffaa39643
diff --git a/.gitleaksignore b/.gitleaksignore
new file mode 100644
index 00000000..2cb0292d
--- /dev/null
+++ b/.gitleaksignore
@@ -0,0 +1 @@
+08b38a1d499d894b009a761a5db04b818ef1af70:tests/reporting/test_tier3_failure_reasons.py:curl-auth-user:581
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c90f3d2b..96d10119 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -118,6 +118,10 @@ All notable changes to SkillEvaluator are documented in this file.
### Changed
+- Write Tier 3 accuracy deductions as plain-language reasons, identify findings
+ without trajectory evidence, and include expected/observed text only when
+ both values are recorded.
+
- Tier 3 LLM suggestions and insight recommendations now keep only evidence
references the run recorded (cited references that don't resolve are
dropped rather than shown as evidence) and keep one copy of a repeated
@@ -126,7 +130,6 @@ All notable changes to SkillEvaluator are documented in this file.
pass threshold, are dropped. Both prompts ask for distinct actions and
label untested causes, such as switching to a different model, as
hypotheses or experiments.
-
- Tier 3 reports missing skill runtime dependencies as environment problems,
with declaration and custom Dockerfile guidance. Unrecovered environment
failures leave the trial UNSCORED instead of reducing the skill's score;
@@ -134,7 +137,6 @@ All notable changes to SkillEvaluator are documented in this file.
- Document per-arm Tier 3 dimension applicability and test how N/A baseline
metrics are handled in Overall scores and Skill Lift comparisons.
-
- Run GitLab Tier 2 without Docker-in-Docker while keeping TLS-protected Docker
for Tier 3. Both native tiers retain shared caller timeout and runner-tag
overrides, advisory outcomes, and aggregate artifacts; local mode still
diff --git a/docs/reports.mdx b/docs/reports.mdx
index 7ac18a81..aab09569 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -16,11 +16,14 @@ lift data that `view` and `compare` read back later.
evaluation framework that runs those Tier 3 trials. SkillEvaluator collects its
job artifacts into the results tree described on this page.
+Tier 3 findings describe failed accuracy criteria in plain language and mark
+when trajectory evidence is unavailable. Expected and observed text appears
+only when both values were recorded in the result.
+
Tier 3 results include a `dataset_sufficiency` policy assessment in `result.json`.
The status is sufficient only when coverage, pass-rate reliability, and paired
lift-comparison requirements are met; otherwise it is provisional and does not
change the evaluation verdict.
-
## Retained three-tier CI evidence
The opt-in all-tier workflows retain one aggregate artifact for 14 days. GitHub
diff --git a/src/skillevaluator/evaluation/tier3_findings.py b/src/skillevaluator/evaluation/tier3_findings.py
index 740082e7..327cd216 100644
--- a/src/skillevaluator/evaluation/tier3_findings.py
+++ b/src/skillevaluator/evaluation/tier3_findings.py
@@ -18,7 +18,7 @@ inflate the artifact and could carry credentials an agent echoed.
from __future__ import annotations
import re
-from collections.abc import Iterator
+from collections.abc import Collection, Iterator
from itertools import islice
from typing import Any
@@ -160,14 +160,21 @@ def _failed_check_reason(key: str, value: dict[str, Any]) -> str:
if isinstance(text, str) and text.strip():
return text
label = key.replace("_", " ").replace("-", " ").strip()
- if not label:
- return "Check failed"
- return f"{label[0].upper()}{label[1:]} failed"
+ return f"Check `{label}` did not pass; no reason was recorded"
-def _reason_candidates(detail: dict[str, Any], *, include_results: bool) -> Iterator[str]:
+def _reason_candidates(
+ detail: dict[str, Any],
+ *,
+ include_results: bool,
+ metric: str | None,
+ reward: dict[str, Any] | None,
+) -> Iterator[str]:
"""Yield candidate reason strings across the known grader detail shapes."""
- from skillevaluator.tier3.harbor.report import _applicable_accuracy_criteria
+ from skillevaluator.tier3.harbor.report import (
+ _applicable_accuracy_criteria,
+ _expected_observed_suffix,
+ )
for field in ("reason", "message"):
if isinstance(detail.get(field), str):
@@ -193,12 +200,15 @@ def _reason_candidates(detail: dict[str, Any], *, include_results: bool) -> Iter
# excluded from the score (such as skill identification when no skill is
# expected) did not fail it.
criteria = _applicable_accuracy_criteria(detail)
- failed_criteria = sorted(
- str(name)[:128] for name, passed in islice(criteria.items(), _MAX_DETAIL_ITEMS) if passed is False
- )
+ failed_criteria = [str(name) for name, passed in islice(criteria.items(), _MAX_DETAIL_ITEMS) if passed is False]
if failed_criteria:
yield f"Failed criteria: {', '.join(failed_criteria)}"
+ if metric in ("accuracy", "goal_accuracy"):
+ suffix = _expected_observed_suffix(detail, reward)
+ if suffix:
+ yield suffix
+
# Other graders nest one dict per named check.
for key, value in islice(detail.items(), _MAX_DETAIL_ITEMS):
if key not in _STRUCTURED_DETAIL_KEYS and _failed(value):
@@ -210,7 +220,14 @@ def _reason_candidates(detail: dict[str, Any], *, include_results: bool) -> Iter
yield detail["attribution_explanation"]
-def failure_reason(detail: object, *, include_results: bool = True) -> str:
+def failure_reason(
+ detail: object,
+ *,
+ include_results: bool = True,
+ metric: str | None = None,
+ reward: dict[str, Any] | None = None,
+ omit: Collection[str] = (),
+) -> str:
"""Return one short, deduplicated, redacted reason for a failing metric.
``include_results=False`` leaves out failing behavior results, for evaluator
@@ -219,13 +236,26 @@ def failure_reason(detail: object, *, include_results: bool = True) -> str:
if not isinstance(detail, dict):
return ""
parts: list[str] = []
+ omitted_texts: set[str] = set()
+ for text in omit:
+ cleaned = sanitize_grader_text(text)
+ if cleaned:
+ omitted_texts.add(cleaned)
+ omitted_texts.update(part for part in cleaned.split("; ") if part)
# Only non-blank candidates count toward the cap, so entries without text
# cannot crowd out the criteria, checks, or attribution that follow them.
candidates = (
- candidate for candidate in _reason_candidates(detail, include_results=include_results) if candidate.strip()
+ cleaned
+ for candidate in _reason_candidates(
+ detail,
+ include_results=include_results,
+ metric=metric,
+ reward=reward,
+ )
+ if (cleaned := sanitize_grader_text(candidate))
+ and not any(cleaned == omitted or cleaned in omitted for omitted in omitted_texts)
)
- for candidate in islice(candidates, _MAX_REASON_CANDIDATES):
- cleaned = sanitize_grader_text(candidate)
+ for cleaned in islice(candidates, _MAX_REASON_CANDIDATES):
if cleaned and cleaned not in parts:
parts.append(cleaned)
if len(parts) >= _MAX_REASON_PARTS:
@@ -235,6 +265,13 @@ def failure_reason(detail: object, *, include_results: bool = True) -> str:
return sanitize_grader_text("; ".join(parts)[: 4 * MAX_REASON_CHARS], MAX_REASON_CHARS)
+def evidence_unavailable_reason(reward: dict[str, Any]) -> str:
+ failure = sanitize_grader_text(reward.get("trajectory_detail") or reward.get("error"), MAX_REASON_CHARS)
+ if failure:
+ return f"Trajectory evidence could not be loaded: {failure}"
+ return "No trajectory evidence was recorded for this metric"
+
+
def failure_excerpt(detail: object) -> str:
"""Return a short, redacted excerpt of the artifact that triggered the failure."""
if not isinstance(detail, dict):
diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.py
index 423e2bc0..75cd363b 100644
--- a/src/skillevaluator/evaluation/tier3_report.py
+++ b/src/skillevaluator/evaluation/tier3_report.py
@@ -2604,6 +2604,7 @@ def _metric_evidence(
report_budget: _ReportBudget,
sampling: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
+ from skillevaluator.tier3.eval_core.atif_helpers import _METRIC_EVIDENCE_REF_METRICS
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, metric_set_for_reward
if report_budget.evidence_remaining <= 0:
@@ -2635,7 +2636,8 @@ def _metric_evidence(
score_failed = raw_score is not None and raw_score < TRIAL_PASS_THRESHOLD
notes: list[str] = []
- reason = sanitize_grader_text(detail.get("reason"))
+ raw_grader_reason = detail.get("reason")
+ reason = sanitize_grader_text(raw_grader_reason)
if reason:
notes.append(reason[:512])
@@ -2664,6 +2666,10 @@ def _metric_evidence(
"checks": checks,
"evidence_refs": _compact_evidence_refs(detail.get("evidence_refs")),
}
+ if score_failed and metric in _METRIC_EVIDENCE_REF_METRICS and not entry["evidence_refs"]:
+ from skillevaluator.evaluation.tier3_findings import evidence_unavailable_reason
+
+ entry["evidence_unavailable_reason"] = evidence_unavailable_reason(reward)
# The built reason and excerpt are not part of the fingerprint: attempts
# of one case often quote different commands under the same grader
# reason, and they still collapse into one entry with ``occurrences``.
@@ -2685,11 +2691,18 @@ def _metric_evidence(
output_truncated = True
continue
if score_failed:
- # Graders that record only a message, findings, or failed criteria
- # still explain a failing score. The built reason stays separate from
- # the grader's own notes so size limits can drop it first, and leaves
- # out failing results, which the card already lists as ``failures``.
- if not notes and (fallback_reason := failure_reason(detail, include_results=False)):
+ # Graders that record only findings or failed criteria still explain
+ # a failing score. Notes already show the grader's own reason, so the
+ # separate built reason omits that text and leaves out failing
+ # results, which the card already lists as ``failures``.
+ omitted_reasons = (raw_grader_reason, *notes) if isinstance(raw_grader_reason, str) else tuple(notes)
+ if fallback_reason := failure_reason(
+ detail,
+ include_results=False,
+ metric=metric,
+ reward=reward,
+ omit=omitted_reasons,
+ ):
entry["reason"] = fallback_reason
if excerpt := failure_excerpt(detail):
entry["excerpt"] = excerpt
@@ -3198,7 +3211,7 @@ def _trial_score_reasons(
continue
if lowest is not None and row_score >= lowest[0]:
continue
- reason = failure_reason(_metric_detail(row, metric))
+ reason = failure_reason(_metric_detail(row, metric), metric=metric, reward=row)
if reason:
lowest = (row_score, reason)
if lowest is not None:
diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.py
index fbbe1d29..9d5ee252 100644
--- a/src/skillevaluator/reporting/html.py
+++ b/src/skillevaluator/reporting/html.py
@@ -801,8 +801,13 @@ def _sanitize_tier3_pass_at_k(value: object) -> dict[str, Any]:
return pass_at_k
-def _sanitize_tier3_evaluator_card(value: dict[str, Any]) -> dict[str, Any] | None:
+def _sanitize_tier3_evaluator_card(
+ value: dict[str, Any],
+ *,
+ include_evidence_unavailable_default: bool = False,
+) -> dict[str, Any] | None:
from skillevaluator.evaluation.tier3_findings import FAILURE_REASON_EVIDENCE_KEYS
+ from skillevaluator.tier3.eval_core.atif_helpers import _METRIC_EVIDENCE_REF_METRICS
card = dict(value)
card["with_skill"] = _finite_number(card.get("with_skill"), minimum=0.0, maximum=1.0)
@@ -825,6 +830,20 @@ def _sanitize_tier3_evaluator_card(value: dict[str, Any]) -> dict[str, Any] | No
for key in FAILURE_REASON_EVIDENCE_KEYS:
if not (isinstance(entry.get(key), str) and entry[key]):
entry.pop(key, None)
+ if (
+ include_evidence_unavailable_default
+ and card.get("id") in _METRIC_EVIDENCE_REF_METRICS
+ and entry["score"] is not None
+ and entry["score"] < 0.8
+ and not entry["evidence_refs"]
+ ):
+ unavailable_reason = entry.get("evidence_unavailable_reason")
+ if isinstance(unavailable_reason, str) and unavailable_reason:
+ entry["evidence_unavailable_reason"] = unavailable_reason[:512]
+ else:
+ entry["evidence_unavailable_reason"] = "No trajectory evidence was recorded for this metric"
+ else:
+ entry.pop("evidence_unavailable_reason", None)
evidence.append(entry)
card["evidence"] = evidence
sampling = card.get("evidence_sampling") if isinstance(card.get("evidence_sampling"), dict) else {}
@@ -863,7 +882,7 @@ def _sanitize_tier3_agent(value: dict[str, Any]) -> dict[str, Any]:
agent["evaluator_cards"] = [
card
for item in _dict_list(agent.get("evaluator_cards"))
- if (card := _sanitize_tier3_evaluator_card(item)) is not None
+ if (card := _sanitize_tier3_evaluator_card(item, include_evidence_unavailable_default=True)) is not None
]
agent["findings"] = [_sanitize_tier3_dimension(item) for item in _dict_list(agent.get("findings"))]
agent["pass_at_k"] = _sanitize_tier3_pass_at_k(agent.get("pass_at_k"))
diff --git a/src/skillevaluator/reporting/templates/report.html.j2 b/src/skillevaluator/reporting/templates/report.html.j2
index b30307e6..751e4b71 100644
--- a/src/skillevaluator/reporting/templates/report.html.j2
+++ b/src/skillevaluator/reporting/templates/report.html.j2
@@ -3498,7 +3498,7 @@
{% set eid = ev.entry_id or 'trial' %}
{% if eid not in dedup_map %}
{% set _ = dedup_order.append(eid) %}
- {% set _ = dedup_map.update({eid: {'count': 0, 'sum': 0.0, 'with_score': 0, 'notes': [], 'failures': [], 'checks': [], 'evidence_refs': [], 'excerpt': none, 'reason_score': none}}) %}
+ {% set _ = dedup_map.update({eid: {'count': 0, 'sum': 0.0, 'with_score': 0, 'notes': [], 'failures': [], 'checks': [], 'evidence_refs': [], 'evidence_unavailable_reason': none, 'excerpt': none, 'reason': none, 'reason_score': none}}) %}
{% endif %}
{% set entry = dedup_map[eid] %}
{% set ev_occurrences = ev.occurrences or 1 %}
@@ -3506,17 +3506,19 @@
{% if ev.score is not none %}
{% set _ = entry.update({'sum': entry.sum + (ev.score * ev_occurrences), 'with_score': entry.with_score + ev_occurrences}) %}
{% endif %}
- {# A built reason stands in for grader notes when the grader recorded none. #}
- {% set ev_notes = ev.notes or ([ev.reason] if ev.reason else []) %}
- {% if (ev_notes or ev.failures or ev.excerpt) and ev.score is not none and (entry.reason_score is none or ev.score < entry.reason_score) %}
- {% set _ = entry.update({'notes': ev_notes, 'failures': ev.failures or [], 'excerpt': ev.excerpt or none, 'reason_score': ev.score}) %}
+ {# Keep built reasons separate from the bounded grader notes. #}
+ {% set ev_notes = ev.notes or [] %}
+ {% if (ev_notes or ev.reason or ev.failures or ev.excerpt) and ev.score is not none and (entry.reason_score is none or ev.score < entry.reason_score) %}
+ {% set _ = entry.update({'notes': ev_notes, 'reason': ev.reason or none, 'failures': ev.failures or [], 'excerpt': ev.excerpt or none, 'reason_score': ev.score}) %}
{% elif entry.reason_score is none %}
{% if ev_notes and not entry.notes %}{% set _ = entry.update({'notes': ev_notes}) %}{% endif %}
+ {% if ev.reason and not entry.reason %}{% set _ = entry.update({'reason': ev.reason}) %}{% endif %}
{% if ev.failures and not entry.failures %}{% set _ = entry.update({'failures': ev.failures}) %}{% endif %}
{% if ev.excerpt and not entry.excerpt %}{% set _ = entry.update({'excerpt': ev.excerpt}) %}{% endif %}
{% endif %}
{% if ev.checks and not entry.checks %}{% set _ = entry.update({'checks': ev.checks}) %}{% endif %}
{% if ev.evidence_refs and not entry.evidence_refs %}{% set _ = entry.update({'evidence_refs': ev.evidence_refs}) %}{% endif %}
+ {% if ev.evidence_unavailable_reason and not entry.evidence_unavailable_reason %}{% set _ = entry.update({'evidence_unavailable_reason': ev.evidence_unavailable_reason}) %}{% endif %}
{% endfor %}
{% set evidence_totals = namespace(trials=0) %}
{% for ev in card.evidence %}
@@ -3539,7 +3541,7 @@
{% if ev.notes %}{{ (ev.notes[:2] | join('; '))[:160] }}{% endif %}
{% if ev.failures %}<span style="color:var(--danger-color);">{{ (ev.failures[:2] | join('; '))[:120] }}</span>{% endif %}
{% if not ev.notes and not ev.failures and ev.checks %}{{ (ev.checks | length) }} checks{% endif %}
- {% if ev.evidence_refs %}<span class="t3-ev-refs">Evidence: {{ ev.evidence_refs[:3] | join('; ') }}</span>{% endif %}
+ {% if ev.evidence_refs %}<span class="t3-ev-refs">Evidence: {{ ev.evidence_refs[:3] | join('; ') }}</span>{% elif ev.evidence_unavailable_reason %}<span class="t3-ev-refs">Evidence unavailable — {{ ev.evidence_unavailable_reason }}</span>{% endif %}
</span>
<span class="t3-ev-count">×{{ ev.count }}</span>
{% if avg_score is not none %}
@@ -3547,6 +3549,7 @@
{% else %}
<span class="t3-bar-value" style="color:var(--text-secondary);">N/A</span>
{% endif %}
+ {% if ev.reason %}<div class="t3-ev-detail t3-ev-reason">{{ ev.reason }}</div>{% endif %}
{% if ev.excerpt %}<div class="t3-ev-detail"><code class="t3-ev-excerpt">{{ ev.excerpt }}</code></div>{% endif %}
</li>
{% endfor %}
diff --git a/src/skillevaluator/tier3/eval_core/llm_judge.py b/src/skillevaluator/tier3/eval_core/llm_judge.py
index 8b9c4b94..22767dcb 100644
--- a/src/skillevaluator/tier3/eval_core/llm_judge.py
+++ b/src/skillevaluator/tier3/eval_core/llm_judge.py
@@ -630,6 +630,13 @@ _ACCURACY_CRITERIA = (
"if the response gives an answer and false if it gives none.",
),
)
+_ACCURACY_DEDUCTION_SENTENCES = {
+ "ACTION_CORRECT": "The agent did not take the action the expected answer calls for",
+ "FACTUALLY_ACCURATE": "The answer's facts do not match the expected answer",
+ "TASK_ADDRESSED": "The answer does not address what the user asked",
+ "ACTIONABLE": "The answer does not give the user a result they can act on",
+ "FORMAT_FOLLOWED": "The answer does not follow the output format the question requires",
+}
_ACCURACY_CRITERIA_KEYS = frozenset(key for key, _question in _ACCURACY_CRITERIA)
ACCURACY_JSON_SCHEMA: dict[str, Any] = {
diff --git a/src/skillevaluator/tier3/harbor/report.py b/src/skillevaluator/tier3/harbor/report.py
index 3d4393a0..44dc8977 100644
--- a/src/skillevaluator/tier3/harbor/report.py
+++ b/src/skillevaluator/tier3/harbor/report.py
@@ -17,7 +17,7 @@ from pathlib import Path
from typing import Any
from skillevaluator.evidence import evidence_ref_identity
-from skillevaluator.tier3.eval_core.llm_judge import _redact_configured_credentials
+from skillevaluator.tier3.eval_core.llm_judge import _ACCURACY_DEDUCTION_SENTENCES, _redact_configured_credentials
from skillevaluator.tier3.harbor import report_data
from skillevaluator.tier3.harbor.metrics import (
DEFAULT_METRICS,
@@ -259,6 +259,8 @@ def _extract_findings(
rewards_complete: bool = True,
) -> list[dict[str, Any]]:
"""Extract actionable findings from reward.json details across trials."""
+ from skillevaluator.tier3.eval_core.atif_helpers import _METRIC_EVIDENCE_REF_METRICS
+
findings: list[dict[str, Any]] = []
metric_names = _finding_metric_names(rewards)
@@ -277,6 +279,7 @@ def _extract_findings(
"score": score,
"detail": detail,
"entry_id": reward.get("entry_id", "?"),
+ "reward": reward,
}
)
if metric_values:
@@ -330,16 +333,17 @@ def _extract_findings(
else:
reasons = _collect_fail_reasons(metric, trials)
severity = "critical" if avg_score < 0.4 else "warning"
- findings.append(
- {
- "metric": metric,
- "label": label,
- "severity": severity,
- "score": avg_score,
- "reasons": reasons[:4],
- "evidence_refs": _refs[:_MAX_FINDING_EVIDENCE_REFS],
- }
- )
+ finding = {
+ "metric": metric,
+ "label": label,
+ "severity": severity,
+ "score": avg_score,
+ "reasons": reasons[:4],
+ "evidence_refs": _refs[:_MAX_FINDING_EVIDENCE_REFS],
+ }
+ if metric in _METRIC_EVIDENCE_REF_METRICS and not finding["evidence_refs"]:
+ finding["evidence_unavailable_reason"] = _evidence_unavailable_reason(trials)
+ findings.append(finding)
return findings
@@ -376,6 +380,11 @@ def _render_findings_body(findings: list[dict[str, Any]]) -> Any:
for ref in (finding.get("evidence_refs") or [])[:3]:
ref_text = ref if isinstance(ref, str) else _compact_evidence_ref(ref)
body.append(f" evidence: {strip_terminal_controls(ref_text)}\n", style="dim")
+ if finding["severity"] != "ok" and finding.get("evidence_unavailable_reason"):
+ body.append(
+ f" evidence: unavailable — {strip_terminal_controls(str(finding['evidence_unavailable_reason']))}\n",
+ style="dim",
+ )
body.append("\n")
return body
@@ -431,6 +440,7 @@ def _collect_fail_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
detail = trial["detail"]
if not isinstance(detail, dict):
continue
+ trial_reason_start = len(reasons)
if metric == "behavior_check":
results = detail.get("results")
@@ -444,7 +454,7 @@ def _collect_fail_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
criteria = _applicable_accuracy_criteria(detail)
for crit, passed in criteria.items():
if not passed:
- reasons.append(f"{str(crit)[:480]} failed")
+ reasons.append(_accuracy_criterion_failure_reason(crit))
if reason := _bounded_reason_text(detail.get("reason")):
reasons.append(reason)
@@ -472,7 +482,10 @@ def _collect_fail_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
elif metric == "skill_execution":
for check_name, check_data in detail.items():
if isinstance(check_data, dict) and not check_data.get("passed", True):
- reason = _bounded_reason_text(check_data.get("reason")) or f"{str(check_name)[:480]} failed"
+ reason = _bounded_reason_text(check_data.get("reason")) or (
+ f"Check `{_bounded_reason_text(str(check_name).replace('_', ' '), max_len=480)}` "
+ "did not pass; no reason was recorded"
+ )
if check_data.get("status") == "environment_invalid":
reason = "[environment] " + reason
reasons.append(reason)
@@ -488,7 +501,10 @@ def _collect_fail_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
elif metric == "skill_efficiency":
for check_name, check_data in detail.items():
if isinstance(check_data, dict) and not check_data.get("passed", True):
- reason = _bounded_reason_text(check_data.get("reason")) or f"{str(check_name)[:480]} failed"
+ reason = _bounded_reason_text(check_data.get("reason")) or (
+ f"Check `{_bounded_reason_text(str(check_name).replace('_', ' '), max_len=480)}` "
+ "did not pass; no reason was recorded"
+ )
reasons.append(reason)
else:
@@ -504,6 +520,12 @@ def _collect_fail_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
):
reasons.append(reason)
+ if metric in ("accuracy", "goal_accuracy") and len(reasons) > trial_reason_start:
+ reward = trial.get("reward")
+ suffix = _expected_observed_suffix(detail, reward if isinstance(reward, dict) else None)
+ if suffix:
+ reasons[-1] = f"{reasons[-1]}; {suffix}"
+
return _dedupe_report_reasons(reasons)
@@ -578,6 +600,55 @@ def _bounded_reason_text(value: Any, *, max_len: int = 512) -> str:
return _bounded_report_reason(value)[:max_len]
+def _accuracy_criterion_failure_reason(criterion: Any) -> str:
+ key = _bounded_reason_text(str(criterion), max_len=480)
+ sentence = _ACCURACY_DEDUCTION_SENTENCES.get(key)
+ return f"{sentence} (criterion {key})" if sentence else f"Criterion {key} was not met"
+
+
+def _expected_observed_suffix(detail: dict[str, Any], reward: dict[str, Any] | None = None) -> str:
+ from skillevaluator.evaluation.tier3_findings import sanitize_grader_text
+
+ sources: list[dict[str, Any]] = [detail]
+ reward = reward if isinstance(reward, dict) else {}
+ reward_details = reward.get("details")
+ if isinstance(reward_details, dict):
+ sources.append(reward_details)
+ sources.append(reward)
+
+ def value_for(keys: tuple[str, ...]) -> Any:
+ for source in sources:
+ for key in keys:
+ if key in source and source[key] is not None:
+ return source[key]
+ return None
+
+ expected = sanitize_grader_text(
+ _bounded_reason_text(value_for(("ground_truth", "expected_answer")), max_len=160),
+ 160,
+ )
+ observed = sanitize_grader_text(_bounded_reason_text(value_for(("final_answer",)), max_len=160), 160)
+ if not expected or not observed:
+ return ""
+ return f"Expected: {expected}; observed: {observed}"
+
+
+def _evidence_unavailable_reason(trials: list[dict[str, Any]]) -> str:
+ from skillevaluator.evaluation.tier3_findings import sanitize_grader_text
+
+ for trial in trials:
+ reward = trial.get("reward")
+ if not isinstance(reward, dict):
+ continue
+ failure = sanitize_grader_text(
+ reward.get("trajectory_detail") or reward.get("error"),
+ _REPORT_REASON_LIMIT,
+ )
+ if failure:
+ return f"Trajectory evidence could not be loaded: {failure}"
+ return "No trajectory evidence was recorded for this metric"
+
+
def _build_evidence_ref_lookup(rewards: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""Build a lookup from each stable compact evidence key to its full dict ref.
@@ -976,6 +1047,8 @@ def _bounded_findings_payload(
suggestions_v2: list[dict[str, Any]] | None,
) -> dict[str, Any]:
"""Project a findings artifact into a fixed, browser-loadable schema."""
+ from skillevaluator.tier3.eval_core.atif_helpers import _METRIC_EVIDENCE_REF_METRICS
+
bounded_findings: list[dict[str, Any]] = []
for finding in findings[:_MAX_FINDINGS]:
if not isinstance(finding, dict):
@@ -1008,6 +1081,18 @@ def _bounded_findings_payload(
],
}
)
+ if (
+ bounded_findings[-1]["severity"] != "ok"
+ and bounded_findings[-1]["metric"] in _METRIC_EVIDENCE_REF_METRICS
+ and not bounded_findings[-1]["evidence_refs"]
+ ):
+ unavailable_reason = finding.get("evidence_unavailable_reason") or (
+ "No trajectory evidence was recorded for this metric"
+ )
+ bounded_findings[-1]["evidence_unavailable_reason"] = _bounded_json_text(
+ unavailable_reason,
+ max_encoded_bytes=512,
+ )
bounded_v2: list[dict[str, Any]] = []
for suggestion in (suggestions_v2 or [])[:_MAX_SUGGESTIONS]:
diff --git a/tests/reporting/test_tier3_failure_reasons.py b/tests/reporting/test_tier3_failure_reasons.py
index 8e5629f7..e3b217f1 100644
--- a/tests/reporting/test_tier3_failure_reasons.py
+++ b/tests/reporting/test_tier3_failure_reasons.py
@@ -22,6 +22,7 @@ from skillevaluator.evaluation import tier3_report
from skillevaluator.evaluation.tier3_findings import (
_MAX_SCAN_CHARS,
MAX_REASON_CHARS,
+ evidence_unavailable_reason,
failure_excerpt,
failure_reason,
failure_reason_text_slots,
@@ -34,6 +35,7 @@ from skillevaluator.reporting import HTMLReporter
from skillevaluator.reporting import html as html_module
from skillevaluator.reporting.dashboard_data import _trial_coverage_truncated
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS
+from skillevaluator.tier3.harbor.report import _expected_observed_suffix
METRICS = list(DEFAULT_METRICS)
DESTRUCTIVE_REASON = "Agent executed potentially destructive command: rm -rf"
@@ -378,7 +380,11 @@ def test_message_only_detail_explains_failing_score() -> None:
assert card["evidence"][0]["notes"] == []
assert card["evidence"][0]["reason"] == "No tool calls in trajectory"
cards_html = _render(payload).split('class="t3-eval-card', 1)[1]
- assert re.search(r'<span class="t3-ev-notes">\s*No tool calls in trajectory', cards_html)
+ reason_index = cards_html.index('<div class="t3-ev-detail t3-ev-reason">')
+ row_start = cards_html.rfind("<li>", 0, reason_index)
+ row_end = cards_html.index("</li>", reason_index)
+ row = cards_html[row_start:row_end]
+ assert row.index('<span class="t3-bar-value"') < row.index('<div class="t3-ev-detail t3-ev-reason">')
def test_passing_attempt_does_not_hide_failing_attempt_reason() -> None:
@@ -522,11 +528,28 @@ def test_reason_names_failed_checks_criteria_and_attribution() -> None:
)
assert reason == (
- "Failed criteria: ACTIONABLE, TASK_ADDRESSED; Command injection check failed; "
+ "Failed criteria: TASK_ADDRESSED, ACTIONABLE; "
+ "Check `command injection check` did not pass; no reason was recorded; "
"The skill instructions requested the command."
)
+def test_reason_retains_all_failed_criteria_compactly() -> None:
+ reason = failure_reason(
+ {
+ "reason": "Ran the right command but reported the wrong installed version.",
+ "criteria": {"FACTUALLY_ACCURATE": False, "ACTIONABLE": False},
+ },
+ metric="accuracy",
+ )
+
+ assert reason == (
+ "Ran the right command but reported the wrong installed version.; "
+ "Failed criteria: FACTUALLY_ACCURATE, ACTIONABLE"
+ )
+ assert len(reason) <= MAX_REASON_CHARS
+
+
def test_reason_is_bounded_and_marks_truncation() -> None:
reason = failure_reason(
{
@@ -550,10 +573,134 @@ def test_reason_ignores_criteria_excluded_from_the_score() -> None:
"excluded_criteria": ["SKILL_IDENTIFIED"],
}
- assert failure_reason(detail) == "Answer omitted the retry limit.; Failed criteria: FACTUALLY_ACCURATE"
+ assert failure_reason(detail) == ("Answer omitted the retry limit.; Failed criteria: FACTUALLY_ACCURATE")
assert failure_reason({"criteria": {"SKILL_IDENTIFIED": False}, "excluded_criteria": ["SKILL_IDENTIFIED"]}) == ""
+def test_expected_observed_suffix_redacts_command_line_credentials() -> None:
+ mysql_password = "dummy"
+ curl_password = "dummy"
+ token = "dummy"
+ curl_auth = "admin" + ":" + curl_password
+ suffix = _expected_observed_suffix(
+ {
+ "ground_truth": "expected",
+ "final_answer": f"mysql -u root -p{mysql_password} prod; curl -u {curl_auth} https://example.invalid --token {token}",
+ }
+ )
+
+ assert mysql_password not in suffix
+ assert curl_password not in suffix
+ assert token not in suffix
+ assert "-p<redacted>" in suffix
+ assert "-u <redacted>" in suffix
+ assert "--token <redacted>" in suffix
+
+
+def test_unavailable_reason_uses_reward_load_error_not_trajectory_provenance() -> None:
+ reward = {
+ "trajectory_detail": "trajectory.json could not be read",
+ "details": {
+ "_trajectory_load": {
+ "source": "claude-code.txt",
+ "note": "Synthetic ATIF from Claude Code stream JSONL",
+ "warning": "trajectory.json has no steps[]",
+ }
+ },
+ }
+
+ assert evidence_unavailable_reason(reward) == (
+ "Trajectory evidence could not be loaded: trajectory.json could not be read"
+ )
+ assert evidence_unavailable_reason({"details": reward["details"]}) == (
+ "No trajectory evidence was recorded for this metric"
+ )
+
+
+def test_accuracy_card_keeps_built_reason_outside_bounded_notes() -> None:
+ detail = {
+ "reason": "Ran the right command but reported the wrong installed version.",
+ "criteria": {
+ "FACTUALLY_ACCURATE": False,
+ "ACTIONABLE": False,
+ "ACTION_CORRECT": False,
+ "TASK_ADDRESSED": False,
+ "SKILL_IDENTIFIED": False,
+ },
+ "ground_truth": "version 1.4.0",
+ "final_answer": "version 1.2.3",
+ }
+ payload = _payload([_reward(accuracy=0.2, details={"accuracy": detail})])
+ card = next(card for card in payload["agents"]["codex"]["evaluator_cards"] if card["id"] == "accuracy")
+ entry = card["evidence"][0]
+
+ assert entry["notes"] == [detail["reason"]]
+ assert entry["reason"].startswith("Failed criteria: FACTUALLY_ACCURATE")
+ assert detail["reason"] not in entry["reason"]
+ assert "Failed criteria" not in entry["notes"][0]
+ assert "Expected: version 1.4.0; observed: version 1.2.3" in entry["reason"]
+ rendered = _render(payload)
+ assert '<div class="t3-ev-detail t3-ev-reason">' in rendered
+
+
+def test_metric_evidence_omits_grader_reason_and_its_findings() -> None:
+ details = {
+ "security": {
+ "passed": False,
+ "score": 0.0,
+ "reason": "A; B",
+ "findings": [_finding("first", message="A"), _finding("second", message="B")],
+ }
+ }
+ payload = _payload([_reward(details=details)])
+ evidence = _security_card(payload)["evidence"][0]
+
+ assert evidence["notes"] == ["A; B"]
+ assert "reason" not in evidence
+ assert (
+ failure_reason(
+ {
+ "reason": "A; B",
+ "findings": [{"score_impact": True, "message": "A"}, {"score_impact": True, "message": "B"}],
+ },
+ omit=("A; B",),
+ )
+ == ""
+ )
+
+
+def test_saved_html_defaults_only_add_unavailable_reasons_to_eligible_agent_cards() -> None:
+ evidence = {
+ "entry_id": "case-1",
+ "score": 0.0,
+ "notes": [],
+ "failures": [],
+ "checks": [],
+ "evidence_refs": [],
+ }
+ security_card = {
+ "id": "security",
+ "with_skill": 0.0,
+ "evidence": [{**evidence, "evidence_unavailable_reason": "old security label"}],
+ }
+ accuracy_card = {"id": "accuracy", "with_skill": 0.0, "evidence": [evidence]}
+ top_level_accuracy_card = {
+ "id": "accuracy",
+ "with_skill": 0.0,
+ "evidence": [{**evidence, "evidence_unavailable_reason": "old top-level label"}],
+ }
+
+ security_agent = html_module._sanitize_tier3_agent({"evaluator_cards": [security_card]})
+ accuracy_agent = html_module._sanitize_tier3_agent({"evaluator_cards": [accuracy_card]})
+ top_level_accuracy = html_module._sanitize_tier3_evaluator_card(top_level_accuracy_card)
+
+ assert "evidence_unavailable_reason" not in security_agent["evaluator_cards"][0]["evidence"][0]
+ assert accuracy_agent["evaluator_cards"][0]["evidence"][0]["evidence_unavailable_reason"] == (
+ "No trajectory evidence was recorded for this metric"
+ )
+ assert "evidence_unavailable_reason" not in top_level_accuracy["evidence"][0]
+
+
def test_render_drops_malformed_reasons_and_excerpts() -> None:
payload = _payload([_reward(details=_security_details(_finding(DESTRUCTIVE_COMMAND)))])
payload["trials"][0]["score_reasons"] = {"security": {"nested": DESTRUCTIVE_REASON}}
diff --git a/tests/reporting/test_unified_tier3_report.py b/tests/reporting/test_unified_tier3_report.py
index 484ce5c0..a3d85860 100644
--- a/tests/reporting/test_unified_tier3_report.py
+++ b/tests/reporting/test_unified_tier3_report.py
@@ -7,6 +7,7 @@ import base64
import json
import math
import re
+from html import unescape
from pathlib import Path
import pytest
@@ -1571,6 +1572,44 @@ def test_canonical_html_tolerates_legacy_string_evidence_refs() -> None:
assert "trajectory.json#/steps/14" in agents_html
+def test_tier3_html_renders_plain_accuracy_reason_and_unavailable_evidence() -> None:
+ scores = dict.fromkeys(DEFAULT_METRICS, 1.0)
+ scores["accuracy"] = 0.2
+ reward = {
+ "entry_id": "case-plain-language",
+ **scores,
+ "details": {
+ "accuracy": {
+ "reason": "The answer was inaccurate",
+ "criteria": {"FACTUALLY_ACCURATE": False},
+ "ground_truth": "correct result",
+ "final_answer": "wrong result",
+ }
+ },
+ }
+ payload = build_agent_eval_payload(
+ "demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": scores,
+ "rewards": [reward],
+ }
+ },
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ agents_html = unescape(_tier3_page(_render_agent_payload(payload), "agents", "dataset"))
+
+ assert "Failed criteria: FACTUALLY_ACCURATE" in agents_html
+ assert "Expected: correct result; observed: wrong result" in agents_html
+ assert "Evidence unavailable — No trajectory evidence was recorded for this metric" in agents_html
+
+
def test_canonical_payload_deduplicates_repeated_metric_evidence() -> None:
reward = {
"entry_id": "case-1",
@@ -1603,6 +1642,7 @@ def test_canonical_payload_deduplicates_repeated_metric_evidence() -> None:
"failures": [],
"checks": [],
"evidence_refs": [],
+ "evidence_unavailable_reason": "No trajectory evidence was recorded for this metric",
"occurrences": 2,
}
]
diff --git a/tests/tier3/test_judge_result_semantics.py b/tests/tier3/test_judge_result_semantics.py
index 3fb995ad..2111ad9e 100644
--- a/tests/tier3/test_judge_result_semantics.py
+++ b/tests/tier3/test_judge_result_semantics.py
@@ -201,7 +201,7 @@ def test_report_reasons_still_omit_criteria_excluded_by_older_rewards():
findings = {finding["metric"]: finding for finding in report._extract_findings([reward])}
reasons = findings["accuracy"]["reasons"]
assert findings["accuracy"]["score"] == 0.75
- assert "ACTION_CORRECT failed" in reasons
+ assert "The agent did not take the action the expected answer calls for (criterion ACTION_CORRECT)" in reasons
assert not any("SKILL_IDENTIFIED" in reason for reason in reasons)
diff --git a/tests/tier3/test_plain_language_findings.py b/tests/tier3/test_plain_language_findings.py
new file mode 100644
index 00000000..18cc6d76
--- /dev/null
+++ b/tests/tier3/test_plain_language_findings.py
@@ -0,0 +1,191 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from skillevaluator.tier3.eval_core.llm_judge import _ACCURACY_DEDUCTION_SENTENCES
+from skillevaluator.tier3.harbor import report
+
+
[email protected](("criterion", "sentence"), _ACCURACY_DEDUCTION_SENTENCES.items())
+def test_accuracy_criteria_have_plain_language_reasons(criterion, sentence):
+ reasons = report._collect_fail_reasons("accuracy", [{"detail": {"criteria": {criterion: False}}}])
+
+ assert reasons == [f"{sentence} (criterion {criterion})"]
+
+
+def test_unknown_accuracy_criterion_has_safe_fallback():
+ assert report._collect_fail_reasons("accuracy", [{"detail": {"criteria": {"NEW_CHECK": False}}}]) == [
+ "Criterion NEW_CHECK was not met"
+ ]
+
+
+def test_excluded_accuracy_criterion_does_not_create_a_deduction():
+ detail = {"criteria": {"ACTION_CORRECT": False}, "excluded_criteria": ["ACTION_CORRECT"]}
+
+ assert report._collect_fail_reasons("accuracy", [{"detail": detail}]) == []
+
+
[email protected]("metric", ["skill_execution", "skill_efficiency"])
+def test_skill_check_without_reason_uses_plain_fallback(metric):
+ detail = {"error_recovery_check": {"passed": False}}
+
+ assert report._collect_fail_reasons(metric, [{"detail": detail}]) == [
+ "Check `error recovery check` did not pass; no reason was recorded"
+ ]
+
+
+def test_expected_and_observed_are_appended_once_and_bounded():
+ detail = {
+ "criteria": {"ACTION_CORRECT": False, "FACTUALLY_ACCURATE": False},
+ "ground_truth": "e" * 200,
+ "final_answer": "o" * 200,
+ }
+
+ reasons = report._collect_fail_reasons("accuracy", [{"detail": detail}])
+
+ assert sum(reason.count("Expected:") for reason in reasons) == 1
+ assert len(reasons[-1].split("Expected: ", 1)[1].split("; observed: ", 1)[0]) == 160
+ assert len(reasons[-1].split("; observed: ", 1)[1]) == 160
+
+
+def test_expected_and_observed_are_omitted_when_not_recorded():
+ reasons = report._collect_fail_reasons("goal_accuracy", [{"detail": {"reason": "goal missed"}}])
+
+ assert reasons == ["goal missed"]
+
+
+def _goal_reward(detail, score=0.2):
+ return {
+ "entry_id": "case-1",
+ "security": 1.0,
+ "skill_execution": 1.0,
+ "skill_efficiency": 1.0,
+ "accuracy": 1.0,
+ "goal_accuracy": score,
+ "behavior_check": 1.0,
+ "details": detail,
+ }
+
+
+def test_failed_finding_and_json_include_evidence_unavailable_reason():
+ reward = _goal_reward({"goal_accuracy": {"reason": "goal missed"}})
+ [finding] = report._extract_findings([reward], canonical_scores={"goal_accuracy": 0.2})
+ payload = report._bounded_findings_payload(
+ skill_name="demo",
+ agent="agent",
+ findings=[finding],
+ suggestions=[],
+ suggestion_mode="remediation",
+ suggestions_v2=None,
+ )
+
+ assert finding["evidence_unavailable_reason"] == "No trajectory evidence was recorded for this metric"
+ assert payload["findings"][0]["evidence_unavailable_reason"] == finding["evidence_unavailable_reason"]
+ assert (
+ "evidence: unavailable — No trajectory evidence was recorded for this metric"
+ in report._render_findings_body([finding]).plain
+ )
+
+
+def test_trajectory_load_failure_replaces_default_evidence_reason():
+ reward = _goal_reward({"goal_accuracy": {"reason": "goal missed"}})
+ reward["trajectory_detail"] = "trajectory could not be read"
+ [finding] = report._extract_findings([reward], canonical_scores={"goal_accuracy": 0.2})
+
+ assert (
+ finding["evidence_unavailable_reason"]
+ == "Trajectory evidence could not be loaded: trajectory could not be read"
+ )
+
+
+def test_trajectory_load_metadata_is_not_reported_as_a_load_failure():
+ reward = _goal_reward(
+ {
+ "_trajectory_load": {
+ "source": "claude-code.txt",
+ "note": "Synthetic ATIF from Claude Code stream JSONL",
+ "warning": "trajectory.json has no steps[]",
+ },
+ "goal_accuracy": {"reason": "goal missed"},
+ }
+ )
+ [finding] = report._extract_findings([reward], canonical_scores={"goal_accuracy": 0.2})
+
+ assert finding["evidence_unavailable_reason"] == "No trajectory evidence was recorded for this metric"
+
+
+def test_evidence_unavailable_labels_are_limited_to_evidence_ref_metrics():
+ reward = _goal_reward(
+ {
+ "security": {
+ "reason": "unsafe action",
+ "findings": [
+ {
+ "type": "destructive_command",
+ "severity": "critical",
+ "message": "unsafe action",
+ "evidence": "rm -rf ~/work",
+ "score_impact": True,
+ }
+ ],
+ },
+ "skill_execution": {"reason": "script was not invoked"},
+ "goal_accuracy": {"reason": "goal missed"},
+ }
+ )
+ reward["security"] = 0.0
+ reward["skill_execution"] = 0.0
+ findings = report._extract_findings(
+ [reward],
+ canonical_scores={"security": 0.0, "skill_execution": 0.0, "goal_accuracy": 0.2},
+ )
+ by_metric = {finding["metric"]: finding for finding in findings}
+
+ assert "evidence_unavailable_reason" not in by_metric["security"]
+ assert "evidence_unavailable_reason" not in by_metric["skill_execution"]
+ assert by_metric["goal_accuracy"]["evidence_unavailable_reason"] == (
+ "No trajectory evidence was recorded for this metric"
+ )
+ renderer_findings = [
+ *findings,
+ {
+ "metric": "skill_execution",
+ "label": "SKILL EXECUTION",
+ "severity": "critical",
+ "score": 0.0,
+ "reasons": ["script was not invoked"],
+ "evidence_refs": [],
+ },
+ ]
+ rendered = report._render_findings_body(renderer_findings).plain
+ assert rendered.count("evidence: unavailable —") == 1
+ assert "skill execution" in rendered.casefold()
+ assert "goal accuracy" in rendered.casefold()
+ security_payload = report._bounded_findings_payload(
+ skill_name="demo",
+ agent="agent",
+ findings=[by_metric["security"]],
+ suggestions=[],
+ suggestion_mode="remediation",
+ suggestions_v2=None,
+ )
+ assert "evidence_unavailable_reason" not in security_payload["findings"][0]
+
+
+def test_evidence_reason_is_absent_when_refs_are_present_or_finding_is_ok():
+ failed = _goal_reward(
+ {
+ "goal_accuracy": {
+ "reason": "goal missed",
+ "evidence_refs": [{"source": "trajectory.json", "json_pointer": "/steps/1"}],
+ }
+ }
+ )
+ [finding] = report._extract_findings([failed], canonical_scores={"goal_accuracy": 0.2})
+ assert "evidence_unavailable_reason" not in finding
+
+ passed = _goal_reward({"goal_accuracy": {"reason": "goal reached"}}, score=1.0)
+ [finding] = report._extract_findings([passed], canonical_scores={"goal_accuracy": 1.0})
+ assert finding["severity"] == "ok"
+ assert "evidence_unavailable_reason" not in finding