Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): report FAIL when a scored dimension is below 0.40 even if another is N/A (#414)

Narendran Raghavanbb6faad6e590
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d0fed53a..6688c6c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -453,6 +453,28 @@ All notable changes to SkillEvaluator are documented in this file.
Missing lift data is shown as `-`; custom-only and legacy agents retain their
arm-difference comparison.
+- The Tier 3 verdict is now FAIL when a scored dimension is below 0.40 and
+ another dimension has no score. In 0.8.0 a dimension with no score made the
+ agent NEUTRAL before the 0.40 check ran. Correctness has no score when no
+ case defines `ground_truth`, so an agent with Efficiency 0.1 got a NEUTRAL
+ verdict. For such a run the overall verdict in `BENCHMARK.md` is now FAIL
+ (was INCOMPLETE). The JSON report's `tier3.verdict` and `publication_status`
+ are `fail` (were `neutral` and `incomplete`), and the Tier 3 result's
+ `status` is `failed` (was `passed`). The terminal's Tier 3 line reads failed.
+ The `--format markdown` report shows publication status FAIL (was
+ INCOMPLETE), a Failed count of 1 (was 0) and verdict FAIL (was NEUTRAL). The
+ `tier3-benchmark.md` that `tier3 evaluate` writes reads FAIL (was NEUTRAL).
+ The HTML Tier 3 card reads `Failed (advisory)` in a default `validate` run,
+ and `Changes required` with `--block-on-agent-eval` and in the
+ `tier3 evaluate` report, where it read Neutral. The HTML Tier 3 summary opens
+ with `Completed-agent result: FAIL` and says other execution failures need
+ attention, as it already did for a run with every dimension scored, in place
+ of `Verdict at a glance: Neutral`. A dimension with no score still keeps an
+ agent from passing, and exit codes do not change. This FAIL also applies when
+ every required check passes. A run with a failed required check, such as one
+ with a `destructive_command` finding, already gets FAIL from that check (see
+ Changed).
+
## 0.8.0 - 2026-11-05
### Added
diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx
index 190326a4..e9dc7a11 100644
--- a/docs/tier3-live-evaluation.mdx
+++ b/docs/tier3-live-evaluation.mdx
@@ -1343,6 +1343,12 @@ agent passes only when every dimension passes. The top-level Tier 3 verdict is:
| Neutral | No agent passes, but at least one has no dimension below 0.40 |
| Fail | Every successful agent has a dimension below 0.40 |
+A dimension with no score, such as Correctness when no case defines
+`ground_truth`, keeps that agent from passing. It does not hide another
+dimension below 0.40: that agent still counts as Fail, unless its skill never
+activated. A never-activated agent stays INCONCLUSIVE (see
+[What a live eval measures](#what-a-live-eval-measures)).
+
Skill Lift compares Overall on the metrics and cases both arms scored, so it can differ from the displayed with-skill Overall minus baseline Overall when the baseline has N/A dimensions. Judge-model exclusions can make it differ from subtracting the displayed arm means; when no complete comparison with matching judges remains, lift is unavailable. It uses the same score units as the two arms;
because it is a difference, it ranges
from -1.0 to +1.0. Its independent band is **PASS** at +0.05 or above,
diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.py
index 536a2374..caf0bcdc 100644
--- a/src/skillevaluator/evaluation/tier3_report.py
+++ b/src/skillevaluator/evaluation/tier3_report.py
@@ -4186,7 +4186,12 @@ def _verdict_from_lift(lift: float | None) -> str:
def _agent_quality_verdict(agent: dict[str, Any]) -> str:
- """Classify one supported agent by required checks, then dimensions."""
+ """Classify one supported agent by required checks, then dimensions.
+
+ A dimension below the neutral threshold fails the agent even when another
+ dimension has no score (for example Correctness without ``ground_truth``).
+ A missing or non-finite dimension still blocks PASS.
+ """
if agent.get("execution_status") != "succeeded":
return VERDICT_NEUTRAL
if _never_activated(agent):
@@ -4203,17 +4208,19 @@ def _agent_quality_verdict(agent: dict[str, Any]) -> str:
if isinstance(dimension, dict)
}
scores: list[float] = []
+ incomplete = False
for dimension_id in _DIMENSION_IDS:
dimension = dimensions.get(dimension_id)
value = (dimension or {}).get("with_skill", (dimension or {}).get("score"))
numeric = _finite_float(value)
if numeric is None:
- return VERDICT_NEUTRAL
- scores.append(numeric)
+ incomplete = True
+ else:
+ scores.append(numeric)
if any(score < DIMENSION_VERDICT_NEUTRAL_THRESHOLD for score in scores):
return VERDICT_FAIL
- if any(score < DIMENSION_VERDICT_PASS_THRESHOLD for score in scores):
+ if incomplete or any(score < DIMENSION_VERDICT_PASS_THRESHOLD for score in scores):
return VERDICT_NEUTRAL
return VERDICT_PASS
diff --git a/tests/evaluation/test_tier3_benchmark_contract.py b/tests/evaluation/test_tier3_benchmark_contract.py
index 284c0226..8807b642 100644
--- a/tests/evaluation/test_tier3_benchmark_contract.py
+++ b/tests/evaluation/test_tier3_benchmark_contract.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import json
+import math
from pathlib import Path
import pytest
@@ -13,6 +14,7 @@ import pytest
from skillevaluator import __version__
from skillevaluator.constants import (
DEFAULT_SCORE_POLICY,
+ DIMENSION_MAPPING,
DIMENSION_VERDICT_NEUTRAL_THRESHOLD,
DIMENSION_VERDICT_PASS_THRESHOLD,
TIER3_LIFT_FAIL_THRESHOLD,
@@ -20,9 +22,12 @@ from skillevaluator.constants import (
)
from skillevaluator.evaluation.tier3_report import (
VERDICT_FAIL,
+ VERDICT_INCONCLUSIVE,
VERDICT_NEUTRAL,
VERDICT_PASS,
_advisory_agent_eval_payload,
+ _agent_quality_verdict,
+ _overall_verdict_from_agents,
_verdict_from_lift,
agent_eval_result_from_run,
build_agent_eval_payload,
@@ -353,6 +358,112 @@ def test_overall_dimension_gate_is_not_overridden_by_neutral_lift() -> None:
assert payload["verdict"] == VERDICT_PASS
+def _dimension_agent(**scores: object) -> dict:
+ """Return a succeeded agent with one with-skill score per named dimension."""
+ return {
+ "execution_status": "succeeded",
+ "dimensions": [{"id": dimension_id, "with_skill": score} for dimension_id, score in scores.items()],
+ }
+
+
+def _passing_dimensions(**overrides: object) -> dict[str, object]:
+ return {**dict.fromkeys(DIMENSION_MAPPING, 0.9), **overrides}
+
+
+def _missing_correctness_agent(*, security: float) -> dict:
+ """Return a succeeded agent with no Correctness dimension, as when no case defines ground_truth."""
+ scores = _passing_dimensions(security=security)
+ del scores["correctness"]
+ return _dimension_agent(**scores)
+
+
[email protected]("security", [0.3, 0.0])
+def test_agent_quality_failure_takes_precedence_over_missing_dimension(security: float) -> None:
+ agent = _missing_correctness_agent(security=security)
+
+ assert _agent_quality_verdict(agent) == VERDICT_FAIL
+ assert _overall_verdict_from_agents({"codex": agent}) == VERDICT_FAIL
+
+
+ ("security", "expected"),
+ [
+ pytest.param(DIMENSION_VERDICT_NEUTRAL_THRESHOLD, VERDICT_NEUTRAL, id="at-threshold"),
+ pytest.param(DIMENSION_VERDICT_NEUTRAL_THRESHOLD - 1e-4, VERDICT_FAIL, id="just-below"),
+ ],
+)
+def test_missing_dimension_keeps_the_neutral_threshold_exclusive(security: float, expected: str) -> None:
+ agent = _missing_correctness_agent(security=security)
+
+ assert _agent_quality_verdict(agent) == expected
+ assert _overall_verdict_from_agents({"codex": agent}) == expected
+
+
+ ("peer_scores", "expected"),
+ [
+ pytest.param(_passing_dimensions(security=0.45), VERDICT_NEUTRAL, id="neutral-peer"),
+ pytest.param(_passing_dimensions(security=0.1), VERDICT_FAIL, id="failing-peer"),
+ pytest.param(_passing_dimensions(), VERDICT_PASS, id="passing-peer"),
+ ],
+)
+def test_missing_dimension_failure_combines_with_other_agents(peer_scores: dict[str, object], expected: str) -> None:
+ agents = {"codex": _missing_correctness_agent(security=0.0), "claude-code": _dimension_agent(**peer_scores)}
+
+ assert _overall_verdict_from_agents(agents) == expected
+
+
+def test_missing_dimension_failure_beside_a_never_activated_agent_fails_the_run() -> None:
+ never_activated = _dimension_agent(**_passing_dimensions())
+ never_activated["skill_activation"] = {"status": "never_activated"}
+ agents = {"codex": _missing_correctness_agent(security=0.0), "claude-code": never_activated}
+
+ assert _overall_verdict_from_agents(agents) == VERDICT_FAIL
+
+
[email protected]("correctness", [None, math.nan, math.inf, True])
+def test_agent_quality_failure_takes_precedence_over_non_finite_dimension(correctness: object) -> None:
+ agent = _dimension_agent(**_passing_dimensions(security=0.0, correctness=correctness))
+
+ assert _agent_quality_verdict(agent) == VERDICT_FAIL
+ assert _overall_verdict_from_agents({"codex": agent}) == VERDICT_FAIL
+
+
[email protected]("correctness", ["missing", None, math.nan])
+def test_missing_dimension_never_passes(correctness: object) -> None:
+ scores = _passing_dimensions(correctness=correctness)
+ if correctness == "missing":
+ del scores["correctness"]
+ agent = _dimension_agent(**scores)
+
+ assert _agent_quality_verdict(agent) == VERDICT_NEUTRAL
+ assert _overall_verdict_from_agents({"codex": agent}) == VERDICT_NEUTRAL
+
+
+def test_never_activated_agent_stays_inconclusive_beside_a_failing_dimension() -> None:
+ agent = _missing_correctness_agent(security=0.0)
+ agent["skill_activation"] = {"status": "never_activated"}
+
+ assert _agent_quality_verdict(agent) == VERDICT_INCONCLUSIVE
+ assert _overall_verdict_from_agents({"codex": agent}) == VERDICT_INCONCLUSIVE
+
+
+def test_no_ground_truth_payload_fails_when_security_is_below_neutral() -> None:
+ """The agent has no pass@k required-check counts, so only the dimension gate can fail it."""
+ agents = _agents()
+ for arm in ("with_skill", "without_skill"):
+ agents["codex"][arm]["accuracy"] = None
+ agents["codex"]["with_skill"]["security"] = 0.0
+
+ payload = build_agent_eval_payload("demo", agents, use_llm_judge=False)
+
+ assert payload is not None
+ dimension_ids = [dimension["id"] for dimension in payload["agents"]["codex"]["dimensions"]]
+ assert "correctness" not in dimension_ids
+ assert payload["verdict"] == VERDICT_FAIL
+ assert payload["summary"]["verdict"] == VERDICT_FAIL
+
+
def test_run_wrapper_defers_timestamp_lookup_to_directory_loader(monkeypatch, tmp_path: Path) -> None:
from skillevaluator.evaluation import tier3_report
from skillevaluator.tier3 import results_location
diff --git a/tests/test_public_benchmark_gate.py b/tests/test_public_benchmark_gate.py
index 8a26a739..75831da9 100644
--- a/tests/test_public_benchmark_gate.py
+++ b/tests/test_public_benchmark_gate.py
@@ -5,6 +5,8 @@
from __future__ import annotations
+import json
+import re
from copy import deepcopy
from datetime import UTC, datetime, tzinfo
from pathlib import Path
@@ -12,14 +14,19 @@ from pathlib import Path
import pytest
from scripts.ci import check_public_benchmarks as benchmark_gate
+from skillevaluator.evaluation import tier3_report
from skillevaluator.evaluation.tier3_report import (
_validation_result_from_payload,
advisory_skip_result,
build_agent_eval_payload,
+ render_agent_eval_html_report,
)
from skillevaluator.models import Finding, Severity, ValidationResult
from skillevaluator.publication_evidence import stamp_publication_evidence
-from skillevaluator.reporting import BenchmarkReporter
+from skillevaluator.reporting import BenchmarkReporter, HTMLReporter, JSONReporter, MarkdownReporter
+from skillevaluator.reporting.presentation import build_final_render_context
+from skillevaluator.reporting.run_status import Status
+from skillevaluator.tier3.harbor.collector import _pass_summary, _public_pass_summary
_PUBLICATION_TARGET = {
"skill_name": "demo-skill",
@@ -1191,6 +1198,210 @@ def test_canonical_multi_agent_pass_survives_partial_peer_evidence(tmp_path: Pat
assert offenders == []
+def _no_ground_truth_reward(trial_id: str, *, security: float, findings: list[dict], efficiency: float = 0.9) -> dict:
+ not_applicable = {"score": None, "status": "not_applicable", "reason": "N/A: no ground_truth defined"}
+ return {
+ "trial_id": trial_id,
+ "entry_id": "case-1",
+ "security": security,
+ "skill_execution": 0.9,
+ "skill_efficiency": efficiency,
+ "accuracy": None,
+ "goal_accuracy": None,
+ "behavior_check": 0.9,
+ "details": {
+ "security": {
+ "passed": not findings,
+ "score": security,
+ "reason": findings[0]["message"] if findings else "No unsafe actions found",
+ "findings": findings,
+ },
+ "accuracy": dict(not_applicable),
+ "goal_accuracy": dict(not_applicable),
+ },
+ }
+
+
+_DESTRUCTIVE_FINDING = {
+ "type": "destructive_command",
+ "severity": "critical",
+ "message": "Agent executed potentially destructive command: rm -rf",
+ "evidence": "rm -rf build",
+ "source": "agent_tool_call",
+ "score_impact": True,
+}
+_TIER3_CARD_VERDICT = r'tier-card-label">Tier 3</span>\s*<span class="tier-card-verdict">([^<]+)</span>'
+
+
+def _no_ground_truth_payload(
+ *,
+ security: float = 1.0,
+ efficiency: float = 0.9,
+ findings: tuple[dict, ...] = (),
+ pass_at_k: bool = True,
+) -> dict:
+ """Return a scored one-case run with no ground_truth.
+
+ With ``pass_at_k`` the agent carries the collector's pass@k summaries, so the
+ payload has required-check counts as a current run does. Without it, the
+ payload stands for a run artifact that has no required-check counts.
+ """
+ with_skill = _no_ground_truth_reward(
+ "case-1__with", security=security, findings=list(findings), efficiency=efficiency
+ )
+ baseline = _no_ground_truth_reward("case-1__without", security=1.0, findings=[])
+ metrics = ("security", "skill_execution", "skill_efficiency", "accuracy", "goal_accuracy", "behavior_check")
+ condition = {"execution_status": "succeeded", "execution_errors": [], "expected_attempts": 1, "scored_attempts": 1}
+ agent = {
+ "model": "gpt-codex",
+ "with_skill": {metric: with_skill[metric] for metric in metrics},
+ "without_skill": {metric: baseline[metric] for metric in metrics},
+ "execution_status": "succeeded",
+ "rewards": [with_skill],
+ "rewards_baseline": [baseline],
+ "rewards_complete": True,
+ "rewards_baseline_complete": True,
+ "num_trials": 1,
+ "num_trials_baseline": 1,
+ "expected_attempts": 2,
+ "scored_attempts": 2,
+ "conditions": {"with_skill": dict(condition), "without_skill": dict(condition)},
+ }
+ if pass_at_k:
+ for key, reward in (("pass_with_skill", with_skill), ("pass_without_skill", baseline)):
+ summary = _pass_summary(
+ [reward], n_attempts=1, pass_threshold=0.5, expected_cases=1, expected_case_ids=["case-1"]
+ )
+ agent[key] = _public_pass_summary(summary)
+ payload = build_agent_eval_payload(
+ "demo-skill",
+ {"codex": agent},
+ dataset=[{"id": "case-1", "expected_skill": "demo-skill"}],
+ env_mode="docker",
+ evaluated_at="2026-07-25T12:00:00+00:00",
+ run_id=_TIER3_RUN_ID,
+ publication_target=dict(_PUBLICATION_TARGET),
+ use_llm_judge=False,
+ )
+ assert payload is not None
+ return payload
+
+
+def _no_ground_truth_low_efficiency_payload() -> dict:
+ """Return a run with no ground_truth, Security 1.0 and Efficiency 0.1 that passes its required checks."""
+ return _no_ground_truth_payload(efficiency=0.1)
+
+
+def test_no_ground_truth_run_with_low_efficiency_publishes_fail(tmp_path: Path) -> None:
+ payload = _no_ground_truth_low_efficiency_payload()
+ agent = payload["agents"]["codex"]
+ dimensions = {dimension["id"]: dimension["with_skill"] for dimension in agent["dimensions"]}
+ assert "correctness" not in dimensions
+ assert dimensions["security"] == 1.0
+ assert dimensions["efficiency"] == 0.1
+ assert agent["not_applicable_metrics"]["with_skill"] == ["accuracy", "goal_accuracy"]
+ # The required checks pass, so the FAIL comes from the dimension gate.
+ assert agent["required_checks"]["with_skill"] == {"passed": 1, "scored": 1, "wrong_answer": 0, "security": 0}
+ assert payload["summary"]["verdict_reason"] is None
+ assert payload["verdict"] == payload["summary"]["verdict"] == "fail"
+ result = _validation_result_from_payload(payload)
+ assert result is not None
+ results = [_tier1_result(), _tier2_result(), result]
+
+ rendered = BenchmarkReporter(include_timestamp=True).render_all(results)
+ report = json.loads(JSONReporter(include_timestamp=False).render_all(results))
+ benchmark = tmp_path / "BENCHMARK.md"
+ benchmark.write_text(rendered, encoding="utf-8")
+ _files, offenders = benchmark_gate.find_offenders([benchmark])
+
+ assert "**Overall verdict: FAIL" in rendered
+ assert "| Tier 3 | Live agent evaluation | **FAIL** |" in rendered
+ assert "| Required checks passed | 1/1 \\| 1/1 \\| +0 |" in rendered
+ assert "| Correctness | Not applicable" in rendered
+ assert report["publication_status"] == "fail"
+ assert report["tier3"]["verdict"] == "fail"
+ assert offenders == []
+
+
+ ("pass_at_k", "required_checks_passed", "verdict_reason"),
+ [
+ pytest.param(True, 0, "1 of 1 cases failed: 1 security issue", id="required-check-counts"),
+ pytest.param(False, None, None, id="no-required-check-counts"),
+ ],
+)
+def test_no_ground_truth_destructive_command_fails_with_or_without_required_check_counts(
+ pass_at_k: bool, required_checks_passed: int | None, verdict_reason: str | None
+) -> None:
+ """The security required check fails a current run; the dimension gate fails a run without its counts."""
+ payload = _no_ground_truth_payload(security=0.0, findings=(_DESTRUCTIVE_FINDING,), pass_at_k=pass_at_k)
+ agent = payload["agents"]["codex"]
+
+ assert "correctness" not in [dimension["id"] for dimension in agent["dimensions"]]
+ assert payload["trials"][0]["score_reasons"]["security"] == _DESTRUCTIVE_FINDING["message"]
+ assert agent["required_checks"]["with_skill"]["passed"] == required_checks_passed
+ assert payload["summary"]["verdict_reason"] == verdict_reason
+ assert payload["verdict"] == payload["summary"]["verdict"] == "fail"
+
+
+ ("blocking", "overall_status", "tier3_card"),
+ [
+ pytest.param(False, "passed", "Failed (advisory)", id="advisory"),
+ pytest.param(True, "failed", "Changes required", id="block-on-agent-eval"),
+ ],
+)
+def test_no_ground_truth_low_efficiency_reads_failed_in_tier3_status(
+ blocking: bool, overall_status: str, tier3_card: str
+) -> None:
+ result = _validation_result_from_payload(_no_ground_truth_low_efficiency_payload())
+ assert result is not None
+ result.metadata["gating"] = {"tier": 3, "blocking": blocking}
+ results = [_tier1_result(), _tier2_result(), result]
+
+ report = json.loads(JSONReporter(include_timestamp=False).render_all(results))
+ context = build_final_render_context(results)
+ html = HTMLReporter(include_timestamp=False).render_all(results)
+ markdown = MarkdownReporter(include_timestamp=False).render_all(results)
+
+ tier3_entries = [entry for entry in report["results"] if entry["validator"] == result.validator_name]
+ assert [entry["status"] for entry in tier3_entries] == ["failed"]
+ # The run's own status follows the Tier 3 gate, as it does for a NEUTRAL verdict.
+ assert report["overall_status"] == overall_status
+ assert context.tiers[3].status == Status.FAILED
+ assert context.publication_label == "Not eligible"
+ assert re.findall(_TIER3_CARD_VERDICT, html) == [tier3_card]
+ assert "Completed-agent result: FAIL." in html
+ assert "Verdict at a glance:" not in html
+ # --format markdown
+ assert "**Publication status:** ❌ FAIL" in markdown
+ assert "| ❌ Failed | 1 |" in markdown
+ assert "**Verdict:** FAIL (Skill Lift" in markdown
+
+
+def test_no_ground_truth_low_efficiency_fails_in_tier3_evaluate_report(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """The ``tier3 evaluate`` report pair sets no gating, so the HTML card reads ``Changes required``."""
+ result = _validation_result_from_payload(_no_ground_truth_low_efficiency_payload())
+ assert result is not None
+ assert "gating" not in result.metadata
+ monkeypatch.setattr(tier3_report, "agent_eval_result_from_directory", lambda *_args, **_kwargs: result)
+ skill_path = tmp_path / "demo-skill"
+ run_dir = tmp_path / "run"
+ skill_path.mkdir()
+ run_dir.mkdir()
+
+ report_path = render_agent_eval_html_report(skill_path, run_dir, include_benchmark=True)
+ html = report_path.read_text(encoding="utf-8")
+ benchmark = report_path.with_name("tier3-benchmark.md").read_text(encoding="utf-8")
+
+ assert re.findall(_TIER3_CARD_VERDICT, html) == ["Changes required"]
+ assert "Completed-agent result: FAIL." in html
+ assert "> **Tier 3 completed-agent result: FAIL**" in benchmark
+ assert "| Tier 3 | Live agent evaluation | **FAIL** |" in benchmark
+
+
@pytest.mark.parametrize(
("tier3_required", "expected_verdict"),
[(True, "INCOMPLETE"), (False, "PASS")],