Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(reporting): include required case checks in the BENCHMARK verdict rule (#644)

Christopher Kevin46336a692ae9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73cc170e..68e7a823 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -320,6 +320,8 @@ All notable changes to SkillEvaluator are documented in this file.
or no final newline. The duplicate table made the file fail to parse.
- Benchmark methodology describes the Overall row's shared comparison basis
rather than the headline with-skill score, in Tier 3 and publication cards.
+- BENCHMARK cards explain the required with-skill goal and security case checks
+ that can fail quality even when all dimensions pass.
- The Tier 2 scale guide names the fixed 8 MiB selected-manifest input budget
and 1 MiB per-manifest limit, including in description-only embedding mode.
- Saved Tier 2 catalogs use compact JSON so indentation no longer rejects dense
diff --git a/src/skillevaluator/reporting/benchmark.py b/src/skillevaluator/reporting/benchmark.py
index 1964dc35..363d3d81 100644
--- a/src/skillevaluator/reporting/benchmark.py
+++ b/src/skillevaluator/reporting/benchmark.py
@@ -1028,7 +1028,11 @@ class BenchmarkReporter(ReporterBase):
),
(
"- Overall verdict: PASS only when every configured dimension passes for at least one "
- "supported agent. Lift is reported as diagnostic evidence and does not override this gate."
+ "supported agent, with no with-skill case failing a required goal-accuracy or security check. "
+ "A scored unmet goal or security issue in any attempt fails its case and that agent's quality "
+ "verdict, even if all dimensions pass. "
+ "Baseline-arm required-check failures do not change this quality verdict. "
+ "Lift is reported as diagnostic evidence and does not override this gate."
),
]
)
diff --git a/tests/golden/benchmark_fail/BENCHMARK.md b/tests/golden/benchmark_fail/BENCHMARK.md
index de3a6f72..59e7070c 100644
--- a/tests/golden/benchmark_fail/BENCHMARK.md
+++ b/tests/golden/benchmark_fail/BENCHMARK.md
@@ -102,7 +102,7 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92
- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%.
- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL.
-- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate.
+- Overall verdict: PASS only when every configured dimension passes for at least one supported agent, with no with-skill case failing a required goal-accuracy or security check. A scored unmet goal or security issue in any attempt fails its case and that agent's quality verdict, even if all dimensions pass. Baseline-arm required-check failures do not change this quality verdict. Lift is reported as diagnostic evidence and does not override this gate.
- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold.
- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`).
- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict.
diff --git a/tests/golden/benchmark_pass/BENCHMARK.md b/tests/golden/benchmark_pass/BENCHMARK.md
index d28e42db..97a793ca 100644
--- a/tests/golden/benchmark_pass/BENCHMARK.md
+++ b/tests/golden/benchmark_pass/BENCHMARK.md
@@ -98,7 +98,7 @@ Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92
- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%.
- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL.
-- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate.
+- Overall verdict: PASS only when every configured dimension passes for at least one supported agent, with no with-skill case failing a required goal-accuracy or security check. A scored unmet goal or security issue in any attempt fails its case and that agent's quality verdict, even if all dimensions pass. Baseline-arm required-check failures do not change this quality verdict. Lift is reported as diagnostic evidence and does not override this gate.
- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold.
- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`).
- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict.
diff --git a/tests/reporting/test_dimension_explanations.py b/tests/reporting/test_dimension_explanations.py
index 0236782c..4b117dd9 100644
--- a/tests/reporting/test_dimension_explanations.py
+++ b/tests/reporting/test_dimension_explanations.py
@@ -111,6 +111,19 @@ def test_benchmark_methodology_states_when_no_baseline_ran() -> None:
assert _row(markdown, "Correctness").endswith("| 90.0% — baseline not run; uplift unavailable |")
+def test_benchmark_verdict_methodology_includes_with_skill_required_checks() -> None:
+ result = _validation_result_from_payload(_payload({"agent": _agent()}))
+ assert result is not None
+ markdown = BenchmarkReporter(include_timestamp=False).render_all([result])
+
+ assert "with no with-skill case failing a required goal-accuracy or security check" in markdown
+ assert (
+ "A scored unmet goal or security issue in any attempt fails its case and that agent's quality "
+ "verdict, even if all dimensions pass."
+ ) in markdown
+ assert "Baseline-arm required-check failures do not change this quality verdict." in markdown
+
+
@pytest.mark.parametrize("scope", ["publication", "tier3"])
@pytest.mark.parametrize("baseline", [True, False])
def test_overall_row_methodology_describes_comparison_basis(scope: str, baseline: bool) -> None:
diff --git a/tests/test_benchmark_source_provenance.py b/tests/test_benchmark_source_provenance.py
index b73f9788..0a49291c 100644
--- a/tests/test_benchmark_source_provenance.py
+++ b/tests/test_benchmark_source_provenance.py
@@ -900,7 +900,7 @@ class TestPassDetection:
"""The verdict line is honoured whether or not the card blockquotes it."""
_POLICY_BULLET = (
- "- Overall verdict: PASS only when every configured dimension passes for at least one supported agent."
+ "- Overall verdict: PASS only when every configured dimension passes for at least one supported agent"
)
_UNRECORDED_REASONS = (
"publication PASS without recorded evaluated source",
@@ -934,13 +934,19 @@ class TestPassDetection:
bullet as the verdict charges it with three offences it never committed.
"""
card = _pass_card(_ABSENT).replace("> **Overall verdict: PASS**", f"> **Overall verdict: {verdict}**", 1)
- card = f"{card}\n## Scoring Policy\n\n{self._POLICY_BULLET}\n"
+ card = f"{card}\n## Scoring Policy\n\n{self._POLICY_BULLET}.\n"
assert self._unrecorded(_scan(tmp_path, card, require=True)) == ()
- def test_the_published_card_carries_that_bullet_and_is_still_detected(self, tmp_path: Path) -> None:
+ @pytest.mark.parametrize("historical", [False, True], ids=["current-wording", "historical-wording"])
+ def test_the_published_card_carries_that_bullet_and_is_still_detected(
+ self, tmp_path: Path, historical: bool
+ ) -> None:
"""Excluding bullets must not blind the check on the card the reporter writes."""
card = (Path(__file__).parent / "golden" / "benchmark_pass" / "BENCHMARK.md").read_text(encoding="utf-8")
assert self._POLICY_BULLET in card
+ if historical:
+ policy_line = next(line for line in card.splitlines() if line.startswith(self._POLICY_BULLET))
+ card = card.replace(policy_line, self._POLICY_BULLET + ".", 1)
assert _scan(tmp_path, card, require=True) == []
stripped = "\n".join(