Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): derive accuracy scores from criterion verdicts (#190)

mavericksea-aice3c523918dd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a656c5f..1d9a4602 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -93,40 +93,10 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
-- Tier 1 pyproject dependency audits now inspect static target declarations
- rather than the evaluator environment. Exact direct pins (including optional
- groups) are queried without installation or build-script execution;
- unresolved declarations and invalid scanner evidence are incomplete.
-- Tier 1 PII scanning now includes `.mdc` rules and workflows.
-- Tier 3 negative-case accuracy excludes inapplicable skill identification and
- records its scoring policy. Reports separately expose measured token and
- agent execution-time differences with complete matching-arm coverage, while
- efficiency continues to measure routing and productive tool use.
-- Include schema-valid ATIF message content parts in final answers, conversation
- summaries, and judge evidence references in the host and Harbor verifier.
- Image and audio parts use markers without exposing their source paths.
-- Preserve scored Harbor trials when agent answers mention authentication errors
- and per-step token usage is present but aggregate ATIF token metrics are absent
- or zero. Explicit runtime exceptions and error events still invalidate trials.
-- Safety-classifier refusals, content filters and output-token truncation are
- detected instead of being read as empty or malformed output. `--llm-verify`
- and rubric evaluation report the cause, and `--llm-verify` keeps the
- complete verdicts of a truncated reply. Claude 5.5 or later requests to the
- native Claude API use server-side fallback to Claude Opus 4.8. In the Harbor
- verifier, Bedrock retries a filtered Claude 5.5 reply on Claude Opus 4.8, and
- a refusal or empty judge reply moves on to the next
- `LLM_JUDGE_FALLBACK_MODELS` entry.
-- The Harbor verifier's last-resort judge model follows the selected provider.
- With no judge or chat model configured, it used to request an OpenAI model
- from Anthropic, Bedrock, NVIDIA Build and gateways. The unused host-side
- `llm_judge.DEFAULT_JUDGE_MODEL` constant is removed.
-- Fixed Docker Codex and Claude Code setup wasting its timeout installing
- distribution Node/npm before Harbor's own installers. Native, gateway Codex,
- and NVIDIA Build agents now provision Node/npm only on the installer branches
- that need them, while preserving installed-agent and version-pin checks.
-- Preserved multiline skill descriptions when saving and loading Tier 2
- embedding catalogs. Descriptions can contain line breaks and tabs, while
- catalog identifiers and other control characters remain strictly validated.
+- Derive accuracy scores from complete five-criterion verdicts in both judge
+ runtimes, so an inconsistent model-supplied total cannot distort Skill Lift.
+ Correct the Harbor prompt example to score five passing criteria as 1.0.
+
- Brought the published documentation in line with 0.5.0. It now covers how
standard grading builds judge evidence and rebuilds trajectories from
OpenCode and Codex logs, structured judge output and retries, the evidence
diff --git a/src/skillevaluator/tier3/eval_core/llm_judge.py b/src/skillevaluator/tier3/eval_core/llm_judge.py
index d9317e94..0bc2f406 100644
--- a/src/skillevaluator/tier3/eval_core/llm_judge.py
+++ b/src/skillevaluator/tier3/eval_core/llm_judge.py
@@ -809,14 +809,12 @@ def judge_accuracy(
criteria = parsed.get("criteria")
criteria_valid = _valid_accuracy_criteria(criteria)
- score = _finite_score(parsed.get("score"))
- if not skill_identification_applicable:
- assert criteria_valid
- score = sum(value is True for key, value in criteria.items() if key != "SKILL_IDENTIFIED") / 4.0
- elif score is None:
- assert criteria_valid
+ if criteria_valid:
yes_count = sum(1 for v in criteria.values() if v is True)
score = yes_count / 5.0
+ else:
+ score = _finite_score(parsed.get("score"))
+ assert score is not None
result = {
"score": round(score, 4),
diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.py
index ba22e079..6f77824c 100644
--- a/src/skillevaluator/tier3/harbor/templates/eval.py
+++ b/src/skillevaluator/tier3/harbor/templates/eval.py
@@ -8250,7 +8250,7 @@ Compute score = count(true) / 5.
Be lenient on exact wording but strict on factual correctness.
Respond with ONLY a JSON object:
-{{"criteria": {{"SKILL_IDENTIFIED": true, "ACTION_CORRECT": true, "FACTUALLY_ACCURATE": true, "TASK_ADDRESSED": true, "ACTIONABLE": true}}, "score": 0.8, "reason": "brief summary"}}
+{{"criteria": {{"SKILL_IDENTIFIED": true, "ACTION_CORRECT": true, "FACTUALLY_ACCURATE": true, "TASK_ADDRESSED": true, "ACTIONABLE": true}}, "score": 1.0, "reason": "brief summary"}}
USER QUESTION:
{question}
@@ -8293,14 +8293,12 @@ SELECTED EVIDENCE (final response + produced artifacts; low-relevance steps may
criteria = parsed.get("criteria")
criteria_valid = _valid_accuracy_criteria(criteria)
- score = _finite_score(parsed.get("score"))
- if not skill_identification_applicable:
- assert criteria_valid
- score = sum(value is True for key, value in criteria.items() if key != "SKILL_IDENTIFIED") / 4.0
- elif score is None:
- assert criteria_valid
+ if criteria_valid:
score = sum(1 for v in criteria.values() if v is True) / 5.0
- result = {
+ else:
+ score = _finite_score(parsed.get("score"))
+ assert score is not None
+ return {
"score": round(score, 4),
"reason": _bounded_judge_text(parsed.get("reason", "")),
"criteria": criteria if criteria_valid else {},
diff --git a/tests/test_accuracy_arithmetic.py b/tests/test_accuracy_arithmetic.py
new file mode 100644
index 00000000..6bb8f6f2
--- /dev/null
+++ b/tests/test_accuracy_arithmetic.py
@@ -0,0 +1,70 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Accuracy scores must implement the five-criterion rubric in both runtimes."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+from tests.conftest import load_harbor_eval_template
+
+from skillevaluator.tier3.eval_core import llm_judge
+from skillevaluator.tier3.harbor.collector import _compute_lift
+
+
[email protected](params=["host", "harbor"])
+def accuracy_judge(request: pytest.FixtureRequest):
+ if request.param == "host":
+ return llm_judge
+ return load_harbor_eval_template("accuracy_arithmetic_template")
+
+
+def _criteria(passed: int) -> dict[str, bool]:
+ return {key: index < passed for index, key in enumerate(sorted(llm_judge._ACCURACY_CRITERIA_KEYS))}
+
+
[email protected]("passed", range(6))
+def test_accuracy_derives_score_from_complete_criteria(monkeypatch, accuracy_judge, passed):
+ payload = {"criteria": _criteria(passed), "score": 1 - passed / 5, "reason": "rubric verdicts"}
+ monkeypatch.setattr(accuracy_judge, "call_public_llm", lambda *_args, **_kwargs: (json.dumps(payload), None))
+
+ result = accuracy_judge.judge_accuracy("Summarize the CSV", "The total is 120", "The total is 120")
+
+ assert result["score"] == pytest.approx(passed / 5)
+ assert result["criteria"] == payload["criteria"]
+ assert "error" not in result
+
+
+ "payload, expected",
+ [
+ ({"criteria": _criteria(3), "score": 0.6}, 0.6),
+ ({"criteria": _criteria(3)}, 0.6),
+ ({"score": 0.6}, 0.6),
+ ],
+)
+def test_accuracy_preserves_consistent_and_legacy_payloads(monkeypatch, accuracy_judge, payload, expected):
+ monkeypatch.setattr(accuracy_judge, "call_public_llm", lambda *_args, **_kwargs: (json.dumps(payload), None))
+ assert accuracy_judge.judge_accuracy("question", "reference", "answer")["score"] == expected
+
+
+def test_accuracy_example_does_not_reverse_skill_lift(monkeypatch, accuracy_judge):
+ # The shipped Harbor prompt used all five true with a numeric score of 0.8.
+ payloads = iter(
+ [
+ {"criteria": _criteria(5), "score": 0.8},
+ {"criteria": _criteria(5), "score": 0.8},
+ {"criteria": _criteria(4), "score": 0.8},
+ {"criteria": _criteria(5), "score": 1.0},
+ ]
+ )
+ monkeypatch.setattr(accuracy_judge, "call_public_llm", lambda *_args, **_kwargs: (json.dumps(next(payloads)), None))
+ scores = [accuracy_judge.judge_accuracy("question", "reference", "answer")["score"] for _ in range(4)]
+ lift = _compute_lift({"accuracy": sum(scores[:2]) / 2}, {"accuracy": sum(scores[2:]) / 2})
+
+ assert lift["accuracy"]["with_skill"] == 1.0
+ assert lift["accuracy"]["without_skill"] == 0.9
+ assert lift["accuracy"]["delta"] == 0.1
+ assert lift["accuracy"]["direction"] == "up"