Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): keep failed judging out of skill failure advice (#579)

Christopher Kevina313ae16afb5
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 788a4260..98a3dd33 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -290,6 +290,11 @@ All notable changes to SkillEvaluator are documented in this file.
`(managed)` for the downloaded Gitleaks binary. Before, the Gitleaks row
ended in `(from None)` and the SkillSpector row named
`SKILLEVALUATOR_SKILLSPECTOR_PATH` although no override was set.
+- Tier 3 case recommendations treat required judge failures as unscored
+ evaluation problems, with advice to check judge configuration and provider
+ availability. Failed judging is no longer counted twice or blamed on the
+ skill, including for negative controls; genuine scored failures retain their
+ existing advice.
- The comments-only consumer template now grants its required read and PR-comment
permissions only to the comment job, with no workflow-wide token permissions.
- The behavior evidence gate no longer keeps a score for an expectation that
diff --git a/src/skillevaluator/tier3/case_summary.py b/src/skillevaluator/tier3/case_summary.py
index 0dd7f055..106195da 100644
--- a/src/skillevaluator/tier3/case_summary.py
+++ b/src/skillevaluator/tier3/case_summary.py
@@ -180,10 +180,46 @@ def _cases(agent: Mapping[str, Any], key: str) -> Mapping[str, Any]:
)
rebuilt = _scored_trials(agent, key)
for case_id, n in errored.items():
- rebuilt.setdefault(case_id, {"attempts": []})["attempts_missing"] = n
+ case = rebuilt.setdefault(case_id, {"attempts": []})
+ case["attempts_missing"] = max(n, case.get("attempts_missing", 0))
return rebuilt
+def _evaluation_failed(reward: Any) -> bool:
+ return isinstance(reward, Mapping) and str(reward.get("evaluation_status") or "").casefold() in {"error", "failed"}
+
+
+# A failed evaluation never reached a score; the cause decides who has to act.
+_EVALUATION_FAILURES = {
+ "environment": (
+ "the sandbox environment is invalid",
+ "Not a skill problem: the sandbox could not run what this case needs. Fix the environment named above "
+ "(for example install the missing tool in evals/environment/Dockerfile), then re-run this case.",
+ ),
+ "collector": (
+ "the attempt's result could not be collected",
+ "Not a skill problem: SkillEvaluator could not read this attempt's verifier output. Re-run this case; "
+ "if it repeats, check the verifier artifacts and any custom grader output in the run directory.",
+ ),
+ "judge": (
+ "required judge evaluation failed",
+ "Not a skill problem: check the judge model configuration, credentials and provider availability, "
+ "then re-run this case.",
+ ),
+}
+
+
+def _evaluation_failure(reward: Mapping[str, Any]) -> tuple[str, str]:
+ """Name what failed an unscored evaluation (environment, collector or judge) and the recorded reason."""
+ errors = reward.get("evaluation_errors")
+ if isinstance(errors, Mapping):
+ for cause in ("environment", "collector"):
+ if cause in errors:
+ return cause, _short(errors[cause], 160)
+ return "judge", _short("; ".join(f"{metric}: {reason}" for metric, reason in errors.items() if reason), 160)
+ return "judge", _short(errors, 160)
+
+
def _scored_trials(agent: Mapping[str, Any], key: str) -> dict[str, dict[str, Any]]:
output_dir = agent.get("output_dir")
trials = Path(str(output_dir)) / dict(_VARIANTS)[key] / "trials" if output_dir else None
@@ -216,6 +252,10 @@ def _scored_trials(agent: Mapping[str, Any], key: str) -> dict[str, dict[str, An
cases: dict[str, dict[str, Any]] = {}
for trial in sorted(trials.iterdir()):
reward = _load_json(trial / "reward.json", root=Path(str(output_dir)))
+ if _evaluation_failed(reward):
+ case = cases.setdefault(trial.name.split("__", 1)[0], {"attempts": []})
+ case["attempts_missing"] = case.get("attempts_missing", 0) + 1
+ continue
score = reward.get("overall") if isinstance(reward, Mapping) else None
if isinstance(score, int | float) and not isinstance(score, bool):
attempts = cases.setdefault(trial.name.split("__", 1)[0], {"attempts": []})["attempts"]
@@ -931,11 +971,45 @@ def _diagnose_agent_case(
wp, wn = _attempt_rate(with_marks)
bp, bn = _attempt_rate(without_marks)
errors = with_marks.count(ERROR)
+ failed_evaluations: dict[str, list[tuple[Path, str]]] = {}
+ with_trials = run_dir / agent_name / "with-skill" / "trials"
+ if errors and with_trials.is_dir() and _SAFE_TRIAL.fullmatch(agent_name):
+ for d in sorted(with_trials.iterdir()):
+ if (
+ d.name.split("__", 1)[0] == case_id
+ and _trial_dir(run_dir, agent_name, "with-skill", d.name) is not None
+ and _evaluation_failed(reward := _load_json(d / "reward.json", root=run_dir))
+ ):
+ cause, reason = _evaluation_failure(reward)
+ failed_evaluations.setdefault(cause, []).append((d, reason))
+ evaluation_issues = []
+ for cause, failed in failed_evaluations.items():
+ lost_unscored = min(errors, len(failed))
+ if not lost_unscored:
+ break
+ errors -= lost_unscored
+ label, action = _EVALUATION_FAILURES[cause]
+ evaluation_issues.append(
+ {
+ "case": case_id,
+ "agent": agent_name,
+ "kind": "run error",
+ "cause": cause,
+ "what": f"{lost_unscored} attempt(s) never produced a score: {label}"
+ + (f": {failed[0][1]}" if failed[0][1] else ""),
+ "attempts_lost": lost_unscored,
+ "reasons": [],
+ "env_errors": [],
+ "final_message": "",
+ "evidence": (failed[0][0].relative_to(run_dir) / "reward.json").as_posix(),
+ "action": action,
+ }
+ )
# Negative controls must avoid the skill; an agent-wide positive miss does not apply to them.
if entry is None:
entry = _dataset_entries(run_dir).get(case_id, {})
should_trigger = resolve_should_trigger(dict(entry))
- not_used = should_trigger is not False and _never_activated(agent)
+ not_used = should_trigger is not False and _never_activated(agent) and (wn > 0 or not evaluation_issues)
failed_trials = []
for attempt in with_case.get("attempts", []) if isinstance(with_case, Mapping) else []:
if isinstance(attempt, Mapping) and attempt.get("passed") is not True:
@@ -962,7 +1036,7 @@ def _diagnose_agent_case(
)
goal = _goal_not_met(run_dir, agent_name, case_id, with_case)
if not lost and not hurts and not not_used:
- return [goal] if goal else []
+ return [*evaluation_issues, goal] if goal else evaluation_issues
rewards = [
(r, d) for _, d in failed_trials if isinstance(r := _load_json(d / "reward.json", root=run_dir), Mapping)
]
@@ -1006,7 +1080,7 @@ def _diagnose_agent_case(
"final_message": _final_message(failed_trials[0][1], run_dir) if failed_trials else "",
"evidence": _evidence(run_dir, evidence_dir, run_error=kind == "run error"),
}
- return [failure, goal] if goal else [failure]
+ return [failure, *evaluation_issues, goal] if goal else [failure, *evaluation_issues]
def _goal_not_met(
@@ -1219,11 +1293,11 @@ def what_to_fix(
if isinstance(v, Mapping)
}
dataset = _dataset_entries(run_dir)
- grouped: dict[tuple[str, str], list[dict[str, Any]]] = {}
+ grouped: dict[tuple[str, str, str | None], list[dict[str, Any]]] = {}
for case_id in _case_ids(agents):
for name, agent in agents.items():
for issue in _diagnose_agent_case(run_dir, name, agent, case_id, entry=dataset.get(case_id, {})):
- grouped.setdefault((case_id, issue["kind"]), []).append(issue)
+ grouped.setdefault((case_id, issue["kind"], issue.get("cause")), []).append(issue)
issues = []
for items in grouped.values():
names = [item["agent"] for item in items]
diff --git a/tests/test_tier3_case_summary.py b/tests/test_tier3_case_summary.py
index f5f24d56..751fd692 100644
--- a/tests/test_tier3_case_summary.py
+++ b/tests/test_tier3_case_summary.py
@@ -228,6 +228,143 @@ def test_run_errors_rank_after_skill_issues_and_are_not_blamed_on_the_skill(tmp_
assert fix["evidence"] == "opencode/with-skill/trials/broken__a/failure.json"
[email protected]("should_trigger", [True, False])
[email protected]("status", ["failed", "error"])
[email protected]("record_failure", [True, False])
+def test_judge_failures_are_unscored_infrastructure_advice(
+ tmp_path: Path, should_trigger: bool, status: str, record_failure: bool
+) -> None:
+ from skillevaluator.tier3.case_summary import ERROR, _cases, _marks
+
+ _trial(
+ tmp_path,
+ "a",
+ "c__1",
+ {"overall": 0.0, "evaluation_status": status, "evaluation_errors": {"accuracy": "Judge reply was empty"}},
+ )
+ (tmp_path / "dataset_snapshot.json").write_text(
+ json.dumps({"dataset": [{"id": "c", "should_trigger": should_trigger}]})
+ )
+ agent = _agent({}, {})
+ agent["output_dir"] = str(tmp_path / "a")
+ agent["skill_activation"] = {"status": "never_activated"}
+ if record_failure:
+ agent["trial_failures"] = {"with_skill": [{"trial": "c__1", "reason": "Required judge evaluation failed"}]}
+
+ assert _marks(_cases(agent, "with_skill")["c"]) == [ERROR]
+ [fix] = what_to_fix({"agents": {"a": agent}}, run_dir=tmp_path, use_llm=False)
+ assert fix["kind"] == "run error"
+ assert fix["attempts_lost"] == 1
+ assert "judge" in fix["action"].lower()
+ assert "Not a skill problem" in fix["action"]
+ assert "scored below" not in fix["what"]
+ assert fix["evidence"].endswith("reward.json")
+
+
+def test_judge_failure_does_not_change_genuine_scored_failure_advice(tmp_path: Path) -> None:
+ _trial(
+ tmp_path,
+ "a",
+ "c__1",
+ {"overall": 0.0, "evaluation_status": "failed", "evaluation_errors": {"accuracy": "Judge unavailable"}},
+ )
+ _trial(tmp_path, "a", "c__2", {"overall": 0.2, **_reward(True, "omitted validation")})
+ agent = _agent({}, {})
+ agent["output_dir"] = str(tmp_path / "a")
+ agent["trial_failures"] = {"with_skill": [{"trial": "c__1", "reason": "Required judge evaluation failed"}]}
+
+ fixes = what_to_fix({"agents": {"a": agent}}, run_dir=tmp_path, use_llm=False)
+ assert [(f["kind"], f["attempts_lost"]) for f in fixes] == [("skill", 1), ("run error", 1)]
+ assert "omitted validation" in fixes[0]["action"]
+ assert "judge" in fixes[1]["action"].lower()
+
+
+ ("errors", "reason", "action_text"),
+ [
+ ({"environment": "required dependency unavailable"}, "required dependency unavailable", "environment"),
+ ({"collector": "Judge sidecar could not be read; trial was not scored"}, "could not be read", "verifier"),
+ ],
+)
+def test_environment_and_collector_failures_do_not_get_judge_advice(
+ tmp_path: Path, errors: dict, reason: str, action_text: str
+) -> None:
+ _trial(tmp_path, "a", "c__1", {"overall": 0.0, "evaluation_status": "failed", "evaluation_errors": errors})
+ agent = _agent({}, {})
+ agent["output_dir"] = str(tmp_path / "a")
+ agent["trial_failures"] = {"with_skill": [{"trial": "c__1", "reason": "Evaluation failed"}]}
+
+ [fix] = what_to_fix({"agents": {"a": agent}}, run_dir=tmp_path, use_llm=False)
+
+ assert (fix["kind"], fix["attempts_lost"]) == ("run error", 1)
+ assert reason in fix["what"]
+ assert "required judge evaluation failed" not in fix["what"]
+ assert fix["action"].startswith("Not a skill problem")
+ assert action_text in fix["action"]
+ assert "judge" not in fix["action"].lower() and "credentials" not in fix["action"]
+ assert fix["evidence"].endswith("c__1/reward.json")
+
+
+def test_runtime_and_judge_failures_in_one_case_keep_their_own_causes_and_remedies(tmp_path: Path) -> None:
+ _trial(
+ tmp_path,
+ "a",
+ "c__judge",
+ {"overall": 0.0, "evaluation_status": "failed", "evaluation_errors": {"accuracy": "Judge reply was empty"}},
+ )
+ timeout = tmp_path / "a" / "with-skill" / "trials" / "c__timeout"
+ timeout.mkdir(parents=True)
+ (timeout / "failure.json").write_text(json.dumps({"error": {"type": "AgentTimeoutError", "message": "900s"}}))
+ agent = _agent({}, {})
+ agent["output_dir"] = str(tmp_path / "a")
+ agent["trial_failures"] = {
+ "with_skill": [
+ {"trial": "c__timeout", "reason": "Agent timed out"},
+ {"trial": "c__judge", "reason": "Required judge evaluation failed"},
+ ]
+ }
+
+ fixes = what_to_fix({"agents": {"a": agent}}, run_dir=tmp_path, use_llm=False)
+
+ assert [(f["kind"], f["attempts_lost"]) for f in fixes] == [("run error", 1), ("run error", 1)]
+ timeout_fix, judge_fix = fixes
+ assert "AgentTimeoutError" in timeout_fix["what"] and "judge" not in timeout_fix["what"].lower()
+ assert "agent/gateway logs" in timeout_fix["action"]
+ assert timeout_fix["evidence"].endswith("c__timeout/failure.json")
+ assert "Judge reply was empty" in judge_fix["what"]
+ assert "judge model configuration" in judge_fix["action"]
+ assert judge_fix["evidence"].endswith("c__judge/reward.json")
+
+
+def test_run_errors_with_different_causes_stay_separate_across_agents(tmp_path: Path) -> None:
+ timeout = tmp_path / "a" / "with-skill" / "trials" / "c__timeout"
+ timeout.mkdir(parents=True)
+ (timeout / "failure.json").write_text(json.dumps({"error": {"type": "AgentTimeoutError", "message": "900s"}}))
+ judge_failed = {
+ "overall": 0.0,
+ "evaluation_status": "failed",
+ "evaluation_errors": {"accuracy": "Judge unavailable"},
+ }
+ for name in ("b", "c"):
+ _trial(tmp_path, name, "c__1", judge_failed)
+ agents = {}
+ for name, trial, reason in (
+ ("a", "c__timeout", "Agent timed out"),
+ ("b", "c__1", "Required judge evaluation failed"),
+ ("c", "c__1", "Required judge evaluation failed"),
+ ):
+ agents[name] = _agent({}, {})
+ agents[name]["output_dir"] = str(tmp_path / name)
+ agents[name]["trial_failures"] = {"with_skill": [{"trial": trial, "reason": reason}]}
+
+ judge_fix, timeout_fix = what_to_fix({"agents": agents}, run_dir=tmp_path, use_llm=False)
+
+ assert (timeout_fix["scope"], timeout_fix["attempts_lost"]) == ("a only", 1)
+ assert "AgentTimeoutError" in timeout_fix["what"] and "agent/gateway logs" in timeout_fix["action"]
+ assert (judge_fix["scope"], judge_fix["attempts_lost"]) == ("b, c only", 2)
+ assert "judge" in judge_fix["what"].lower() and "judge model configuration" in judge_fix["action"]
+
+
def test_multi_agent_issue_scope_names_the_failing_agent(tmp_path: Path) -> None:
agents = {
"claude-code": _agent({"c": _case(True, True)}, {}),