Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): show the run's Overall lift in tier3 compare when a metric is N/A in one arm (#386)

Christopher Kevind81315e6b0f7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0c17504a..b4e27d46 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -287,6 +287,11 @@ All notable changes to SkillEvaluator are documented in this file.
`Use HTTP POST to create an item.` in a here-document or an unpiped `echo`
still does not warn.
+- `tier3 compare` reads standard-policy Overall lift from `lift.json` when the
+ arms have different metric applicability, matching the canonical report.
+ Missing lift data is shown as `-`; custom-only and legacy agents retain their
+ arm-difference comparison.
+
## 0.8.0 - 2026-11-05
### Added
diff --git a/docs/reports.mdx b/docs/reports.mdx
index 93b99eb3..505cef7e 100644
--- a/docs/reports.mdx
+++ b/docs/reports.mdx
@@ -660,7 +660,10 @@ and mark the overall comparison as incomplete, without selecting a best agent.
A custom-only agent, one whose conditions use only its grader's own overall
(`custom-overall-v1`), is still ranked against standard agents by that score,
and `tier3 compare` shows it beside them. `tier3 compare` refuses standard
-agents scored under different policies or metric sets.
+agents scored under different policies or metric sets. When a metric is N/A in
+only one arm, `tier3 compare` drops it from both arms for the Overall lift, as
+`lift.json` does, so its lift can differ from the difference between the two
+arm overall scores.
### Skill Lift
diff --git a/src/skillevaluator/tier3/commands.py b/src/skillevaluator/tier3/commands.py
index a2ce8e82..7df35dde 100644
--- a/src/skillevaluator/tier3/commands.py
+++ b/src/skillevaluator/tier3/commands.py
@@ -49,6 +49,7 @@ from skillevaluator.tier3.harbor.metrics import (
LEGACY_METRICS,
LEGACY_SCORE_POLICY,
PARTIAL_SCORE_POLICY,
+ not_applicable_list,
overall_score_from_metrics,
)
from skillevaluator.tier3.harbor.progress import (
@@ -1211,6 +1212,7 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
"score_policy_with_skill": _summary_score_policy(data),
"standard_metrics": _summary_standard_metrics(data),
"custom_only": _summaries_are_custom_only([data]),
+ "not_applicable_with_skill": not_applicable_list(data.get("not_applicable_metrics")),
}
wo_summary = agent_dir / "without-skill" / "summary.json"
if wo_summary.exists():
@@ -1238,6 +1240,9 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
{
"overall_without_skill": wo_overall,
"score_policy_without_skill": _summary_score_policy(wo_data),
+ "not_applicable_without_skill": not_applicable_list(
+ wo_data.get("not_applicable_metrics")
+ ),
}
)
except (ValueError, OSError):
@@ -1304,6 +1309,7 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
table.add_row(*[""] * (1 + sum(2 if agent in agent_without else 1 for agent in agents)))
overall_row: list[str | Text] = [Text("Overall", style="bold")]
+ lift_json_agents: list[str] = []
for agent in agents:
meta = agent_meta[agent]
with_avg = _overall_score_for_display(
@@ -1324,7 +1330,13 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
persisted_overall=meta.get("overall_without_skill"),
score_policy=meta.get("score_policy_without_skill"),
)
- if with_avg is None or without_avg is None:
+ uses_lift_json, shared_lift = _shared_dimension_overall_lift(
+ agent_with[agent], agent_without[agent], overall_metrics, meta
+ )
+ if uses_lift_json and shared_lift is not None:
+ lift_json_agents.append(agent)
+ overall_row.append(_lift_text(shared_lift))
+ elif uses_lift_json or with_avg is None or without_avg is None:
overall_row.append(Text("-", style="dim"))
else:
delta = with_avg - without_avg
@@ -1348,10 +1360,50 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> int
# Pad before escaping so escape backslashes do not count toward the column width.
run_line = f"{agent:<16s} {meta['timestamp']} (Harbor, {meta.get('num_trials', '?')} trials)"
console.print(f" [dim]{escape_markup(run_line)}[/dim]")
+ if lift_json_agents:
+ console.print(
+ f" [dim]Overall lift for {escape_markup(', '.join(lift_json_agents))} compares only the "
+ "dimensions both arms scored, as in lift.json and the reports.[/dim]"
+ )
console.print()
return 0
+def _shared_dimension_overall_lift(
+ with_scores: dict[str, float],
+ without_scores: dict[str, float],
+ metrics: tuple[str, ...],
+ meta: dict[str, Any],
+) -> tuple[bool, float | None]:
+ """Return the canonical standard-policy Overall lift when shared metrics differ."""
+ shared_not_applicable = sorted(
+ {*meta.get("not_applicable_with_skill", ()), *meta.get("not_applicable_without_skill", ())}
+ )
+ if (
+ not shared_not_applicable
+ or metrics != DEFAULT_METRICS
+ or meta.get("score_policy_with_skill") != DEFAULT_SCORE_POLICY
+ or meta.get("score_policy_without_skill") != DEFAULT_SCORE_POLICY
+ ):
+ return False, None
+ agent_path = meta.get("path")
+ if not isinstance(agent_path, str) or not agent_path:
+ return True, None
+ try:
+ lift_data = json.loads((Path(agent_path) / "lift.json").read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return True, None
+ if not isinstance(lift_data, dict) or not isinstance(lift_data.get("overall"), dict):
+ return True, None
+ return True, _finite_numeric(lift_data["overall"].get("delta"))
+
+
+def _lift_text(delta: float) -> Text:
+ delta_text = f"+{delta:.2f}" if delta > 0 else f"{delta:.2f}"
+ delta_style = "bold green" if delta > 0 else ("bold red" if delta < 0 else "bold dim")
+ return Text(delta_text, style=delta_style)
+
+
def _summary_scores(data: dict[str, Any], *, allow_missing_status: bool = False) -> dict[str, float]:
if not isinstance(data, dict):
return {}
diff --git a/tests/test_tier3_compare.py b/tests/test_tier3_compare.py
index 5125dde4..76c39e62 100644
--- a/tests/test_tier3_compare.py
+++ b/tests/test_tier3_compare.py
@@ -784,3 +784,141 @@ def test_compare_never_mixes_agents_across_candidate_roots(
output = capsys.readouterr().out
assert "codex" in output
assert "opencode" not in output
+
+
+def _write_harbor_job(
+ job_dir: Path,
+ scores: dict[str, float],
+ not_applicable: tuple[str, ...] = (),
+ *,
+ case_scores: dict[str, dict[str, float]] | None = None,
+ judge_models_by_case: dict[str, dict[str, str]] | None = None,
+) -> None:
+ from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET
+
+ trials = []
+ for index in (1, 2):
+ trial = f"case-{index}__trial"
+ case_id = f"case-{index}"
+ trials.append(trial)
+ reward: dict[str, object] = dict((case_scores or {}).get(case_id, scores))
+ details: dict[str, dict[str, object]] = {}
+ for metric in not_applicable:
+ reward[metric] = None
+ details[metric] = {"score": None, "status": "not_applicable", "reason": "no skill under test"}
+ for metric in DEFAULT_METRICS:
+ if metric not in not_applicable and metric != "skill_execution":
+ details[metric] = {
+ "score": reward[metric],
+ "judge_model_used": (judge_models_by_case or {}).get(case_id, {}).get(metric, "judge-a"),
+ }
+ if "skill_execution" not in not_applicable:
+ details["skill_execution"] = {"activation": {"passed": True, "score": 1.0}}
+ reward.update(
+ entry_id=case_id,
+ metric_set=DEFAULT_METRIC_SET,
+ score_policy=DEFAULT_SCORE_POLICY,
+ details=details,
+ )
+ path = job_dir / trial / "verifier" / "reward.json"
+ path.parent.mkdir(parents=True)
+ path.write_text(json.dumps(reward), encoding="utf-8")
+ stats = {
+ "n_completed_trials": 2,
+ "n_errored_trials": 0,
+ "n_running_trials": 0,
+ "n_pending_trials": 0,
+ "n_cancelled_trials": 0,
+ "n_retries": 0,
+ "evals": {"agent__model___tasks": {"n_trials": 2, "n_errors": 0, "reward_stats": {"overall": {"0.5": trials}}}},
+ }
+ (job_dir / "result.json").write_text(json.dumps({"n_total_trials": 2, "stats": stats}), encoding="utf-8")
+
+
+ ("baseline_not_applicable", "with_overall", "lift", "judge_fallback", "never_activated"),
+ [
+ # A six-metric mean would give a passing +0.10; the five-dimension policy gives -0.08.
+ ((), 0.6, -0.08, False, False),
+ # The baseline has no skill under test, so the skill metrics drop out of both arms for lift.
+ (("skill_execution", "skill_efficiency"), 0.6, 0.1, False, False),
+ (("skill_execution", "skill_efficiency"), None, None, True, False),
+ (("skill_execution", "skill_efficiency"), 0.6, None, False, True),
+ ],
+ ids=[
+ "five-dimension-reverses-six-metric-lift",
+ "baseline-skill-metrics-not-applicable",
+ "judge-fallback-drops-baseline-case",
+ "never-activated-without-lift-file",
+ ],
+)
+def test_compare_publishes_the_same_overall_and_lift_as_the_run_outputs(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+ baseline_not_applicable: tuple[str, ...],
+ with_overall: float | None,
+ lift: float | None,
+ judge_fallback: bool,
+ never_activated: bool,
+) -> None:
+ from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory
+ from skillevaluator.tier3.harbor.collector import collect_harbor_results
+
+ skill_path = tmp_path / "demo"
+ skill_path.mkdir()
+ (skill_path / "SKILL.md").write_text("---\nname: demo\ndescription: Demo skill.\n---\nBody.\n", encoding="utf-8")
+ with_skill = {**dict.fromkeys(DEFAULT_METRICS, 0.5), "goal_accuracy": 1.0, "behavior_check": 1.0}
+ baseline = {**dict.fromkeys(DEFAULT_METRICS, 0.85), "goal_accuracy": 0.0, "behavior_check": 0.0}
+ jobs = tmp_path / "jobs"
+ with_case_scores = None
+ baseline_case_scores = None
+ baseline_judge_models = None
+ if judge_fallback:
+ with_case_scores = {
+ "case-1": dict.fromkeys(DEFAULT_METRICS, 0.9),
+ "case-2": dict.fromkeys(DEFAULT_METRICS, 0.1),
+ }
+ baseline_case_scores = {
+ "case-1": dict.fromkeys(DEFAULT_METRICS, 0.5),
+ "case-2": dict.fromkeys(DEFAULT_METRICS, 0.1),
+ }
+ baseline_judge_models = {"case-2": {"accuracy": "judge-b"}}
+ _write_harbor_job(jobs / "demo-codex-with", with_skill, case_scores=with_case_scores)
+ _write_harbor_job(
+ jobs / "demo-codex-without",
+ baseline,
+ baseline_not_applicable,
+ case_scores=baseline_case_scores,
+ judge_models_by_case=baseline_judge_models,
+ )
+ run_dir = tmp_path / "results" / "demo" / "run"
+ collect_harbor_results("demo", ["codex"], run_dir, jobs, expected_case_ids=["case-1", "case-2"], expected_trials=2)
+ summary = json.loads((run_dir / "codex" / "with-skill" / "summary.json").read_text(encoding="utf-8"))
+ if with_overall is not None:
+ assert summary["overall_score"] == pytest.approx(with_overall)
+ lift_path = run_dir / "codex" / "lift.json"
+ if never_activated:
+ summary["skill_activation"] = {"status": "never_activated"}
+ (run_dir / "codex" / "with-skill" / "summary.json").write_text(json.dumps(summary), encoding="utf-8")
+ lift_path.unlink()
+ recorded_lift = None
+ else:
+ recorded_lift = json.loads(lift_path.read_text(encoding="utf-8"))["overall"]["delta"]
+ agent_eval = agent_eval_result_from_directory(skill_path, run_dir, use_llm_judge=False).metadata["agent_eval"]
+ assert recorded_lift == agent_eval["overall_lift"]
+ if lift is not None:
+ assert recorded_lift == pytest.approx(lift)
+ if judge_fallback:
+ without_summary = json.loads((run_dir / "codex" / "without-skill" / "summary.json").read_text(encoding="utf-8"))
+ assert recorded_lift != pytest.approx(summary["overall_score"] - without_summary["overall_score"])
+ capsys.readouterr()
+
+ assert compare_results(skill_path, results_dir=tmp_path / "results") == 0
+
+ output = capsys.readouterr().out
+ with_overall = summary["overall_score"]
+ expected_lift = (
+ "-" if recorded_lift is None else (f"+{recorded_lift:.2f}" if recorded_lift > 0 else f"{recorded_lift:.2f}")
+ )
+ assert re.search(rf"Overall\s+{with_overall:.2f}\s+{re.escape(expected_lift)}\s", output)
+ assert ("compares only the" in output) is (bool(baseline_not_applicable) and not never_activated)