Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

Preserve judge-aware lift in dashboard comparisons

YMuskrat2ff332f7daa8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 537cfdb0..06a0be14 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -83,7 +83,9 @@ All notable changes to SkillEvaluator are documented in this file.
- `run_config.json` records the judge reasoning effort, and each standard
judge's details record `judge_model_used`, `judge_reasoning_effort` and
`judge_fallback_used`. Skill lift leaves out a case whose two conditions were
- judged by different models and counts it in `judge_mixed_cases`.
+ judged by different models and counts it in `judge_mixed_cases`. Dashboard
+ score differences preserve this matching-judge lift, including unavailable
+ comparisons.
- Removed the unused `CONTENT_DEDUP_LLM_*` constants and `LLM_VERIFY_MODEL`; an
LLM client built from explicit credentials uses the provider default model.
- Tier 3 pins the agent CLIs Harbor installs: Codex 0.161.0 and Claude Code
diff --git a/src/skillevaluator/reporting/dashboard_data.py b/src/skillevaluator/reporting/dashboard_data.py
index 0cea8bca..65d4db03 100644
--- a/src/skillevaluator/reporting/dashboard_data.py
+++ b/src/skillevaluator/reporting/dashboard_data.py
@@ -425,6 +425,10 @@ def parse_dashboard_report(payload: Any, source: str) -> DashboardData:
),
"_ambiguous_trial_ids": tuple(sorted(ambiguous_ids)),
}
+ if "lift" in agent:
+ # Canonical lift can use only the matching-judge cohort;
+ # an explicit null must not become a full-cohort delta.
+ row["_score_lift"] = _number(agent["lift"])
_set_features(row, details, agent, report)
_set_usage(
row,
@@ -528,7 +532,10 @@ def comparison_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
result[f"with_{metric}"] = with_value
result[f"without_{metric}"] = without_value
available = paired and with_value is not None and without_value is not None
- result[f"{metric}_delta"] = _number(with_value - without_value) if available else None
+ delta = _number(with_value - without_value) if available else None
+ if metric == "score" and available and "_score_lift" in with_rows[0]:
+ delta = _number(with_rows[0]["_score_lift"])
+ result[f"{metric}_delta"] = delta
if metric in _USAGE:
result[f"{metric}_percent"] = (
_number(100 * (with_value - without_value) / without_value)
diff --git a/tests/reporting/test_dashboard_data.py b/tests/reporting/test_dashboard_data.py
index acab15e1..e7d6c9a7 100644
--- a/tests/reporting/test_dashboard_data.py
+++ b/tests/reporting/test_dashboard_data.py
@@ -76,6 +76,19 @@ def test_with_without_differences_and_explicit_features() -> None:
assert comparison["cost_usd_percent"] == 100
[email protected]("lift", [-0.1, 0.0, None])
+def test_score_delta_preserves_canonical_judge_aware_lift(lift: float | None) -> None:
+ report = _report()
+ report["agents"]["claude-code"]["lift"] = lift
+ comparison = comparison_rows(parse_dashboard_report(report, "report.json").rows)[0]
+ assert comparison["comparison_status"] == "comparable"
+ assert comparison["with_score"] == 0.8
+ assert comparison["without_score"] == 0.6
+ assert comparison["score_delta"] == lift
+ assert comparison["pass_rate_delta"] == 1.0
+ assert comparison["total_tokens_delta"] == 30
+
+
def test_single_trial_coverage_label_is_singular() -> None:
assert parse_dashboard_report(_report(), "r").rows[0]["coverage"].startswith("1 trial (complete)")