Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
Preserve matching-judge comparisons in skill lift
YMuskratf27059fdfcec
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 82207bbf..1ba799b8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -144,6 +144,10 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Tier 3 reports preserve judge-model exclusions when calculating dimension and+ overall lift. Native multi-step results also retain the judge provenance needed+ for those comparisons, so fallback judges cannot turn a regression into an+ apparent improvement.- Tier 3 reports an inconclusive result when every observed positive with-skilltrial has zero activation, and omits skill-lift comparisons for that agent.Activation counts are reported separately; negative controls and missingdiff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.pyindex d68342b8..711bee93 100644--- a/src/skillevaluator/evaluation/tier3_report.py+++ b/src/skillevaluator/evaluation/tier3_report.py@@ -1555,6 +1555,96 @@ def _condition_quality_available(info: dict[str, Any], condition: str) -> bool:return status not in {"failed", "unknown", "skipped"}+def _judge_aware_lift(+ info: dict[str, Any],+ with_scores: dict[str, Any],+ without_scores: dict[str, Any],+ score_policy: str,+) -> dict[str, Any] | None:+ """Rebuild judge exclusions from complete trial evidence, never cached deltas."""+ from skillevaluator.tier3.harbor.collector import (+ _LLM_JUDGE_METRICS,+ _compute_lift,+ _logical_attempt_rewards,+ _reward_judge_models,+ )+ from skillevaluator.tier3.harbor.metrics import metric_value+ from skillevaluator.tier3.harbor.report_data import logical_trial_reward_groups++ recorded = info.get("lift")+ recorded_mixed = recorded.get("judge_mixed_cases") if isinstance(recorded, dict) else None+ mixed = (+ {+ metric: count+ for metric, count in recorded_mixed.items()+ if metric in _LLM_JUDGE_METRICS and isinstance(count, int) and not isinstance(count, bool) and count > 0+ }+ if isinstance(recorded_mixed, dict)+ else {}+ )+ if not mixed and not any(+ _reward_judge_models(row, metric)+ for rows in (info.get("rewards"), info.get("rewards_baseline"))+ if isinstance(rows, list)+ for row in rows+ if isinstance(row, dict)+ for metric in _LLM_JUDGE_METRICS+ ):+ return None+ conditions: list[list[dict[str, Any]]] = []+ complete = True+ for key in ("rewards", "rewards_baseline"):+ rows = info.get(key)+ complete = complete and info.get(f"{key}_complete") is True+ groups = logical_trial_reward_groups(+ [row for row in rows if isinstance(row, dict)] if isinstance(rows, list) else []+ )+ logical_rows: list[dict[str, Any]] = []+ for index, group in enumerate(groups):+ if not _logical_group_entry_identity_is_consistent(group):+ complete = False+ continue+ logical_rows.extend(+ {+ **row,+ "_trial_root_name": str(index),+ **({"_step_name": str(step)} if len(group) > 1 else {}),+ }+ for step, row in enumerate(group)+ )+ conditions.append(_logical_attempt_rewards(logical_rows))+ clean_scores = [+ {metric: value for metric, raw in scores.items() if (value := _unit_interval_score(raw)) is not None}+ for scores in (with_scores, without_scores)+ ]+ rebuilt = _compute_lift(*clean_scores, *conditions, score_policy=score_policy)+ rebuilt_mixed = rebuilt.get("judge_mixed_cases", {})+ unsupported_exclusions = {metric for metric, count in mixed.items() if rebuilt_mixed.get(metric, 0) < count}+ for metric, count in rebuilt_mixed.items():+ mixed[metric] = max(count, mixed.get(metric, 0))+ if not mixed:+ return None+ unsupported_exclusions.update(+ metric+ for metric in mixed+ if any(+ metric_value(row, metric) is not None and not _reward_judge_models(row, metric)+ for rows in conditions+ for row in rows+ )+ )+ if not complete:+ # A bounded or incomplete reward load cannot reconstruct the retained+ # cohort. Preserve unavailable instead of undoing the exclusion.+ return {"judge_mixed_cases": mixed}+ rebuilt["judge_mixed_cases"] = mixed+ for metric in unsupported_exclusions:+ rebuilt.pop(metric, None)+ if unsupported_exclusions:+ rebuilt.pop("overall", None)+ return rebuilt++def _build_agent(name: str,info: dict[str, Any],@@ -1583,7 +1673,7 @@ def _build_agent(comparable_scores = (not never_activated and set(with_metrics) == set(baseline_metrics) and len(condition_policies) == 1)- lift_data = info.get("lift") or {}+ lift_data = info.get("lift") if isinstance(info.get("lift"), dict) else {}with_quality_available = _condition_quality_available(info, "with_skill")baseline_quality_available = _condition_quality_available(info, "without_skill")if not with_quality_available:@@ -1591,12 +1681,16 @@ def _build_agent(if not baseline_quality_available:without_scores = {}+ judge_lift = _judge_aware_lift(info, with_scores, without_scores, score_policy)+ if judge_lift is not None:+ lift_data = judge_liftevaluators = _build_evaluators(with_metrics, with_scores, without_scores, lift_data)dimensions = _build_dimensions(with_scores,without_scores,info.get("dimensions_with_skill") or {},info.get("dimensions_without_skill") or {},+ judge_lift,)if never_activated:for evaluator in evaluators.values():@@ -1677,6 +1771,12 @@ def _build_agent(sum(w - b for w, b in zip(with_values, baseline_values, strict=True)) / len(with_metrics),4,)+ if judge_lift is not None:+ overall_lift = (+ _lift_value("overall", judge_lift)+ if comparable_scores and not (with_mixed_contract or baseline_mixed_contract)+ else None+ )trials = _normalize_trials(info.get("rewards") or [], with_metrics)baseline_trials = _normalize_trials(info.get("rewards_baseline") or [], baseline_metrics)@@ -1770,18 +1870,21 @@ def _build_evaluators(lift_data: dict[str, Any],) -> dict[str, dict[str, Any]]:evaluators: dict[str, dict[str, Any]] = {}+ mixed = lift_data.get("judge_mixed_cases")+ mixed = mixed if isinstance(mixed, dict) else {}for metric in metrics:ws = _finite_float(with_scores.get(metric))if ws is None:continuebl = _finite_float(without_scores.get(metric))lift = _lift_value(metric, lift_data)- if lift is None and bl is not None:+ excluded = metric in mixed+ if lift is None and bl is not None and not excluded:lift = round(ws - bl, 4)evaluators[metric] = {"with_skill": ws,"baseline": bl,- "lift": lift if lift is not None else 0.0,+ "lift": lift if lift is not None or excluded else 0.0,}return evaluators@@ -1791,6 +1894,7 @@ def _build_dimensions(without_scores: dict[str, Any],precomputed_with: dict[str, Any],precomputed_without: dict[str, Any],+ judge_lift: dict[str, Any] | None = None,) -> list[dict[str, Any]]:dimensions: list[dict[str, Any]] = []for dim_id in _DIMENSION_IDS:@@ -1811,8 +1915,37 @@ def _build_dimensions(signals = _dimension_signals(entry, with_scores, cfg)explanation = entry.get("explanation")reasoning_bullets = entry.get("reasoning_bullets")+ comparison_signals = _dimension_signals({}, with_scores, cfg)+ excluded = judge_lift is not None and any(+ metric in judge_lift.get("judge_mixed_cases", {}) for metric in comparison_signals+ )+ reasoning_baseline = bl+ if excluded and judge_lift is not None:+ pairs = {metric: judge_lift.get(metric) for metric in comparison_signals}+ paired_with = {metric: pair.get("with_skill") for metric, pair in pairs.items() if isinstance(pair, dict)}+ paired_without = {+ metric: pair.get("without_skill") for metric, pair in pairs.items() if isinstance(pair, dict)+ }+ comparison_ws = _dimension_score(paired_with, cfg) if len(paired_with) == len(comparison_signals) else None+ reasoning_baseline = (+ _dimension_score(paired_without, cfg) if len(paired_without) == len(comparison_signals) else None+ )+ lift = (+ round(comparison_ws - reasoning_baseline, 4)+ if comparison_ws is not None and reasoning_baseline is not None+ else None+ )+ reasoning_bullets = explanation = Noneif not reasoning_bullets and not explanation:- reasoning_bullets, explanation = _deterministic_reasoning(ws, bl, lift, signals, with_scores)+ reasoning_bullets, explanation = _deterministic_reasoning(+ ws, reasoning_baseline, lift, signals, with_scores+ )+ if excluded:+ note = "Cases scored by different judge models are excluded from lift."+ if lift is None:+ reasoning_bullets[1] = "Lift unavailable: no complete comparison with matching judge models."+ reasoning_bullets.append(note)+ explanation = " ".join(reasoning_bullets)verdict = entry.get("verdict") or _deterministic_verdict(ws)dimensions.append({diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.pyindex de812e5f..42ec7ebd 100644--- a/src/skillevaluator/tier3/harbor/collector.py+++ b/src/skillevaluator/tier3/harbor/collector.py@@ -3513,6 +3513,7 @@ def _extract_rewards(data = _reward_from_harbor_result(result)if not data:continue+ _merge_constituent_judge_models(data, result, trial_dir)_merge_constituent_default_reward_failure(data, result, trial_dir)_merge_trial_evaluation_failures(data, trial_dir)trial_name = str(result.get("trial_name") or trial_dir.name)@@ -4951,6 +4952,30 @@ def _pass_summary(_LLM_JUDGE_METRICS = ("accuracy", "goal_accuracy", "behavior_check")+def _merge_constituent_judge_models(data: dict[str, Any], result: dict[str, Any], trial_root: Path) -> None:+ """Keep step judge provenance without replacing Harbor's aggregate scores."""+ names = _valid_step_result_names(result.get("step_results"))+ if names is None:+ return+ models: dict[str, set[str]] = {metric: set() for metric in _LLM_JUDGE_METRICS}+ for name, step in zip(names, result["step_results"], strict=True):+ verifier = step.get("verifier_result")+ embedded = verifier.get("rewards") if isinstance(verifier, dict) else None+ row = dict(embedded) if isinstance(embedded, dict) else {}+ verifier_dir = trial_root / "steps" / name / "verifier"+ if _safe_trial_path_component(name) == name and _path_is_within(verifier_dir, trial_root):+ _merge_reward_sidecars(row, verifier_dir)+ for metric, values in models.items():+ values.update(_reward_judge_models(row, metric))+ carried = {metric: sorted(values) for metric, values in models.items() if values}+ if carried:+ data["_judge_models"] = carried+ # Details survive publication; private collector fields do not.+ details = data.setdefault("details", {})+ for metric, values in carried.items():+ details[metric] = {"judge_models": values}++def _reward_judge_models(reward: dict[str, Any], metric: str) -> set[str]:"""Return the judge models that scored ``metric`` in one reward row.@@ -4962,6 +4987,8 @@ def _reward_judge_models(reward: dict[str, Any], metric: str) -> set[str]:return {model for model in models if isinstance(model, str) and model} if isinstance(models, list) else set()details = reward.get("details")detail = details.get(metric) if isinstance(details, dict) else None+ if isinstance(detail, dict) and isinstance(detail.get("judge_models"), list):+ return {model for model in detail["judge_models"] if isinstance(model, str) and model}model = detail.get("judge_model_used") if isinstance(detail, dict) else Nonereturn {model} if isinstance(model, str) and model else set()diff --git a/tests/test_judge_lift_integrity.py b/tests/test_judge_lift_integrity.pynew file mode 100644index 00000000..38e51666--- /dev/null+++ b/tests/test_judge_lift_integrity.py@@ -0,0 +1,192 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Judge fallback exclusions survive collection and canonical report reloads."""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory+from skillevaluator.tier3.harbor.collector import _extract_rewards, collect_harbor_results+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS, DEFAULT_SCORE_POLICY+++def _write_json(path: Path, payload: dict) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(json.dumps(payload), encoding="utf-8")+++def _write_job(job: Path, accuracies: list[float], judges: list[tuple[str, ...]], *, native: bool) -> None:+ trials = []+ for index, (accuracy, step_judges) in enumerate(zip(accuracies, judges, strict=True), 1):+ case = f"case-{index}"+ trial_name = f"{case}__trial"+ trials.append(trial_name)+ trial = job / trial_name+ scores = {**dict.fromkeys(DEFAULT_METRICS, 0.5), "accuracy": accuracy}+ steps = []+ for step_index, judge in enumerate(step_judges):+ name = f"step-{step_index}"+ verifier = trial / "steps" / name / "verifier" if native else trial / "verifier"+ rich = {+ **scores,+ "entry_id": case,+ "metric_set": DEFAULT_METRIC_SET,+ "score_policy": DEFAULT_SCORE_POLICY,+ "details": {"accuracy": {"score": accuracy, "judge_model_used": judge}},+ }+ _write_json(verifier / "reward.json", scores if native else rich)+ if native:+ _write_json(verifier / "skill_evaluator_reward.json", rich)+ steps.append({"step_name": name, "verifier_result": {"rewards": scores}})+ if native:+ _write_json(+ trial / "result.json",+ {+ "trial_name": trial_name,+ "task_name": case,+ "verifier_result": {"rewards": scores},+ "step_results": steps,+ },+ )+ _write_json(+ job / "result.json",+ {+ "n_total_trials": len(trials),+ "stats": {+ "n_completed_trials": len(trials),+ "n_errored_trials": 0,+ "n_running_trials": 0,+ "n_pending_trials": 0,+ "n_cancelled_trials": 0,+ "n_retries": 0,+ "evals": {+ "agent__model___tasks": {+ "n_trials": len(trials),+ "n_errors": 0,+ "reward_stats": {"overall": {"0.5": trials}},+ }+ },+ },+ },+ )+++def _collect(root: Path, *, native: bool = False, all_mixed: bool = False) -> tuple[dict, Path]:+ judges_with = [("primary", "fallback"), ("primary", "fallback" if all_mixed else "primary")]+ judges_without = [("primary", "primary"), ("primary", "primary")]+ if not native:+ judges_with = [(row[-1],) for row in judges_with]+ judges_without = [(row[-1],) for row in judges_without]+ _write_job(root / "jobs" / "demo-codex-with", [1.0, 0.0], judges_with, native=native)+ _write_job(root / "jobs" / "demo-codex-without", [0.0, 0.4], judges_without, native=native)+ run = root / "run"+ collected = collect_harbor_results(+ "demo", ["codex"], run, root / "jobs", expected_case_ids=["case-1", "case-2"], expected_trials=2+ )+ assert collected["execution_status"] == "succeeded"+ return collected, run+++def _report(run: Path) -> dict:+ return agent_eval_result_from_directory(run.parent / "demo", run, use_llm_judge=False).metadata["agent_eval"]++[email protected]("native", [False, True])+def test_report_retains_judge_filtered_regression(tmp_path: Path, native: bool) -> None:+ collected, run = _collect(tmp_path, native=native)+ lift = collected["agents"]["codex"]["lift"]+ assert lift["judge_mixed_cases"] == {"accuracy": 1}+ assert lift["accuracy"]["delta"] == -0.4+ assert lift["overall"]["delta"] == -0.08++ # A stale cache must not reverse the comparison or its explanation.+ cached = run / "codex" / "lift.json"+ payload = json.loads(cached.read_text())+ payload["accuracy"]["delta"] = 0.9+ payload["overall"]["delta"] = 0.9+ _write_json(cached, payload)+ summary = run / "codex" / "with-skill" / "summary.json"+ payload = json.loads(summary.read_text())+ payload["dimensions"]["correctness"].update(+ explanation="+0.30 lift over baseline 0.20.",+ reasoning_bullets=["Improvement!"],+ sources={"goal_accuracy": 0.5},+ )+ _write_json(summary, payload)++ report = _report(run)+ agent = report["agents"]["codex"]+ assert report["overall_lift"] == agent["lift"] == -0.08+ assert agent["with_skill"] == 0.5+ assert agent["baseline"] == 0.44+ assert agent["evaluators"]["accuracy"]["lift"] == -0.4+ correctness = next(dim for dim in agent["dimensions"] if dim["id"] == "correctness")+ assert correctness["lift"] == -0.4+ assert "-0.40 regression versus baseline 0.40" in correctness["explanation"]+ assert "different judge models" in correctness["explanation"]+ assert "+0.30" not in correctness["explanation"]++[email protected]("native", [False, True])+def test_all_mixed_judges_leave_metric_dimension_and_overall_unavailable(tmp_path: Path, native: bool) -> None:+ collected, run = _collect(tmp_path, native=native, all_mixed=True)+ lift = collected["agents"]["codex"]["lift"]+ assert lift["judge_mixed_cases"] == {"accuracy": 2}+ assert "accuracy" not in lift+ assert "overall" not in lift+ report = _report(run)+ agent = report["agents"]["codex"]+ assert report["overall_lift"] is None+ assert agent["lift"] is None+ assert agent["evaluators"]["accuracy"]["lift"] is None+ correctness = next(dim for dim in agent["dimensions"] if dim["id"] == "correctness")+ assert correctness["lift"] is None+ assert "Lift unavailable" in correctness["explanation"]+ assert "No baseline run" not in correctness["explanation"]+++ "missing_evidence", ["reward", "judge_metadata", "partial_judge_metadata", "retained_judge_metadata"]+)+def test_exclusions_cannot_be_reconstructed_from_incomplete_evidence(tmp_path: Path, missing_evidence: str) -> None:+ _, run = _collect(tmp_path, all_mixed=missing_evidence == "partial_judge_metadata")+ reward = next((run / "codex" / "with-skill" / "trials").glob("*/reward.json"))+ if missing_evidence == "reward":+ reward.unlink()+ else:+ for reward in (run / "codex").glob("*/trials/*/reward.json"):+ payload = json.loads(reward.read_text())+ if (+ missing_evidence in {"partial_judge_metadata", "retained_judge_metadata"}+ and payload["entry_id"] != "case-2"+ ):+ continue+ payload.pop("details", None)+ _write_json(reward, payload)+ report = _report(run)+ assert report["overall_lift"] is None+ assert report["agents"]["codex"]["evaluators"]["accuracy"]["lift"] is None+++def test_native_aggregate_keeps_numeric_authority_and_only_declared_step_judges(tmp_path: Path) -> None:+ job = tmp_path / "job"+ _write_job(job, [0.25], [("primary", "fallback")], native=True)+ trial = job / "case-1__trial"+ for sidecar in trial.glob("steps/*/verifier/skill_evaluator_reward.json"):+ payload = json.loads(sidecar.read_text())+ payload["accuracy"] = 1.0+ _write_json(sidecar, payload)+ _write_json(+ trial / "steps" / "orphan" / "verifier" / "skill_evaluator_reward.json",+ {"details": {"accuracy": {"judge_model_used": "unrelated"}}},+ )+ rows = _extract_rewards(job)+ assert len(rows) == 1+ assert rows[0]["accuracy"] == 0.25+ assert rows[0]["_judge_models"] == {"accuracy": ["fallback", "primary"]}+ assert rows[0]["details"]["accuracy"]["judge_models"] == ["fallback", "primary"]