Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(reporting): let default Tier 3 runs with a baseline pass publication (#332)
Saibernard194f7f975e9d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b122a44c..4dfe5d98 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -317,6 +317,18 @@ All notable changes to SkillEvaluator are documented in this file.the dataset-size status in `result.json` and the `tier3 evaluate` summary. Itis taken from the agent with the best baseline score, as in the reports.+- A default Tier 3 run with a baseline can reach a PASS or NEUTRAL publication+ verdict in BENCHMARK.md again. The baseline arm has no skill under test, so+ it records `skill_execution` and `skill_efficiency` as not applicable and has+ no Discoverability or Efficiency baseline. The publication check treated+ those two dimensions as missing evidence and compared Skill Lift with the+ five-dimension score, so those cards said INCOMPLETE instead. Discoverability+ and Efficiency now count as not applicable for the baseline when it recorded+ their skill metrics that way, and Skill Lift is checked over the dimensions+ both arms scored. On the card those two rows say `baseline N/A` instead of+ `baseline not run`. Any other missing or inconsistent baseline still makes+ the card INCOMPLETE.+- `BENCHMARK.md` no longer publishes an Overall uplift that disagrees with`lift.json` and the JSON, HTML, Markdown and terminal reports when a metric isN/A in only one arm. The Overall row now shows the two scores the publisheddiff --git a/src/skillevaluator/reporting/base.py b/src/skillevaluator/reporting/base.pyindex 10d5a79b..007c8b4a 100644--- a/src/skillevaluator/reporting/base.py+++ b/src/skillevaluator/reporting/base.py@@ -1145,6 +1145,30 @@ def _agent_eval_execution_errors_clear(payload: dict[str, Any] | None) -> bool:return True+# The metrics a baseline without the skill under test cannot score. It has no+# skill to discover or route to, so the verifier records them as not applicable.+BASELINE_NOT_APPLICABLE_SKILL_METRICS = ("skill_execution", "skill_efficiency")+++def agent_eval_baseline_not_applicable(agent: dict[str, Any], dimension: dict[str, Any]) -> bool:+ """Return whether the baseline recorded a dimension as not applicable instead of scoring it.++ Only the skill metrics qualify. Any other metric is not applicable when the+ case has no reference to judge against, which holds for both arms, so a+ dimension that is missing only its baseline is missing evidence.+ """+ config = DIMENSION_MAPPING.get(_agent_eval_safe_text(dimension.get("id")))+ if config is None or dimension.get("baseline") is not None or dimension.get("lift") is not None:+ return False+ raw_not_applicable = agent.get("not_applicable_metrics")+ recorded = raw_not_applicable.get("without_skill") if isinstance(raw_not_applicable, dict) else None+ if not isinstance(recorded, list):+ return False+ return all(+ metric in BASELINE_NOT_APPLICABLE_SKILL_METRICS and metric in recorded for metric in config["evaluators"]+ )++def _agent_eval_dimension_scores(agent: dict[str, Any],*,@@ -1165,6 +1189,7 @@ def _agent_eval_dimension_scores(scores: list[float] = []baselines: list[float] = []+ compared_scores: list[float] = []for dimension_id in DIMENSION_MAPPING:dimension = dimensions.get(dimension_id)if dimension is None:@@ -1172,14 +1197,18 @@ def _agent_eval_dimension_scores(value = dimension.get("with_skill") if "with_skill" in dimension else dimension.get("score")score = _agent_eval_finite_score(value)baseline = _agent_eval_finite_score(dimension.get("baseline"))- if score is None or (require_baseline and baseline is None):+ # A baseline without the skill under test has no baseline and no lift+ # on the dimensions the skill metrics feed.+ baseline_not_applicable = agent_eval_baseline_not_applicable(agent, dimension)+ if score is None or (require_baseline and baseline is None and not baseline_not_applicable):return None- if require_baseline:+ if require_baseline and not baseline_not_applicable:assert baseline is not Nonelift = _agent_eval_finite_number(dimension.get("lift"))if lift is None or not math.isclose(lift, score - baseline, rel_tol=1e-9, abs_tol=5e-4):return Nonebaselines.append(baseline)+ compared_scores.append(score)scores.append(score)agent_score = _agent_eval_finite_score(agent.get("with_skill", agent.get("overall_score")))@@ -1194,7 +1223,8 @@ def _agent_eval_dimension_scores(agent_baseline = _agent_eval_finite_score(agent.get("baseline"))agent_lift = _agent_eval_finite_number(agent.get("lift"))if (- agent_baseline is None+ not baselines+ or agent_baseline is Noneor agent_lift is Noneor not math.isclose(agent_baseline,@@ -1202,7 +1232,13 @@ def _agent_eval_dimension_scores(rel_tol=1e-9,abs_tol=5e-4,)- or not math.isclose(agent_lift, agent_score - agent_baseline, rel_tol=1e-9, abs_tol=5e-4)+ # The lift compares both arms on the dimensions the baseline scored.+ or not math.isclose(+ agent_lift,+ sum(compared_scores) / len(compared_scores) - agent_baseline,+ rel_tol=1e-9,+ abs_tol=5e-4,+ )):return Nonereturn scoresdiff --git a/src/skillevaluator/reporting/benchmark.py b/src/skillevaluator/reporting/benchmark.pyindex 0e28c280..b82b953e 100644--- a/src/skillevaluator/reporting/benchmark.py+++ b/src/skillevaluator/reporting/benchmark.py@@ -28,6 +28,7 @@ from skillevaluator.publication_text import publication_confusable_skeletonfrom skillevaluator.reporting.base import (PublicationTargetIdentity,ReporterBase,+ agent_eval_baseline_not_applicable,agent_eval_dimension_verdict,agent_eval_publication_dataset_provenance,agent_eval_publication_evaluated_at,@@ -1429,6 +1430,7 @@ def _score_transition(dimension: dict[str, Any] | None,*,skill_never_activated: bool = False,+ baseline_not_applicable: bool = False,baseline_only: bool = False,) -> str:if not dimension:@@ -1437,6 +1439,7 @@ def _score_transition(dimension.get("baseline"),dimension.get("with_skill", dimension.get("score")),skill_never_activated=skill_never_activated,+ baseline_not_applicable=baseline_not_applicable,baseline_only=baseline_only,)@@ -1497,7 +1500,12 @@ def _dimension_cell("""Render one Results at a Glance dimension cell, explaining a missing score."""dimension = _agent_dimension(agent, dim_id)if dimension is not None:- return _score_transition(dimension, skill_never_activated=skill_never_activated, baseline_only=baseline_only)+ return _score_transition(+ dimension,+ skill_never_activated=skill_never_activated,+ baseline_only=baseline_only,+ baseline_not_applicable=agent_eval_baseline_not_applicable(agent, dimension),+ )not_applicable = agent.get("not_applicable_dimensions")if isinstance(not_applicable, list) and any(isinstance(entry, dict) and entry.get("id") == dim_id for entry in not_applicable@@ -1513,13 +1521,15 @@ def _score_transition_values(score: object,*,skill_never_activated: bool = False,+ baseline_not_applicable: bool = False,baseline_only: bool = False,) -> str:"""Render ``baseline → score (uplift)`` so the visible arithmetic is exact.Scores are fractions in [0, 1]. A missing, non-finite, or out-of-range score,or a recorded baseline that is not a valid score, is reported as unavailable- rather than clamped or published.+ rather than clamped or published. ``baseline_not_applicable`` marks a+ dimension the baseline ran but could not score because it has no skill."""parsed_score = _number(score)parsed_baseline = _number(baseline)@@ -1542,6 +1552,8 @@ def _score_transition_values(if skill_never_activated:return f"{skill_label} — INCONCLUSIVE: skill never activated; uplift unavailable"if parsed_baseline is None:+ if baseline_not_applicable:+ return f"{skill_label} — baseline N/A (no skill in the baseline arm)"return f"{skill_label} — baseline not run; uplift unavailable"# Subtract the displayed values, not the raw scores, so a rounding carry can# never publish an uplift that disagrees with the two numbers beside it.diff --git a/tests/golden/test_benchmark_card.py b/tests/golden/test_benchmark_card.pyindex 8675a0c3..62b1e8b6 100644--- a/tests/golden/test_benchmark_card.py+++ b/tests/golden/test_benchmark_card.py@@ -5,6 +5,7 @@from __future__ import annotations+import jsonimport unicodedatafrom copy import deepcopyfrom datetime import UTC, datetime, timedelta, timezone@@ -13,11 +14,16 @@ from pathlib import Pathimport pytestfrom scripts.ci import check_public_benchmarks as benchmark_gate+from skillevaluator.evaluation.tier3_report import _validation_result_from_payload, build_agent_eval_payloadfrom skillevaluator.models.result import Finding, Severity, ValidationResultfrom skillevaluator.publication_evidence import stamp_publication_evidencefrom skillevaluator.reporting import BenchmarkReporter+from skillevaluator.reporting.base import agent_eval_dimension_verdictfrom skillevaluator.reporting.benchmark import _publication_safe_inline, _verdict_calloutfrom skillevaluator.reporting.summary_copy import SKILL_LIFT_DEFINITION+from skillevaluator.tier3.harbor.collector import collect_harbor_results+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_SCORE_POLICY, overall_score+from skillevaluator.tier3.harbor.report_data import load_agent_dataPASS_GOLDEN = Path(__file__).resolve().parent / "benchmark_pass" / "BENCHMARK.md"FAIL_GOLDEN = Path(__file__).resolve().parent / "benchmark_fail" / "BENCHMARK.md"@@ -1134,6 +1140,148 @@ def test_benchmark_no_baseline_never_fabricates_uplift() -> None:assert "Publication Recommendation" not in rendered+# The baseline has no skill to discover or route to, so the verifier records+# skill_execution and skill_efficiency as not applicable in that arm.+_DEFAULT_RUN_SCORES = {+ "with": {+ "security": 1.0,+ "skill_execution": 1.0,+ "skill_efficiency": 0.9,+ "accuracy": 0.9,+ "goal_accuracy": 1.0,+ "behavior_check": 0.8,+ },+ "without": {+ "security": 1.0,+ "skill_execution": None,+ "skill_efficiency": None,+ "accuracy": 0.5,+ "goal_accuracy": 0.4,+ "behavior_check": 0.5,+ },+}+++def _collected_default_run(tmp_path: Path) -> ValidationResult:+ """Collect a default run with its baseline from Harbor verifier output and build its Tier 3 result."""+ cases = ["case-1", "case-2"]+ for arm, scores in _DEFAULT_RUN_SCORES.items():+ job = tmp_path / "jobs" / f"demo-skill-codex-{arm}"+ trials = [f"{case}__attempt1" for case in cases]+ for case, trial in zip(cases, trials, strict=True):+ row = {+ **scores,+ "entry_id": case,+ "metric_set": DEFAULT_METRIC_SET,+ "score_policy": DEFAULT_SCORE_POLICY,+ "details": {+ metric: {"score": None, "status": "not_applicable", "reason": "No skill under test"}+ for metric, value in scores.items()+ if value is None+ },+ }+ verifier = job / trial / "verifier"+ verifier.mkdir(parents=True)+ numeric = {metric: value for metric, value in scores.items() if value is not None}+ (verifier / "reward.json").write_text(json.dumps({**numeric, "overall": overall_score(row)}))+ (verifier / "skill_evaluator_reward.json").write_text(json.dumps(row))+ idle = ("n_errored_trials", "n_running_trials", "n_pending_trials", "n_cancelled_trials", "n_retries")+ stats = {+ "n_completed_trials": len(trials),+ **dict.fromkeys(idle, 0),+ "evals": {"codex": {"n_trials": len(trials), "n_errors": 0, "reward_stats": {"reward": {"1.0": trials}}}},+ }+ (job / "result.json").write_text(json.dumps({"n_total_trials": len(trials), "stats": stats}))+ collect_harbor_results(+ "demo-skill",+ ["codex"],+ tmp_path / "results",+ tmp_path / "jobs",+ expected_cases=len(cases),+ expected_case_ids=cases,+ expected_trials=len(cases),+ )+ payload = build_agent_eval_payload(+ "demo-skill",+ load_agent_data(tmp_path / "results"),+ dataset=[{"id": case, "expected_skill": "demo-skill"} for case in cases],+ attempt_policy={"max_attempts": 1, "pass_threshold": 0.5, "stop_on_pass": False},+ run_config={"agents": {"codex": {"model": "gpt-codex"}}},+ env_mode="docker",+ evaluated_at="2026-07-24T12:30:00+00:00",+ evaluator_version="0.8.2",+ run_id=_TIER3_RUN_ID,+ publication_target=dict(_PUBLICATION_TARGET),+ use_llm_judge=False,+ )+ result = _validation_result_from_payload(payload)+ assert result is not None+ return _bind_target(result)+++def test_benchmark_baseline_without_the_skill_can_pass(tmp_path: Path) -> None:+ tier3 = _collected_default_run(tmp_path)+ agent = tier3.metadata["agent_eval"]["agents"]["codex"]+ # Discoverability and Efficiency have no baseline, so the lift compares the+ # other three dimensions: 0.9333 - 0.65, not 0.94 - 0.65.+ assert agent["not_applicable_metrics"]["without_skill"] == ["skill_execution", "skill_efficiency"]+ assert [row["id"] for row in agent["dimensions"] if row["baseline"] is None] == ["discoverability", "efficiency"]+ assert (agent["with_skill"], agent["baseline"], agent["lift"]) == (0.94, 0.65, 0.2833)++ rendered = BenchmarkReporter(include_timestamp=True).render_all([*_deterministic_results(), tier3])+ benchmark = tmp_path / "BENCHMARK.md"+ benchmark.write_text(rendered, encoding="utf-8")+ _files, offenders = benchmark_gate.find_offenders([benchmark])++ assert "**Overall verdict: PASS" in rendered+ assert "| Tier 3 | Live agent evaluation | **PASS** |" in rendered+ # The Overall row shows the two scores the lift subtracts, 0.9333 and 0.65.+ assert "| Overall | 65.0% → 93.3% (+28.3 points) |" in rendered+ assert "Overall for codex compares only the dimensions both arms scored" in rendered+ # The baseline ran, it just has no skill to discover or route to.+ assert "| Discoverability | 100.0% — baseline N/A (no skill in the baseline arm) |" in rendered+ assert "| Efficiency | 90.0% — baseline N/A (no skill in the baseline arm) |" in rendered+ assert "baseline not run" not in rendered+ assert offenders == []+++ "changes",+ [+ # Without the applicability record, a missing baseline is missing evidence.+ {"not_applicable_metrics": {"without_skill": []}},+ # The baseline scored accuracy, so Correctness needs its baseline.+ {"correctness": {"baseline": None, "lift": None}},+ # A dimension the baseline could not score has no lift either.+ {"efficiency": {"lift": 0.0}},+ # A five-dimension score minus a three-dimension baseline.+ {"agent": {"lift": 0.29}},+ # accuracy is not applicable when a case has no reference, which holds for+ # both arms. A baseline that alone records it is missing its Correctness+ # score, even when its baseline and lift add up over Security and Effectiveness.+ {+ "not_applicable_metrics": {"without_skill": ["skill_execution", "skill_efficiency", "accuracy"]},+ "correctness": {"baseline": None, "lift": None},+ "agent": {"baseline": 0.725, "lift": 0.225},+ },+ ],+)+def test_benchmark_baseline_without_the_skill_still_needs_matching_evidence(+ tmp_path: Path,+ changes: dict[str, dict[str, object]],+) -> None:+ payload = _collected_default_run(tmp_path).metadata["agent_eval"]+ agent = payload["agents"]["codex"]+ assert agent_eval_dimension_verdict(payload) == "pass"++ containers = {"agent": agent, "not_applicable_metrics": agent["not_applicable_metrics"]}+ containers.update((row["id"], row) for row in agent["dimensions"])+ for target, change in changes.items():+ containers[target].update(change)++ assert agent_eval_dimension_verdict(payload) is None++def test_benchmark_zero_and_negative_lift_use_percentage_points() -> None:from skillevaluator.reporting.benchmark import _score_transition, _score_transition_values