Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): retain partial arm evidence in collector JSON (#670)
Christopher Kevincc3563e5f958
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 355d4839..5cc85b53 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -414,6 +414,11 @@ All notable changes to SkillEvaluator are documented in this file.- Keep Steps per Eval Case chart labels readable for long case IDs while preserving full IDs in hover tooltips.- Negative controls with complete, confirmed skill non-use report wrong answers as task outcomes instead of recommending skill edits; truncated or recovered attempt evidence cannot establish complete non-use.- Repeated negative-case task outcomes share one advice item, retain task-failure wording and avoid describing passing attempts as lost, keeping distinct skill fixes visible.+- Partial Tier 3 evidence is retained only for valid reward identities and attempt+ coverage. Corrupt rewards or duplicate attempts still suppress arm scores;+ incomplete cases are reported as unresolved instead of failed.++- Partially executed evaluation arms retain scored attempts and per-arm scores without treating missing attempts as failures or enabling aggregate comparisons.- Failed Tier 3 arms retain scored and errored trial activation diagnostics without restoring aggregate quality or lift, or double-counting token usage.- Budget-limited Tier 3 coverage checks retain admitted attempt ordinals, avoiding false missing/excess errors when a denied slot leaves a gap.- HTML reports retain budget context for fully budget-denied runs withoutdiff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.pyindex 45c85418..67926370 100644--- a/src/skillevaluator/tier3/harbor/collector.py+++ b/src/skillevaluator/tier3/harbor/collector.py@@ -4469,6 +4469,8 @@ def _mark_reward_collection_failure(reward: dict[str, Any], reason: str) -> Nonedef _partition_scoreable_rewards(rewards: list[dict[str, Any]],+ *,+ integrity_errors: list[str] | None = None,) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:"""Separate complete finite rewards from diagnostic-only reward artifacts."""scoreable: list[dict[str, Any]] = []@@ -4485,6 +4487,13 @@ def _partition_scoreable_rewards(if not evaluation_failed and overall_score(reward) is not None:scoreable.append(reward)continue+ # Judge/service failures are missing evaluations, while collector-invalid+ # identities, structures and numbers make the reward evidence untrustworthy.+ evaluation_errors = reward.get("evaluation_errors")+ if integrity_errors is not None and (+ not evaluation_failed or (isinstance(evaluation_errors, dict) and "collector" in evaluation_errors)+ ):+ integrity_errors.append(_unscoreable_reward_reason(reward))raw_trial = str(reward.get("_trial_name") or reward.get("_trial_root_name") or "unknown trial")if raw_trial in failed_trials:continue@@ -4499,6 +4508,8 @@ def _partition_scoreable_rewards(custom_names = {name for reward in scoreable for name in extract_custom_metrics(reward)}if len(custom_names) > MAX_CUSTOM_METRICS:+ if integrity_errors is not None:+ integrity_errors.append(UNSAFE_CUSTOM_METRIC_UNION_REASON)for reward in scoreable:_mark_reward_collection_failure(reward, UNSAFE_CUSTOM_METRIC_UNION_REASON)raw_trial = str(reward.get("_trial_name") or reward.get("_trial_root_name") or "unknown trial")@@ -5084,6 +5095,34 @@ def _public_pass_summary(summary: dict[str, Any]) -> dict[str, Any]:return {key: value for key, value in summary.items() if not key.startswith("_")}+def _partial_pass_summary(summary: dict[str, Any]) -> dict[str, Any]:+ return {+ **summary,+ "failed_cases": summary["_resolved_failed_cases"],+ "unresolved_cases": summary["total_cases"] - summary["passed_cases"] - summary["_resolved_failed_cases"],+ "rate": None,+ "rate_interval": None,+ }+++def _unscored_attempts_before_first_pass(+ first_pass: int,+ scored_ordinals: set[int],+ recovered_ordinals: set[int],+ admitted_ordinals: set[int] | None,+) -> int:+ """Count attempt slots up to the first pass that produced no scored result.++ Stop-on-pass skips only the attempts after the first pass. A slot at or+ before it ran (or was owed) and has no score, so it failed rather than+ being skipped. Slots the budget never admitted are not owed.+ """+ covered = scored_ordinals | recovered_ordinals+ if admitted_ordinals is not None:+ return sum(1 for ordinal in admitted_ordinals if ordinal <= first_pass and ordinal not in covered)+ return first_pass - sum(1 for ordinal in covered if 1 <= ordinal <= first_pass)++def _pass_summary(rewards: list[dict[str, Any]],*,@@ -5093,6 +5132,7 @@ def _pass_summary(expected_cases: int | None,expected_case_ids: list[str] | None = None,expected_attempts_by_case: Mapping[str, int] | None = None,+ expected_attempt_ordinals_by_case: Mapping[str, set[int]] | None = None,recovered_attempts: list[dict[str, Any]] | None = None,scorer: Callable[[dict[str, Any]], float | None] | None = None,shared_metrics_by_case: Mapping[str, set[str]] | None = None,@@ -5130,6 +5170,7 @@ def _pass_summary(pairing_cases: dict[str, dict[str, Any]] = {}passed_cases = 0scored_cases = 0+ resolved_failed_cases = 0required_checks_passed = 0wrong_answer_cases = 0security_failed_cases = 0@@ -5151,8 +5192,10 @@ def _pass_summary(attempt_verdicts = []best_score: float | None = Nonefirst_pass_attempt: int | None = None+ scored_ordinals: set[int] = set()for idx, (reward, overall) in enumerate(sorted(attempts, key=lambda item: _attempt_sort_key(item[0])), start=1):idx = _attempt_ordinal(reward) or idx+ scored_ordinals.add(idx)score = round(overall, 4)applicable_metrics = shared_metrics_by_case.get(entry_id) if shared_metrics_by_case is not None else Noneverdict = attempt_verdict(@@ -5210,8 +5253,26 @@ def _pass_summary(expected_attempts_by_case.get(entry_id, n_attempts) if expected_attempts_by_case is not None else n_attempts)unscored = max(0, expected_attempt_count - len(attempts) - recovered_count)- skipped = unscored if stop_on_pass and case_passed else 0- missing = 0 if skipped else unscored+ missing = unscored+ if stop_on_pass and first_pass_attempt is not None:+ # Only attempts after the first pass were skipped on purpose. An unscored+ # attempt before it failed, and must not be hidden behind the pass.+ missing = min(+ unscored,+ _unscored_attempts_before_first_pass(+ first_pass_attempt,+ scored_ordinals,+ {+ item["attempt"]+ for item in recovered_attempts or []+ if item["case_id"] == entry_id and type(item["attempt"]) is int+ },+ (expected_attempt_ordinals_by_case or {}).get(entry_id),+ ),+ )+ skipped = unscored - missing+ if is_expected_case and attempts and not case_passed and missing == 0:+ resolved_failed_cases += 1# Every attempt, also past the published caps: pass^k counts all k of them.pairing_cases[entry_id] = {@@ -5282,6 +5343,7 @@ def _pass_summary("case_details_limit": PUBLISHED_CASE_DETAILS_MAX,"cases": cases,"_pairing_cases": pairing_cases,+ "_resolved_failed_cases": resolved_failed_cases,}@@ -6856,6 +6918,7 @@ def _condition_execution_summary(stop_on_pass: bool = False,pass_threshold: float = 0.50,attempt_slots: dict[str, dict[str, Any]] | None = None,+ integrity_errors: list[str] | None = None,expected_attempts_by_case: Mapping[str, int] | None = None,expected_attempt_ordinals_by_case: Mapping[str, set[int]] | None = None,) -> dict[str, Any]:@@ -6868,7 +6931,8 @@ def _condition_execution_summary(When ``attempt_slots`` is a dict, it receives ``{"required", "missing","missing_ordinals"}`` for each expected case; ``missing_ordinals`` liststhe absent attempt ordinals when the case's attempts carry labels, and is- None when only a count is known.+ None when only a count is known. ``integrity_errors`` receives structural+ reward/identity errors separately from missing attempts and runtime failures."""expected_ids = _validated_expected_case_ids(expected_case_ids)expected_count = len(expected_ids) if expected_ids else int(expected_cases or 0)@@ -6926,6 +6990,12 @@ def _condition_execution_summary("Unscoreable reward details were truncated "f"(showing {len(public_reward_failures)} of {len(reward_failures or [])})")++ def integrity_error(message: str) -> None:+ errors.append(message)+ if integrity_errors is not None:+ integrity_errors.append(message)+expected_set = set(expected_ids)logical_passed: dict[str, bool] = {}for reward in _logical_attempt_rewards(rewards):@@ -6941,14 +7011,14 @@ def _condition_execution_summary(case_id = _entry_id(reward, expected_set or None)step_name = str(reward.get("_step_name") or "").strip()if not root:- errors.append("A scored reward is missing its Harbor trial root name")+ integrity_error("A scored reward is missing its Harbor trial root name")continueif not case_id or case_id == "unknown":- errors.append(f"Scored trial {published_root!r} has no case identifier")+ integrity_error(f"Scored trial {published_root!r} has no case identifier")continuescore = overall_score(reward)if score is None:- errors.append(f"Scored trial {published_root!r} has incomplete or non-finite reward metrics")+ integrity_error(f"Scored trial {published_root!r} has incomplete or non-finite reward metrics")continueexisting = roots.get(root)if existing is None:@@ -6961,9 +7031,9 @@ def _condition_execution_summary(}continueif existing["case_id"] != case_id:- errors.append(f"Harbor trial {published_root!r} maps to multiple cases")+ integrity_error(f"Harbor trial {published_root!r} maps to multiple cases")elif not step_name or step_name in existing["steps"]:- errors.append(f"Harbor trial {published_root!r} has duplicate reward rows")+ integrity_error(f"Harbor trial {published_root!r} has duplicate reward rows")else:existing["steps"].add(step_name)@@ -6978,9 +7048,9 @@ def _condition_execution_summary(explicit = [item["attempt_ordinal"] for item in attempts if item["attempt_ordinal"] is not None]all_explicit = len(explicit) == len(attempts) and bool(attempts)if explicit and len(explicit) != len(attempts):- errors.append(f"Scored case {case_id!r} mixes explicit and implicit attempt labels")+ integrity_error(f"Scored case {case_id!r} mixes explicit and implicit attempt labels")if len(explicit + recovered_ordinals) != len(set(explicit + recovered_ordinals)):- errors.append(f"Scored case {case_id!r} has duplicate attempt ordinals")+ integrity_error(f"Scored case {case_id!r} has duplicate attempt ordinals")required = (expected_attempts_by_case.get(case_id, n_attempts) if expected_attempts_by_case is not None else n_attempts@@ -6994,7 +7064,7 @@ def _condition_execution_summary(observed_index, passed_attempt = first_passrequired = int(passed_attempt["attempt_ordinal"] or observed_index)if required > n_attempts:- errors.append(f"Scored case {case_id!r} has an attempt ordinal above configured maximum {n_attempts}")+ integrity_error(f"Scored case {case_id!r} has an attempt ordinal above configured maximum {n_attempts}")expected_ordinals = set(range(1, required + 1))if expected_attempt_ordinals_by_case is not None and case_id in expected_attempt_ordinals_by_case:@@ -7048,17 +7118,18 @@ def _condition_execution_summary(if expected_ids:unexpected = sorted(case_id for case_id in by_case if case_id not in expected_set)if unexpected:- errors.append(_sampled_case_id_diagnostic("Unexpected scored cases", unexpected))+ integrity_error(_sampled_case_id_diagnostic("Unexpected scored cases", unexpected))else:if expected_count and len(by_case) != expected_count:- errors.append(f"Scored case coverage is {len(by_case)}/{expected_count}")+ report_case_coverage = integrity_error if len(by_case) > expected_count else errors.append+ report_case_coverage(f"Scored case coverage is {len(by_case)}/{expected_count}")if stop_on_pass and expected_count > len(by_case):expected_attempts += (expected_count - len(by_case)) * n_attemptsif missing:errors.append(_sampled_case_id_diagnostic("Missing scored attempts for cases", sorted(missing)))if excess:- errors.append(_sampled_case_id_diagnostic("Excess scored attempts for cases", sorted(excess)))+ integrity_error(_sampled_case_id_diagnostic("Excess scored attempts for cases", sorted(excess)))scored_attempts = len(roots)physical_expected_attempts = expected_attempts@@ -7701,6 +7772,7 @@ def collect_harbor_results(with_recovered: list[dict[str, Any]] = []with_job_recovered = Falsewith_execution: dict[str, Any] = {}+ with_integrity_errors: list[str] = []with_overall_score: float | None = Nonewith_policy = fallback_policywith_policy_ambiguous = False@@ -7735,7 +7807,9 @@ def collect_harbor_results(if with_job_ok or preserve_partialelse [])- with_rewards, invalid_score_failures = _partition_scoreable_rewards(with_collected_rewards)+ with_rewards, invalid_score_failures = _partition_scoreable_rewards(+ with_collected_rewards, integrity_errors=with_integrity_errors+ )_observe_private_trials(agent, "with_skill", with_job_dir, with_collected_rewards, with_rewards, case_id_by_task_selector)@@ -7765,10 +7839,12 @@ def collect_harbor_results(expected_cases=with_expected_cases,expected_case_ids=with_expected_case_ids,expected_attempts_by_case=with_expected_attempts,+ expected_attempt_ordinals_by_case=with_expected_ordinals,recovered_attempts=with_recovered,)with_execution = _condition_execution_summary(with_rewards,+ integrity_errors=with_integrity_errors,expected_case_ids=with_expected_case_ids,expected_cases=with_expected_cases,n_attempts=n_attempts,@@ -7783,7 +7859,14 @@ def collect_harbor_results()if with_policy == "mixed-score-policies":_fail_execution_with_error(with_execution, "Conflicting score policies within the with-skill condition")- if with_execution["execution_status"] != "succeeded":+ if (+ with_execution["execution_status"] != "succeeded"+ and 0 < with_execution["scored_attempts"] < with_execution["expected_attempts"]+ and with_policy != "mixed-score-policies"+ and not with_integrity_errors+ ):+ with_pass = _partial_pass_summary(with_pass)+ elif with_execution["execution_status"] != "succeeded":with_scores = {}with_custom_scores = {}with_pass = {}@@ -7923,6 +8006,7 @@ def collect_harbor_results(without_recovered: list[dict[str, Any]] = []without_job_recovered = Falsewithout_execution: dict[str, Any] = {}+ without_integrity_errors: list[str] = []without_overall_score: float | None = Nonewithout_policy = fallback_policywithout_policy_ambiguous = False@@ -7944,7 +8028,9 @@ def collect_harbor_results(if reused_baseline_dir is not None:without_job_ok, without_job_failure = True, ""without_collected_rewards = load_baseline_rows(reused_baseline_dir, agent)- without_rewards, invalid_score_failures = _partition_scoreable_rewards(without_collected_rewards)+ without_rewards, invalid_score_failures = _partition_scoreable_rewards(+ without_collected_rewards, integrity_errors=without_integrity_errors+ )else:without_job_ok, without_job_failure = validate_harbor_job_result(without_job_dir / "result.json",@@ -7972,7 +8058,9 @@ def collect_harbor_results(if without_job_ok or preserve_partialelse [])- without_rewards, invalid_score_failures = _partition_scoreable_rewards(without_collected_rewards)+ without_rewards, invalid_score_failures = _partition_scoreable_rewards(+ without_collected_rewards, integrity_errors=without_integrity_errors+ )_observe_private_trials(agent,"without_skill",@@ -8015,10 +8103,12 @@ def collect_harbor_results(expected_cases=without_expected_cases,expected_case_ids=without_expected_case_ids,expected_attempts_by_case=without_expected_attempts,+ expected_attempt_ordinals_by_case=without_expected_ordinals,recovered_attempts=without_recovered,)without_execution = _condition_execution_summary(without_rewards,+ integrity_errors=without_integrity_errors,expected_case_ids=without_expected_case_ids,expected_cases=without_expected_cases,n_attempts=n_attempts,@@ -8037,7 +8127,14 @@ def collect_harbor_results(without_execution,"Conflicting score policies within the without-skill condition",)- if without_execution["execution_status"] != "succeeded":+ if (+ without_execution["execution_status"] != "succeeded"+ and 0 < without_execution["scored_attempts"] < without_execution["expected_attempts"]+ and without_policy != "mixed-score-policies"+ and not without_integrity_errors+ ):+ without_pass = _partial_pass_summary(without_pass)+ elif without_execution["execution_status"] != "succeeded":without_scores = {}without_custom_scores = {}without_pass = {}@@ -8605,9 +8702,15 @@ def _build_comparison(agents_data: dict[str, Any]) -> dict[str, Any]:comparison["metrics"][metric] = {}for agent, data in agents_data.items():succeeded = data.get("execution_status") == "succeeded"- with_skill = data.get("with_skill", {}).get(metric) if succeeded else None- without_skill = data.get("without_skill", {}).get(metric) if succeeded else None- lift = data.get("lift", {}).get(metric, {}).get("delta") if succeeded else None+ with_succeeded = (+ succeeded and data.get("conditions", {}).get("with_skill", {}).get("execution_status") == "succeeded"+ )+ without_succeeded = (+ succeeded and data.get("conditions", {}).get("without_skill", {}).get("execution_status") == "succeeded"+ )+ with_skill = data.get("with_skill", {}).get(metric) if with_succeeded else None+ without_skill = data.get("without_skill", {}).get(metric) if without_succeeded else None+ lift = data.get("lift", {}).get(metric, {}).get("delta") if with_succeeded and without_succeeded else Nonecomparison["metrics"][metric][agent] = {"with_skill": with_skillif isinstance(with_skill, int | float) and not isinstance(with_skill, bool)diff --git a/tests/test_harbor_collector_runtime_failures.py b/tests/test_harbor_collector_runtime_failures.pyindex f904af21..d2d32f69 100644--- a/tests/test_harbor_collector_runtime_failures.py+++ b/tests/test_harbor_collector_runtime_failures.py@@ -13,7 +13,12 @@ from uuid import UUIDimport pytest-from skillevaluator.evaluation.tier3_report import render_agent_eval_html_report+from skillevaluator.evaluation.tier3_report import build_agent_eval_payload, render_agent_eval_html_report+from skillevaluator.models import ValidationResult+from skillevaluator.reporting.base import assess_publication+from skillevaluator.reporting.summary_copy import pass_at_k_summary+from skillevaluator.reporting.tier3_scope import completed_agent_result+from skillevaluator.tier3.case_summary import verdictfrom skillevaluator.tier3.harbor import collector as collector_modulefrom skillevaluator.tier3.harbor import report_datafrom skillevaluator.tier3.harbor.collector import (@@ -22,6 +27,7 @@ from skillevaluator.tier3.harbor.collector import ()from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICSfrom skillevaluator.tier3.result_display import render_result+from skillevaluator.tier3.results_location import _legacy_pass_at_k_is_complete_HARBOR_022_AGENT_RUNTIME_EXCEPTION_TYPES = ("AgentAuthenticationError",@@ -874,6 +880,117 @@ def test_partially_errored_job_preserves_only_completed_trial_coverage(tmp_path:assert any("Scored attempt coverage is 3/4" in error for error in results["execution_errors"])[email protected]("partial_variant", ["with", "without"])[email protected]("score", [0.0, 1.0])+def test_partial_arm_retains_scored_evidence_without_enabling_quality(+ tmp_path: Path, partial_variant: str, score: float+) -> None:+ jobs_dir = tmp_path / "jobs"+ for variant in ("with", "without"):+ for case_id in CASES:+ value = score if variant == partial_variant and case_id == "case-a" else 1.0+ _write_reward(jobs_dir, variant=variant, case_id=case_id, attempt=1, score=value)+ trial = jobs_dir / f"demo-opencode-{variant}" / f"{case_id}_attempt001"+ path = trial / "verifier/reward.json"+ reward = json.loads(path.read_text(encoding="utf-8"))+ reward["custom_metrics"] = {"native_quality": {"score": value}}+ path.write_text(json.dumps(reward), encoding="utf-8")+ (trial / "agent").mkdir()+ prompt, completion = (20, 3) if variant == partial_variant and case_id == "case-b" else (10, 2)+ (trial / "agent/trajectory.json").write_text(+ json.dumps(+ {+ "schema_version": "ATIF-v1.7",+ "agent": {"name": "opencode", "version": "test"},+ "steps": [{"step_id": 1, "source": "agent", "message": "Done"}],+ "final_metrics": {"total_prompt_tokens": prompt, "total_completion_tokens": completion},+ }+ ),+ encoding="utf-8",+ )+ _write_variant_job_results(jobs_dir)+ partial_job = jobs_dir / f"demo-opencode-{partial_variant}"+ failure = {"exception_info": {"exception_type": "AgentTimeoutError", "exception_message": "Agent timed out"}}+ (partial_job / "case-b_attempt001/result.json").write_text(json.dumps(failure), encoding="utf-8")+ job_result = json.loads((partial_job / "result.json").read_text(encoding="utf-8"))+ job_result["stats"]["n_errors"] = 1+ (partial_job / "result.json").write_text(json.dumps(job_result), encoding="utf-8")++ result = _collect(tmp_path, n_attempts=1, stop_on_pass=False)++ agent = result["agents"]["opencode"]+ arm = f"{partial_variant}_skill"+ complete_arm = "without_skill" if partial_variant == "with" else "with_skill"+ assert result["execution_status"] == agent["execution_status"] == "failed"+ assert (result["scored_attempts"], result["expected_attempts"]) == (3, 4)+ condition = agent["conditions"][arm]+ assert condition["execution_status"] == "failed"+ assert (condition["scored_attempts"], condition["expected_attempts"]) == (1, 2)+ assert agent[arm] and set(agent[arm].values()) == {score}+ assert agent[f"custom_{arm}"] == {"native_quality": score}+ assert agent[f"overall_{arm}"] is None+ assert agent["lift"] == agent["custom_lift"] == agent["pass_at_k"]["lift"] == {}+ partial = agent["pass_at_k"][arm]+ assert partial["rate"] is None and partial["rate_interval"] is None+ assert partial["passed_cases"] == int(score == 1.0)+ assert partial["failed_cases"] == int(score == 0.0)+ assert partial["attempts_used"] == 1 and partial["max_attempts_possible"] == 2+ [attempt] = partial["cases"]["case-a"]["attempts"]+ assert attempt["score"] == score and attempt["passed"] is (score == 1.0)+ missing = partial["cases"]["case-b"]+ assert missing["attempts"] == [] and missing["best_score"] is None+ assert missing["attempts_missing"] == 1 and missing["attempts_used"] == 0+ summary = json.loads(+ (tmp_path / "results/opencode" / f"{partial_variant}-skill/summary.json").read_text(encoding="utf-8")+ )+ assert summary["pass_at_k"] == partial+ assert summary["scores"] == agent[arm] and summary["custom_scores"] == agent[f"custom_{arm}"]+ assert not _legacy_pass_at_k_is_complete(partial, num_trials=1, require_scored_attempt=True)+ assert "N/A" in pass_at_k_summary(agent["pass_at_k"])+ assert verdict(result["agents"])[0] == "inconclusive"+ terminal = render_result(result)+ check_line = next(line for line in terminal.splitlines() if "REQUIRED CHECKS PASSED" in line)+ assert check_line.strip(" │").endswith("N/A")+ if partial_variant == "with":+ assert "QUALITY VERDICT: FAIL" not in terminal++ complete = agent["pass_at_k"][complete_arm]+ assert complete["passed_cases"] == complete["attempts_used"] == 2+ assert complete["failed_cases"] == 0 and complete["rate"] == 1.0+ assert complete["rate_interval"]["confidence_level"] == 0.95+ assert agent["conditions"][complete_arm]["execution_status"] == "succeeded"+ assert set(agent[complete_arm].values()) == {1.0}+ assert agent[f"custom_{complete_arm}"] == {"native_quality": 1.0}++ loaded = report_data.load_agent_data(tmp_path / "results")+ assert loaded["opencode"][arm] == loaded["opencode"][f"custom_{arm}"] == {}+ assert loaded["opencode"][f"pass_{arm}"] == {"attempts_used": 1, "max_attempts_possible": 2}+ payload = build_agent_eval_payload("demo", loaded, use_llm_judge=False)+ assert payload["execution_status"] == "failed" and payload["overall_score"] is None+ assert completed_agent_result(payload) is None+ assert payload["agents"]["opencode"]["lift"] is None+ trial_key = "trials" if partial_variant == "with" else "trials_baseline"+ trials = payload["agents"]["opencode"][trial_key]+ assert len(trials) == 2 and all(row["diagnostic_only"] for row in trials)+ assert sum(row.get("scored") is False for row in trials) == 1+ usage = payload["token_usage"]["agents"]["opencode"]["with_skill" if partial_variant == "with" else "baseline"]+ assert usage["coverage"]["observed_attempts"] == usage["coverage"]["measured_attempts"] == 2+ assert usage["coverage"]["unscored_attempts"] == 1+ assert (+ usage["totals"]["prompt_tokens"],+ usage["totals"]["completion_tokens"],+ usage["totals"]["total_tokens"],+ ) == (+ 30,+ 5,+ 35,+ )+ validation = ValidationResult(validator_name="AGENT_EVAL")+ validation.metadata["agent_eval"] = payload+ assessment = assess_publication([validation])+ assert assessment.status != "pass" and not assessment.tier3.evidence_complete++def test_partial_rewards_stay_suppressed_when_not_every_job_error_maps_to_a_trial(tmp_path: Path) -> None:jobs_dir = tmp_path / "jobs"job_dir = jobs_dir / "demo-opencode-with"@@ -2219,6 +2336,88 @@ def test_stop_on_pass_rejects_failed_attempt_before_pass(tmp_path: Path) -> Noneassert result["agents"]["opencode"]["trial_failures"]["with_skill"]+def _crash_attempt(jobs_dir: Path, *, variant: str, case_id: str, attempt: int) -> None:+ """Leave one attempt as an unrecovered agent crash instead of a scored trial."""+ _write_reward(jobs_dir, variant=variant, case_id=case_id, attempt=attempt, score=1.0)+ failure = {"exception_info": {"exception_type": "TaskFailure", "exception_message": "attempt crashed"}}+ (jobs_dir / f"demo-opencode-{variant}" / f"{case_id}_attempt{attempt:03d}" / "result.json").write_text(+ json.dumps(failure), encoding="utf-8"+ )++[email protected]("crashed_variant", ["with", "without"])+def test_stop_on_pass_crash_before_pass_is_missing_not_skipped(tmp_path: Path, crashed_variant: str) -> None:+ jobs_dir = tmp_path / "jobs"+ for variant in ("with", "without"):+ if variant == crashed_variant:+ _crash_attempt(jobs_dir, variant=variant, case_id="case-a", attempt=1)+ _write_reward(jobs_dir, variant=variant, case_id="case-a", attempt=2, score=1.0)+ else:+ _write_reward(jobs_dir, variant=variant, case_id="case-a", attempt=1, score=1.0)+ _write_variant_job_results(jobs_dir)++ result = _collect(tmp_path, n_attempts=3, stop_on_pass=True, expected_cases=1, expected_case_ids=["case-a"])++ agent = result["agents"]["opencode"]+ arm = f"{crashed_variant}_skill"+ other_arm = "without_skill" if crashed_variant == "with" else "with_skill"+ assert agent["conditions"][arm]["execution_status"] == "failed"+ assert (agent["conditions"][arm]["scored_attempts"], agent["conditions"][arm]["expected_attempts"]) == (1, 2)+ partial = agent["pass_at_k"][arm]+ assert partial["rate"] is None+ case = partial["cases"]["case-a"]+ assert case["passed"] is True and case["first_pass_attempt"] == 2+ assert case["attempts_used"] == 1+ # Attempt 1 crashed (a failed attempt, not a skipped one); only attempt 3 was never needed.+ assert case["attempts_missing"] == 1+ assert case["attempts_skipped"] == 1+ # A pass that needed no earlier retry still skips the unused attempts on the healthy arm.+ healthy = agent["pass_at_k"][other_arm]["cases"]["case-a"]+ assert (healthy["attempts_missing"], healthy["attempts_skipped"]) == (0, 2)+++ ("outcomes", "denied", "missing", "skipped"),+ [+ pytest.param({1: "crash", 3: "pass"}, [2], 1, 0, id="crash-then-denied-slot-then-pass"),+ pytest.param({2: "crash", 3: "pass"}, [1], 1, 0, id="denied-first-slot-then-crash-then-pass"),+ pytest.param({1: "fail", 3: "pass"}, [2], 0, 0, id="scored-fail-then-denied-slot-then-pass"),+ pytest.param({2: "pass"}, [1], 0, 1, id="denied-first-slot-then-pass"),+ pytest.param({1: "pass"}, [2], 0, 1, id="pass-then-denied-slot"),+ ],+)+def test_stop_on_pass_budget_denied_slots_are_neither_missing_nor_skipped(+ tmp_path: Path, outcomes: dict[int, str], denied: list[int], missing: int, skipped: int+) -> None:+ jobs_dir = tmp_path / "jobs"+ for attempt, outcome in outcomes.items():+ if outcome == "crash":+ _crash_attempt(jobs_dir, variant="with", case_id="case-a", attempt=attempt)+ else:+ _write_reward(+ jobs_dir, variant="with", case_id="case-a", attempt=attempt, score=1.0 if outcome == "pass" else 0.2+ )+ _write_variant_job_results(jobs_dir, variants=("with",))+ not_run = [+ {"agent": "opencode", "arm": "with", "case_id": "case-a", "attempt": attempt, "reason": "budget_exhausted"}+ for attempt in denied+ ]++ result = _collect(+ tmp_path,+ skip_baseline=True,+ n_attempts=3,+ stop_on_pass=True,+ expected_cases=1,+ expected_case_ids=["case-a"],+ budget_not_run=not_run,+ )++ case = result["agents"]["opencode"]["pass_at_k"]["with_skill"]["cases"]["case-a"]+ assert case["passed"] is True+ assert (case["attempts_missing"], case["attempts_skipped"]) == (missing, skipped)++def test_multistep_stop_on_pass_uses_authoritative_root_reward(tmp_path: Path) -> None:jobs_dir = tmp_path / "jobs"_write_reward(jobs_dir, variant="with", case_id="case-a", attempt=1, score=1.0, steps=("prepare",))@@ -2254,7 +2453,10 @@ def test_multistep_stop_on_pass_uses_authoritative_root_reward(tmp_path: Path) -assert result["execution_status"] == "failed"assert result["expected_attempts"] == 2assert result["scored_attempts"] == 1- assert agent["pass_at_k"]["with_skill"] == {}+ partial = agent["pass_at_k"]["with_skill"]+ assert partial["rate"] is None+ assert partial["attempts_used"] == 1+ assert partial["cases"]["case-a"]["attempts_missing"] == 1def test_duplicate_logical_attempt_ordinals_fail(tmp_path: Path) -> None:@@ -2735,3 +2937,83 @@ def test_trial_failure_names_docker_address_pool_exhaustion(tmp_path: Path) -> N[failure] = results["agents"]["opencode"]["trial_failures"]["with_skill"]assert failure["trial"] == "case-001__attempt"assert failure["reason"].startswith(f"RuntimeError: {_ADDRESS_POOL_NOTE_START} ")++[email protected]("variant", ["with", "without"])+ "corruption", ["duplicate-ordinal", "excess-ordinal", "unexpected-case", "nonfinite", "unsafe-identity"]+)+def test_partial_integrity_failure_clears_scores(tmp_path: Path, variant: str, corruption: str) -> None:+ jobs = tmp_path / "jobs"+ _write_reward(jobs, variant=variant, case_id="case-a", attempt=1, score=1.0)+ if corruption == "duplicate-ordinal":+ _write_reward(+ jobs, variant=variant, case_id="case-a", attempt=1, score=0.0, trial_name="duplicate-case-a_attempt001"+ )+ elif corruption == "excess-ordinal":+ _write_reward(jobs, variant=variant, case_id="case-a", attempt=3, score=0.0)+ elif corruption == "unexpected-case":+ _write_reward(jobs, variant=variant, case_id="case-extra", attempt=1, score=0.0)+ else:+ _write_reward(jobs, variant=variant, case_id="case-b", attempt=1, score=0.0)+ path = jobs / f"demo-opencode-{variant}/case-b_attempt001/verifier/reward.json"+ reward = json.loads(path.read_text())+ if corruption == "nonfinite":+ reward["accuracy"] = float("nan")+ reward["overall"] = float("nan")+ else:+ reward["entry_id"] = "bad\nidentity"+ path.write_text(json.dumps(reward))+ _write_variant_job_results(jobs, variants=(variant,))+ result = _collect(tmp_path, skip_baseline=variant == "with", skip_with_skill=variant == "without")+ agent = result["agents"]["opencode"]+ arm = f"{variant}_skill"+ assert agent["conditions"][arm]["execution_status"] == "failed"+ assert agent[arm] == {}+ assert agent["pass_at_k"][arm] == {}+ saved = json.loads((tmp_path / "results/opencode" / f"{variant}-skill/summary.json").read_text())+ assert saved["scores"] == {} and saved["pass_at_k"] == {}++[email protected]("variant", ["with", "without"])[email protected]("case_detail_cap", [1, 400])+def test_partial_failed_case_requires_all_attempts(+ tmp_path: Path, variant: str, case_detail_cap: int, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(collector_module, "PUBLISHED_CASE_DETAILS_MAX", case_detail_cap)+ jobs = tmp_path / "jobs"+ for case_id, scores in {"case-a": [0.0], "case-b": [0.0, 0.0], "case-c": [1.0]}.items():+ for attempt, score in enumerate(scores, start=1):+ _write_reward(jobs, variant=variant, case_id=case_id, attempt=attempt, score=score)+ _write_variant_job_results(jobs, variants=(variant,))+ result = _collect(+ tmp_path,+ skip_baseline=variant == "with",+ skip_with_skill=variant == "without",+ expected_cases=4,+ expected_case_ids=["case-a", "case-b", "case-c", "case-d"],+ )+ partial = result["agents"]["opencode"]["pass_at_k"][f"{variant}_skill"]+ assert partial["passed_cases"] == 1+ assert partial["failed_cases"] == 1+ assert partial["unresolved_cases"] == 2+ assert sum(partial[key] for key in ("passed_cases", "failed_cases", "unresolved_cases")) == partial["total_cases"]+ saved = json.loads((tmp_path / "results/opencode" / f"{variant}-skill/summary.json").read_text())+ assert saved["pass_at_k"] == partial++[email protected]("variant", ["with", "without"])+def test_partial_extra_case_count_without_expected_ids_clears_scores(tmp_path: Path, variant: str) -> None:+ for case_id in ["case-a", "case-b", "case-extra"]:+ _write_reward(tmp_path / "jobs", variant=variant, case_id=case_id, attempt=1, score=1.0)+ _write_variant_job_results(tmp_path / "jobs", variants=(variant,))+ result = _collect(+ tmp_path,+ skip_baseline=variant == "with",+ skip_with_skill=variant == "without",+ expected_cases=2,+ expected_case_ids=None,+ )+ agent = result["agents"]["opencode"]+ assert agent[f"{variant}_skill"] == {}+ assert agent["pass_at_k"][f"{variant}_skill"] == {}diff --git a/tests/test_issue55_collector_truth.py b/tests/test_issue55_collector_truth.pyindex d9799ccb..a216ccf0 100644--- a/tests/test_issue55_collector_truth.py+++ b/tests/test_issue55_collector_truth.py@@ -2664,10 +2664,14 @@ def test_mixed_failed_condition_suppresses_published_quality_and_paired_artifactassert agent["conditions"]["with_skill"]["execution_status"] == "failed"assert agent["conditions"]["with_skill"]["scored_attempts"] == 1assert agent["conditions"]["without_skill"]["execution_status"] == "succeeded"- assert agent["with_skill"] == {}+ assert agent["with_skill"] == dict.fromkeys(DEFAULT_METRICS, 0.9)assert agent["custom_with_skill"] == {}- assert agent["dimensions_with_skill"] == {}- assert agent["pass_at_k"]["with_skill"] == {}+ assert agent["dimensions_with_skill"]+ assert agent["overall_with_skill"] is None+ partial = agent["pass_at_k"]["with_skill"]+ assert partial["attempts_used"] == 1 and partial["rate"] is None+ assert partial["cases"]["case-001"]["attempts"][0]["score"] == 0.9+ assert partial["cases"]["case-002"]["attempts"] == []assert agent["without_skill"] == dict.fromkeys(DEFAULT_METRICS, 0.2)assert agent["dimensions_without_skill"]assert agent["pass_at_k"]["without_skill"]["rate"] == 0.0@@ -2678,10 +2682,10 @@ def test_mixed_failed_condition_suppresses_published_quality_and_paired_artifactresults_dir = tmp_path / "results" / "opencode"persisted = json.loads((results_dir / "with-skill" / "summary.json").read_text(encoding="utf-8"))- assert persisted["scores"] == {}+ assert persisted["scores"] == agent["with_skill"]assert persisted["custom_scores"] == {}- assert persisted["dimensions"] == {}- assert persisted["pass_at_k"] == {}+ assert persisted["dimensions"] == agent["dimensions_with_skill"]+ assert persisted["pass_at_k"] == partialassert not (results_dir / "lift.json").exists()assert not (results_dir / "custom_lift.json").exists()assert not (results_dir / "pass_at_k_lift.json").exists()diff --git a/tests/tier3/test_baseline_timeouts.py b/tests/tier3/test_baseline_timeouts.pyindex 9af70630..9f94b394 100644--- a/tests/tier3/test_baseline_timeouts.py+++ b/tests/tier3/test_baseline_timeouts.py@@ -33,7 +33,7 @@ from skillevaluator.tier3.harbor.baseline_timeouts import (settle,unexplained_launch_errors,)-from skillevaluator.tier3.harbor.collector import collect_harbor_results+from skillevaluator.tier3.harbor.collector import _build_comparison, collect_harbor_resultsSKILL = "demo"CASES = ["case-001", "case-002"]@@ -249,12 +249,12 @@ def test_tolerated_agent_stays_scored_on_its_with_skill_arm(tmp_path: Path) -> Nassert opencode["execution_status"] == "succeeded"assert opencode["execution_errors"] == []assert opencode["overall_with_skill"] is not None- # The baseline stays honest: failed, with its errors, and never scored or compared.+ # The baseline stays failed, with its errors and no aggregate or comparison.baseline = opencode["conditions"]["without_skill"]assert baseline["execution_status"] == "failed"assert (baseline["scored_attempts"], baseline["expected_attempts"]) == (1, 2)assert "Scored attempt coverage is 1/2" in baseline["execution_errors"]- assert opencode["without_skill"] == {}+ assert opencode["without_skill"] == SCORESassert opencode["overall_without_skill"] is Noneassert opencode["lift"] == {}assert opencode["pass_at_k"]["lift"] == {}@@ -262,6 +262,12 @@ def test_tolerated_agent_stays_scored_on_its_with_skill_arm(tmp_path: Path) -> Nassert (opencode["expected_attempts"], opencode["scored_attempts"]) == (4, 3)assert results["comparison"]["metrics"]["accuracy"]["opencode"]["without_skill"] is Noneassert results["comparison"]["metrics"]["accuracy"]["opencode"]["lift"] is None+ comparison = json.loads((tmp_path / "results" / "comparison.json").read_text(encoding="utf-8"))+ assert comparison["metrics"]["accuracy"]["opencode"] == {+ "with_skill": SCORES["accuracy"],+ "without_skill": None,+ "lift": None,+ }assert opencode[RECORD_KEY] == {"status": "partial","reason": "agent_execution_timeout",@@ -288,6 +294,28 @@ def test_tolerated_agent_stays_scored_on_its_with_skill_arm(tmp_path: Path) -> Nassert results["agents"]["codex"]["lift"][email protected]("failed_arm", ["with_skill", "without_skill"])+def test_succeeded_agent_comparison_withholds_failed_arm(failed_arm: str) -> None:+ agent = {+ "execution_status": "succeeded",+ "conditions": {+ arm: {"execution_status": "failed" if arm == failed_arm else "succeeded"}+ for arm in ("with_skill", "without_skill")+ },+ "with_skill": {"accuracy": 0.9},+ "without_skill": {"accuracy": 0.8},+ "lift": {"accuracy": {"delta": 0.1}},+ }++ comparison = _build_comparison({"opencode": agent})["metrics"]["accuracy"]["opencode"]++ assert comparison == {+ "with_skill": None if failed_arm == "with_skill" else 0.9,+ "without_skill": None if failed_arm == "without_skill" else 0.8,+ "lift": None,+ }++def test_reports_reload_the_tolerated_agent_from_disk(tmp_path: Path) -> None:from skillevaluator.evaluation.tier3_report import build_agent_eval_payloadfrom skillevaluator.reporting.base import (