Official Codebase of 3DCodeVerse
Python
bench: a harness run whose judge never answered is infra_failed, and its redo starts fresh
gaoypengd94950677f5d
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex 9aa2986f..5edc12d6 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -82,7 +82,7 @@ from codeverse3d.addons import select # noqa: E402from codeverse3d.config import get_settings # noqa: E402from codeverse3d.contracts.artifacts import RenderSet # noqa: E402from codeverse3d.contracts.common import ENTRY_FILE, Track # noqa: E402-from codeverse3d.contracts.run import RunRecord # noqa: E402+from codeverse3d.contracts.run import RunRecord, RunStatus # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402from codeverse3d.cost import run_ledger # noqa: E402from codeverse3d.cost.tally import tally # noqa: E402@@ -245,6 +245,13 @@ def entry_of(spec: Spec) -> str:return ENTRY_FILE[spec.language]+def _stopped_on_judge(run_ws: Workspace) -> bool:+ try:+ return RunRecord.model_validate_json(run_ws.record_path.read_text()).status is RunStatus.JUDGE_UNAVAILABLE+ except (OSError, ValueError):+ return False++def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptions,deps: CompareDeps, res: CellResult) -> None:run_ws = Workspace(cell / "run")@@ -256,6 +263,10 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOption# the measured story). The old tree is archived, never deleted: it holds that# attempt's cost ledger.archive_attempt(run_ws.root)+ elif run_ws.exists() and _stopped_on_judge(run_ws):+ # a redo of a cell dropped for its judge's outage: that run is finished, and resuming it+ # would hand back the same judge_unavailable record — start fresh, keep the old tree+ archive_attempt(run_ws.root)resume = run_ws.exists()if not resume:run_ws.create()@@ -274,6 +285,14 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptionres.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), summary.picked_scoreres.harness_stop_reason = summary.stop_reasonres.harness_aborted_rounds = len(rec.extra.get("aborted_rounds") or [])+ if rec.status is RunStatus.JUDGE_UNAVAILABLE:+ # The run stopped because its judge never answered (p3_graphics_v2 mushroom_forest, 2026-09-23:+ # two 900 s pro read timeouts after round 0). Scoring what it had then would grade a one-round+ # run for someone else's outage — the asymmetry §7 ends for the one-shot arms — so the cell is+ # dropped as infra_failed and --redo-status re-runs it.+ res.error = f"harness run stopped at {summary.stop_reason or 'judge_unavailable'}: the in-loop judge never answered"+ res.error_is_infra = True+ return# the DELIVERED code is the picked round's deliverable/ (src/ + public/ at its commit);# a run with no judged round handed nothing over, and its working tree — the last# round — is what it produced@@ -341,7 +360,7 @@ def run_cell(battery: Battery, item: BenchPrompt, arm: Arm, out: Path, opts: Com# evaluated and scored 0 — a hard zero for someone else's downtime, the exact# asymmetry tests/compare_bench/test_infra_failures.py exists to end. Drop the# cell instead (infra_failed); --redo-status re-runs the lost attempt only.- truncated = arm.kind not in ("harness", "agent") and res.error_is_infra+ truncated = res.error_is_infra # an arm whose protocol a provider outage cut shortif (eval_ws.root / entry_of(spec)).is_file() and not truncated:outcome = deps.evaluator.evaluate(eval_ws, spec)eval_ws.write_json(eval_ws.root / "eval.json", outcome)diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.pyindex f7f6c938..c932cc89 100644--- a/eval/tests/test_infra_failures.py+++ b/eval/tests/test_infra_failures.py@@ -277,3 +277,38 @@ def test_the_ab_viewer_refuses_to_call_a_winner_it_cannot_support():Run(slug="bx", arm="b", brief="brief x", picked=0.0)]head, why = verdict(arm("a", 0.5, 0.5, 0.5), b)assert head.startswith("Inconclusive") and "over 3 paired briefs" in why+++def test_a_harness_run_whose_judge_never_answered_is_dropped_and_redone_fresh(tmp_path):+ """judge_unavailable is the provider's failure, not the harness's: infra_failed, no score — and a redo+ archives the finished run instead of resuming it (p3_graphics_v2 mushroom_forest, 2026-09-23)."""+ from bench.compare_backends import CompareDeps, CompareOptions, parse_arm, run_cell+ from bench.run_bench import Battery+ from codeverse3d.contracts.run import RunStatus+ from tests.conftest import BATTERY, FakeEvaluator, fake_run_track++ storm, calm = fake_run_track(), fake_run_track()++ def judge_down(spec, ws, resume):+ rec = storm(spec, ws, resume)+ rec.status = RunStatus.JUDGE_UNAVAILABLE+ ws.write_json(ws.record_path, rec)+ return rec++ battery = Battery.load(BATTERY)+ opts = CompareOptions(judge="gemini:x", loop_judge="gemini:x")+ arm = parse_arm("harness:gemini-cli:gemini-3.8-flash")+ ev = FakeEvaluator()+ r = run_cell(battery, battery.prompts[0], arm, tmp_path, opts, CompareDeps(ev, run_track=judge_down))+ assert (r.status, r.score, r.passed) == ("infra_failed", None, None), r+ assert not ev.evaluated and "judge never answered" in r.error++ seen: list[bool] = []++ def redo(spec, ws, resume):+ seen.append(resume)+ return calm(spec, ws, resume)++ r2 = run_cell(battery, battery.prompts[0], arm, tmp_path, opts, CompareDeps(FakeEvaluator(), run_track=redo))+ assert seen == [False] and r2.status == "scored", (seen, r2)+ assert any(p.name.startswith("run.attempt") for p in (tmp_path / "cells" / battery.prompts[0].id / arm.slug).iterdir())