Official Codebase of 3DCodeVerse
Python
bench: an infra-failed cell makes its pairwise row unavailable, and a bare agent the provider interrupted is infra, not a zero
gaoypeng9cb210d6f07d
diff --git a/eval/bench/_compare_report.py b/eval/bench/_compare_report.pyindex 5770f9d1..8dd6c0fb 100644--- a/eval/bench/_compare_report.py+++ b/eval/bench/_compare_report.py@@ -96,13 +96,16 @@ class PairRow(BaseModel):prompt_id: strarm_a: strarm_b: str- winner: str = Field(description="a | b | tie")+ winner: str = Field(description="a | b | tie | unavailable")confidence: float = 0.0reasons: list[str] = Field(default_factory=list)orderings: list[dict] = Field(default_factory=list)cost_usd: float = 0.0judged: bool = Field(default=True, description="False when decided by a missing build (no judge call)")error: str = ""+ eligible: bool = Field(default=True, description="False when an infrastructure-failed cell makes this pair unavailable")+ exclusion_reason: str = ""+ rubric: str = Field(default="", description="Track rubric used by the pairwise judge; empty on historical rows")def natural_key(self) -> tuple[str, ...]:"""Row identity: one pairwise verdict is one (prompt, arm A, arm B)."""@@ -176,6 +179,8 @@ def arm_stats(rows: list[CellResult]) -> list[ArmStats]:def pair_stats(pairs: list[PairRow]) -> list[PairStats]:groups: dict[tuple[str, str], list[PairRow]] = {}for p in pairs:+ if not p.eligible:+ continuegroups.setdefault((p.arm_a, p.arm_b), []).append(p)out = []for (a, b), ps in groups.items():diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex 5edc12d6..b1cdd176 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -57,7 +57,7 @@ for _p in (REPO, Path(__file__).resolve().parents[1]): # its codeverse3d +from datetime import UTC # noqa: E402from bench._compare_report import CellResult, PairRow, build_compare_report # noqa: E402-from bench._fixed_eval import RUBRIC, EvalOutcome, FixedEvaluator # noqa: E402+from bench._fixed_eval import EvalOutcome, FixedEvaluator, rubric_for # noqa: E402from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402from bench._oneshot import ( # noqa: E402@@ -323,6 +323,11 @@ def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts:res.gen_seconds = result.duration_sres.minutes = round(max(0.0, result.duration_s - result.provider_wait_s) / 60, 2)res.harness_status = result.exit_reason+ if not result.ok and result.transient:+ # A provider can fail after the CLI has written a placeholder or an+ # unfinished revision. File existence is not a completed agent session.+ res.error = f"bare agent interrupted by provider ({result.exit_reason}: {'; '.join(result.errors)[:600]})"+ res.error_is_infra = Trueif not (eval_ws.root / entry_of(spec)).is_file():res.error = f"bare agent delivered no {entry_of(spec)} ({result.exit_reason}: {'; '.join(result.errors)[:300]})"res.error_is_infra = bool(result.transient)@@ -461,20 +466,33 @@ def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], armfor oa in oneshot:key = (item.id, ha.raw, oa.raw)ca, cb = cells.get((item.id, ha.raw)), cells.get((item.id, oa.raw))- if key in done or ca is None or cb is None:+ if ca is None or cb is None:continue- spec = spec_for(battery, item, ha, opts)- ra, rb = _renders_of(out, ca), _renders_of(out, cb)- if ra is None or rb is None:- winner = "tie" if ra is None and rb is None else ("a" if rb is None else "b")- row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=winner, confidence=1.0,- reasons=["decided without the judge: a side has no renders (build failed)"], judged=False)+ infra = any(c.status == "infra_failed" or c.error_is_infra for c in (ca, cb))+ if infra:+ if key in done and not done[key].eligible:+ continue+ # A provider outage is unavailable evidence, even if it left+ # a partial render. Supersede cached automatic wins as well.+ row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw,+ winner="unavailable", judged=False, eligible=False,+ exclusion_reason="A cell failed because of infrastructure; no quality comparison.",+ reasons=["excluded: infrastructure failure"])else:- judge = judge or deps.pairwise_judge(opts.judge)- pr = judge.compare(spec, ra, rb, rubric=RUBRIC)- row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=pr.winner, confidence=pr.confidence,- reasons=list(pr.reasons), orderings=list(pr.orderings), cost_usd=pr.usage.cost_usd,- error=pr.error, judged=True)+ if key in done and done[key].eligible:+ continue+ spec = spec_for(battery, item, ha, opts)+ ra, rb = _renders_of(out, ca), _renders_of(out, cb)+ if ra is None or rb is None:+ winner = "tie" if ra is None and rb is None else ("a" if rb is None else "b")+ row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=winner, confidence=1.0,+ reasons=["decided without the judge: a side has no renders (build failed)"], judged=False)+ else:+ judge = judge or deps.pairwise_judge(opts.judge)+ pr = judge.compare(spec, ra, rb, rubric=rubric_for(spec))+ row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=pr.winner, confidence=pr.confidence,+ reasons=list(pr.reasons), orderings=list(pr.orderings), cost_usd=pr.usage.cost_usd,+ error=pr.error, judged=True, rubric=rubric_for(spec))done[key] = rowfh.write(row.model_dump_json() + "\n")fh.flush()diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.pyindex c932cc89..bfbb76c7 100644--- a/eval/tests/test_infra_failures.py+++ b/eval/tests/test_infra_failures.py@@ -9,6 +9,64 @@ import pytestfrom bench._compare_report import CellResult, arm_statsfrom bench._infra import is_infra_failure+[email protected]("failed_side", [0, 1])+def test_pairwise_outage_is_unavailable_even_with_cached_win(tmp_path, monkeypatch, failed_side):+ from bench import compare_backends as cb+ from bench._compare_report import PairRow, pair_stats+ from bench.run_bench import Battery+ from tests.conftest import BATTERY, FakeEvaluator++ battery = Battery.load(BATTERY)+ battery = battery.model_copy(update={"prompts": battery.prompts[:1]})+ prompt = battery.prompts[0].id+ arms = cb.parse_arms("harness:codex:gpt-6-astra@high,agent:gemini-cli:gemini-3.8-flash")+ rows = [CellResult(prompt_id=prompt, arm=a.raw, kind=a.kind, status="scored", score=0.6) for a in arms]+ rows[failed_side] = rows[failed_side].model_copy(update={"status": "infra_failed", "error_is_infra": True, "score": None})+ cells = {(r.prompt_id, r.arm): r for r in rows}+ old = PairRow(prompt_id=prompt, arm_a=arms[0].raw, arm_b=arms[1].raw, winner="a", judged=False)+ path = tmp_path / "pairwise.jsonl"+ path.write_text(old.model_dump_json() + "\n")++ def no_render_lookup(*args):+ pytest.fail("An outage cannot enter the missing-render or partial-render comparison path")++ monkeypatch.setattr(cb, "_renders_of", no_render_lookup)+ deps = cb.CompareDeps(FakeEvaluator())+ pairs = cb.run_pairwise(battery, cells, arms, tmp_path, cb.CompareOptions(), deps)+ assert len(pairs) == 1 and not pairs[0].eligible+ assert pairs[0].winner == "unavailable" and not pairs[0].judged+ assert pair_stats(pairs) == [] # neither a loss nor a tie in any denominator+ assert len(path.read_text().splitlines()) == 2 # old evidence retained, superseded+ cb.run_pairwise(battery, cells, arms, tmp_path, cb.CompareOptions(), deps)+ assert len(path.read_text().splitlines()) == 2 # resume does not duplicate the exclusion+++def test_scene_pairwise_uses_the_same_track_rubric_as_fixed_evaluation(tmp_path, monkeypatch):+ from bench import compare_backends as cb+ from bench.run_bench import Battery+ from codeverse3d.contracts.common import Language, Track+ from tests.conftest import BATTERY, FakeEvaluator, FakePairwise++ battery = Battery.load(BATTERY)+ battery = battery.model_copy(update={"prompts": battery.prompts[:1], "track": Track.SCENE,+ "language": Language.SCENE_THREEJS})+ arms = cb.parse_arms("harness:codex:gpt-6-astra@high,agent:gemini-cli:gemini-3.8-flash")+ cells = {(battery.prompts[0].id, a.raw): CellResult(prompt_id=battery.prompts[0].id,+ arm=a.raw, kind=a.kind, status="scored", score=0.6) for a in arms}+ monkeypatch.setattr(cb, "_renders_of", lambda *args: object())+ seen = []++ class Judge(FakePairwise):+ def compare(self, spec, ra, rb, *, rubric=None):+ seen.append(rubric)+ return super().compare(spec, ra, rb, rubric=rubric)++ pairs = cb.run_pairwise(battery, cells, arms, tmp_path, cb.CompareOptions(),+ cb.CompareDeps(FakeEvaluator(), pairwise_judge=Judge))+ assert seen == ["scene_v1"]+ assert pairs[0].rubric == "scene_v1"+OUTAGES = ["ModelError: Gemini API error 503: This model is currently experiencing high demand.","ModelError: Gemini request timed out: The read operation timed out",@@ -312,3 +370,33 @@ def test_a_harness_run_whose_judge_never_answered_is_dropped_and_redone_fresh(tmr2 = run_cell(battery, battery.prompts[0], arm, tmp_path, opts, CompareDeps(FakeEvaluator(), run_track=redo))assert seen == [False] and r2.status == "scored", (seen, r2)assert any(p.name.startswith("run.attempt") for p in (tmp_path / "cells" / battery.prompts[0].id / arm.slug).iterdir())++[email protected]('outage', [True, False])+def test_bare_agent_provider_failure_with_a_partial_file_is_not_a_quality_zero(tmp_path, monkeypatch, outage):+ """A CLI can leave a placeholder before its API fails; preserve but don't grade it."""+ from bench.compare_backends import CompareDeps, CompareOptions, parse_arm, run_cell+ from bench.run_bench import Battery+ from codeverse3d.contracts.agent import AgentResult+ from tests.conftest import BATTERY, GOOD, FakeEvaluator++ def interrupted(spec, target, cell, eval_ws, *, minutes):+ # A complete file on disk is still a partial session; use invalid model+ # code so the calm control demonstrates that capability errors keep 0.+ eval_ws.src.mkdir(parents=True, exist_ok=True)+ (eval_ws.src / 'model.py').write_text(GOOD.format(score=0.5).replace('\n', '\n# BOOM\n', 1))+ return AgentResult(ok=False, exit_reason='error', transient=outage,+ errors=['Gemini API error 503' if outage else 'model produced invalid code'])++ monkeypatch.setattr('bench._bare_agent.run_bare_agent', interrupted)+ battery=Battery.load(BATTERY)+ evaluator=FakeEvaluator()+ result=run_cell(battery,battery.prompts[0],parse_arm('agent:gemini-cli:gemini-3.8-flash'),+ tmp_path,CompareOptions(judge='gemini:x',loop_judge='gemini:x'),CompareDeps(evaluator))+ if outage:+ assert (result.status,result.score,result.error_is_infra)==('infra_failed',None,True)+ assert evaluator.evaluated==[]+ assert '503' in result.error+ else:+ assert (result.status,result.score)==('build_failed',0.0)+ assert evaluator.evaluated