Skip to content

Official Codebase of 3DCodeVerse

Python

bench: an infra-failed cell makes its pairwise row unavailable, and a bare agent the provider interrupted is infra, not a zero

gaoypeng9cb210d6f07d
diff --git a/eval/bench/_compare_report.py b/eval/bench/_compare_report.py
index 5770f9d1..8dd6c0fb 100644
--- a/eval/bench/_compare_report.py
+++ b/eval/bench/_compare_report.py
@@ -96,13 +96,16 @@ class PairRow(BaseModel):
prompt_id: str
arm_a: str
arm_b: str
- winner: str = Field(description="a | b | tie")
+ winner: str = Field(description="a | b | tie | unavailable")
confidence: float = 0.0
reasons: list[str] = Field(default_factory=list)
orderings: list[dict] = Field(default_factory=list)
cost_usd: float = 0.0
judged: bool = Field(default=True, description="False when decided by a missing build (no judge call)")
error: str = ""
+ eligible: bool = Field(default=True, description="False when an infrastructure-failed cell makes this pair unavailable")
+ exclusion_reason: str = ""
+ rubric: str = Field(default="", description="Track rubric used by the pairwise judge; empty on historical rows")
def natural_key(self) -> tuple[str, ...]:
"""Row identity: one pairwise verdict is one (prompt, arm A, arm B)."""
@@ -176,6 +179,8 @@ def arm_stats(rows: list[CellResult]) -> list[ArmStats]:
def pair_stats(pairs: list[PairRow]) -> list[PairStats]:
groups: dict[tuple[str, str], list[PairRow]] = {}
for p in pairs:
+ if not p.eligible:
+ continue
groups.setdefault((p.arm_a, p.arm_b), []).append(p)
out = []
for (a, b), ps in groups.items():
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.py
index 5edc12d6..b1cdd176 100644
--- a/eval/bench/compare_backends.py
+++ b/eval/bench/compare_backends.py
@@ -57,7 +57,7 @@ for _p in (REPO, Path(__file__).resolve().parents[1]): # its codeverse3d +
from datetime import UTC # noqa: E402
from bench._compare_report import CellResult, PairRow, build_compare_report # noqa: E402
-from bench._fixed_eval import RUBRIC, EvalOutcome, FixedEvaluator # noqa: E402
+from bench._fixed_eval import EvalOutcome, FixedEvaluator, rubric_for # noqa: E402
from bench._infra import is_budget_exhaustion, is_infra_failure # noqa: E402
from bench._jsonl import latest, read_jsonl, seal_for_append # noqa: E402
from bench._oneshot import ( # noqa: E402
@@ -323,6 +323,11 @@ def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts:
res.gen_seconds = result.duration_s
res.minutes = round(max(0.0, result.duration_s - result.provider_wait_s) / 60, 2)
res.harness_status = result.exit_reason
+ if not result.ok and result.transient:
+ # A provider can fail after the CLI has written a placeholder or an
+ # unfinished revision. File existence is not a completed agent session.
+ res.error = f"bare agent interrupted by provider ({result.exit_reason}: {'; '.join(result.errors)[:600]})"
+ res.error_is_infra = True
if not (eval_ws.root / entry_of(spec)).is_file():
res.error = f"bare agent delivered no {entry_of(spec)} ({result.exit_reason}: {'; '.join(result.errors)[:300]})"
res.error_is_infra = bool(result.transient)
@@ -461,20 +466,33 @@ def run_pairwise(battery: Battery, cells: dict[tuple[str, str], CellResult], arm
for oa in oneshot:
key = (item.id, ha.raw, oa.raw)
ca, cb = cells.get((item.id, ha.raw)), cells.get((item.id, oa.raw))
- if key in done or ca is None or cb is None:
+ if ca is None or cb is None:
continue
- spec = spec_for(battery, item, ha, opts)
- ra, rb = _renders_of(out, ca), _renders_of(out, cb)
- if ra is None or rb is None:
- winner = "tie" if ra is None and rb is None else ("a" if rb is None else "b")
- row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=winner, confidence=1.0,
- reasons=["decided without the judge: a side has no renders (build failed)"], judged=False)
+ infra = any(c.status == "infra_failed" or c.error_is_infra for c in (ca, cb))
+ if infra:
+ if key in done and not done[key].eligible:
+ continue
+ # A provider outage is unavailable evidence, even if it left
+ # a partial render. Supersede cached automatic wins as well.
+ row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw,
+ winner="unavailable", judged=False, eligible=False,
+ exclusion_reason="A cell failed because of infrastructure; no quality comparison.",
+ reasons=["excluded: infrastructure failure"])
else:
- judge = judge or deps.pairwise_judge(opts.judge)
- pr = judge.compare(spec, ra, rb, rubric=RUBRIC)
- row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=pr.winner, confidence=pr.confidence,
- reasons=list(pr.reasons), orderings=list(pr.orderings), cost_usd=pr.usage.cost_usd,
- error=pr.error, judged=True)
+ if key in done and done[key].eligible:
+ continue
+ spec = spec_for(battery, item, ha, opts)
+ ra, rb = _renders_of(out, ca), _renders_of(out, cb)
+ if ra is None or rb is None:
+ winner = "tie" if ra is None and rb is None else ("a" if rb is None else "b")
+ row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=winner, confidence=1.0,
+ reasons=["decided without the judge: a side has no renders (build failed)"], judged=False)
+ else:
+ judge = judge or deps.pairwise_judge(opts.judge)
+ pr = judge.compare(spec, ra, rb, rubric=rubric_for(spec))
+ row = PairRow(prompt_id=item.id, arm_a=ha.raw, arm_b=oa.raw, winner=pr.winner, confidence=pr.confidence,
+ reasons=list(pr.reasons), orderings=list(pr.orderings), cost_usd=pr.usage.cost_usd,
+ error=pr.error, judged=True, rubric=rubric_for(spec))
done[key] = row
fh.write(row.model_dump_json() + "\n")
fh.flush()
diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.py
index c932cc89..bfbb76c7 100644
--- a/eval/tests/test_infra_failures.py
+++ b/eval/tests/test_infra_failures.py
@@ -9,6 +9,64 @@ import pytest
from bench._compare_report import CellResult, arm_stats
from bench._infra import is_infra_failure
+
[email protected]("failed_side", [0, 1])
+def test_pairwise_outage_is_unavailable_even_with_cached_win(tmp_path, monkeypatch, failed_side):
+ from bench import compare_backends as cb
+ from bench._compare_report import PairRow, pair_stats
+ from bench.run_bench import Battery
+ from tests.conftest import BATTERY, FakeEvaluator
+
+ battery = Battery.load(BATTERY)
+ battery = battery.model_copy(update={"prompts": battery.prompts[:1]})
+ prompt = battery.prompts[0].id
+ arms = cb.parse_arms("harness:codex:gpt-6-astra@high,agent:gemini-cli:gemini-3.8-flash")
+ rows = [CellResult(prompt_id=prompt, arm=a.raw, kind=a.kind, status="scored", score=0.6) for a in arms]
+ rows[failed_side] = rows[failed_side].model_copy(update={"status": "infra_failed", "error_is_infra": True, "score": None})
+ cells = {(r.prompt_id, r.arm): r for r in rows}
+ old = PairRow(prompt_id=prompt, arm_a=arms[0].raw, arm_b=arms[1].raw, winner="a", judged=False)
+ path = tmp_path / "pairwise.jsonl"
+ path.write_text(old.model_dump_json() + "\n")
+
+ def no_render_lookup(*args):
+ pytest.fail("An outage cannot enter the missing-render or partial-render comparison path")
+
+ monkeypatch.setattr(cb, "_renders_of", no_render_lookup)
+ deps = cb.CompareDeps(FakeEvaluator())
+ pairs = cb.run_pairwise(battery, cells, arms, tmp_path, cb.CompareOptions(), deps)
+ assert len(pairs) == 1 and not pairs[0].eligible
+ assert pairs[0].winner == "unavailable" and not pairs[0].judged
+ assert pair_stats(pairs) == [] # neither a loss nor a tie in any denominator
+ assert len(path.read_text().splitlines()) == 2 # old evidence retained, superseded
+ cb.run_pairwise(battery, cells, arms, tmp_path, cb.CompareOptions(), deps)
+ assert len(path.read_text().splitlines()) == 2 # resume does not duplicate the exclusion
+
+
+def test_scene_pairwise_uses_the_same_track_rubric_as_fixed_evaluation(tmp_path, monkeypatch):
+ from bench import compare_backends as cb
+ from bench.run_bench import Battery
+ from codeverse3d.contracts.common import Language, Track
+ from tests.conftest import BATTERY, FakeEvaluator, FakePairwise
+
+ battery = Battery.load(BATTERY)
+ battery = battery.model_copy(update={"prompts": battery.prompts[:1], "track": Track.SCENE,
+ "language": Language.SCENE_THREEJS})
+ arms = cb.parse_arms("harness:codex:gpt-6-astra@high,agent:gemini-cli:gemini-3.8-flash")
+ cells = {(battery.prompts[0].id, a.raw): CellResult(prompt_id=battery.prompts[0].id,
+ arm=a.raw, kind=a.kind, status="scored", score=0.6) for a in arms}
+ monkeypatch.setattr(cb, "_renders_of", lambda *args: object())
+ seen = []
+
+ class Judge(FakePairwise):
+ def compare(self, spec, ra, rb, *, rubric=None):
+ seen.append(rubric)
+ return super().compare(spec, ra, rb, rubric=rubric)
+
+ pairs = cb.run_pairwise(battery, cells, arms, tmp_path, cb.CompareOptions(),
+ cb.CompareDeps(FakeEvaluator(), pairwise_judge=Judge))
+ assert seen == ["scene_v1"]
+ assert pairs[0].rubric == "scene_v1"
+
OUTAGES = [
"ModelError: Gemini API error 503: This model is currently experiencing high demand.",
"ModelError: Gemini request timed out: The read operation timed out",
@@ -312,3 +370,33 @@ def test_a_harness_run_whose_judge_never_answered_is_dropped_and_redone_fresh(tm
r2 = run_cell(battery, battery.prompts[0], arm, tmp_path, opts, CompareDeps(FakeEvaluator(), run_track=redo))
assert seen == [False] and r2.status == "scored", (seen, r2)
assert any(p.name.startswith("run.attempt") for p in (tmp_path / "cells" / battery.prompts[0].id / arm.slug).iterdir())
+
+
[email protected]('outage', [True, False])
+def test_bare_agent_provider_failure_with_a_partial_file_is_not_a_quality_zero(tmp_path, monkeypatch, outage):
+ """A CLI can leave a placeholder before its API fails; preserve but don't grade it."""
+ from bench.compare_backends import CompareDeps, CompareOptions, parse_arm, run_cell
+ from bench.run_bench import Battery
+ from codeverse3d.contracts.agent import AgentResult
+ from tests.conftest import BATTERY, GOOD, FakeEvaluator
+
+ def interrupted(spec, target, cell, eval_ws, *, minutes):
+ # A complete file on disk is still a partial session; use invalid model
+ # code so the calm control demonstrates that capability errors keep 0.
+ eval_ws.src.mkdir(parents=True, exist_ok=True)
+ (eval_ws.src / 'model.py').write_text(GOOD.format(score=0.5).replace('\n', '\n# BOOM\n', 1))
+ return AgentResult(ok=False, exit_reason='error', transient=outage,
+ errors=['Gemini API error 503' if outage else 'model produced invalid code'])
+
+ monkeypatch.setattr('bench._bare_agent.run_bare_agent', interrupted)
+ battery=Battery.load(BATTERY)
+ evaluator=FakeEvaluator()
+ result=run_cell(battery,battery.prompts[0],parse_arm('agent:gemini-cli:gemini-3.8-flash'),
+ tmp_path,CompareOptions(judge='gemini:x',loop_judge='gemini:x'),CompareDeps(evaluator))
+ if outage:
+ assert (result.status,result.score,result.error_is_infra)==('infra_failed',None,True)
+ assert evaluator.evaluated==[]
+ assert '503' in result.error
+ else:
+ assert (result.status,result.score)==('build_failed',0.0)
+ assert evaluator.evaluated