Official Codebase of 3DCodeVerse
Python
eval: bench readers read select — baseline → picked, the harness arm delivers its picked round
gaoypeng599c2a486650
diff --git a/eval/bench/ab_gate_rates.py b/eval/bench/ab_gate_rates.pyindex 444d2471..9bb25137 100644--- a/eval/bench/ab_gate_rates.py+++ b/eval/bench/ab_gate_rates.py@@ -2,7 +2,8 @@The judged score carries the planner's variance (docs/EVAL.md 8.1); these do not carry thejudge's. They are what c3d-part-contact and c3d-bbox-contract are FOR, so they are the-primary readout and the score is the second.+primary readout and the score is the second. Both are read off the round+``codeverse3d.addons.select`` picks (the last round when nothing was judged)."""from __future__ import annotations@@ -11,6 +12,11 @@ import reimport sysfrom pathlib import Path+for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):+ sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)++from codeverse3d.addons import select # noqa: E402+if len(sys.argv) < 2:raise SystemExit("usage: python bench/ab_gate_rates.py bench/out/<ab-run>")OUT = Path(sys.argv[1])@@ -21,10 +27,11 @@ def cell_stats(record: Path) -> dict | None:rounds = rec.get("rounds") or []if not rounds:return None- last = rounds[-1]+ summary = select.summarise(record.parent)+ rnd = next((r for r in rounds if r.get("index") == summary.picked_round), rounds[-1])pairs = floats = islands = contract_n = 0worst = 0.0- for g in last.get("gates") or []:+ for g in rnd.get("gates") or []:for f in g.get("findings") or []:sev, msg = str(f.get("severity", "")), str(f.get("message", ""))if sev == "info":@@ -42,7 +49,7 @@ def cell_stats(record: Path) -> dict | None:contract_n += 1return {"pairs": pairs, "worst_depth_mm": worst, "floating": floats,"islands": islands, "contract_findings": contract_n,- "score": rec.get("final_score"), "status": rec.get("status")}+ "score": summary.picked_score, "status": summary.stop_reason}rows: dict[str, dict[str, dict]] = {}for arm in ("control", "variant"):diff --git a/eval/bench/ab_view.py b/eval/bench/ab_view.pyindex 3461837b..21715636 100644--- a/eval/bench/ab_view.py+++ b/eval/bench/ab_view.py@@ -23,9 +23,15 @@ import htmlimport ioimport jsonimport statistics+import sysfrom dataclasses import dataclass, fieldfrom pathlib import Path+for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):+ sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)++from codeverse3d.addons import select # noqa: E402+SHEET_W = 1100@@ -35,7 +41,7 @@ class Run:arm: strbrief: str = ""status: str = "?"- final: float | None = None+ picked: float | None = Nonebaseline: float | None = Nonerounds: int = 0usd: float = 0.0@@ -73,8 +79,11 @@ def load_run(run_dir: Path, arm: str) -> Run:d = json.loads(rec.read_text())b = (d.get("extra") or {}).get("budget") or {}r.brief = str((d.get("spec") or {}).get("prompt") or "")- r.status = str(d.get("status") or "?")- r.final, r.baseline = d.get("final_score"), d.get("baseline_score")+ try: # the round addons/select picks; a record it cannot read keeps only its status+ s = select.summarise(run_dir)+ r.status, r.picked, r.baseline = s.stop_reason or "?", s.picked_score, s.baseline_score+ except Exception: # noqa: BLE001 - one unreadable record must not blank the page+ r.status = str(d.get("status") or "?")r.rounds = len(d.get("rounds") or [])r.usd, r.minutes = round(float(b.get("spent_usd") or 0), 2), round(float(b.get("elapsed_min") or 0))r.error = str(d.get("error") or "")[:300]@@ -97,8 +106,8 @@ MIN_RUNS_PER_ARM = 3def verdict(a: list[Run], b: list[Run]) -> tuple[str, str]:"""(headline, why) — refuses to call a winner the sample cannot support."""- sa = [x.final for x in a if x.final is not None]- sb = [x.final for x in b if x.final is not None]+ sa = [x.picked for x in a if x.picked is not None]+ sb = [x.picked for x in b if x.picked is not None]if len(sa) < MIN_RUNS_PER_ARM or len(sb) < MIN_RUNS_PER_ARM:return ("Inconclusive — not enough scored runs",f"arm A scored {len(sa)} of {len(a)} runs, arm B {len(sb)} of {len(b)}; this needs "@@ -122,7 +131,7 @@ def _num(v: object) -> str:def _stats(rs: list[Run]) -> str:- sc = [x.final for x in rs if x.final is not None]+ sc = [x.picked for x in rs if x.picked is not None]if not sc:return "no scored run"return (f"n={len(sc)}/{len(rs)} · mean {statistics.mean(sc):.3f} · "@@ -135,7 +144,7 @@ def _panel(r: Run | None, side: str) -> str:page can keep them hidden until the reader has actually looked."""if r is None:return f'<div class="panel empty" data-side="{side}"><p>this arm has no run for this brief</p></div>'- tone = {"passed": "good", "failed": "bad", "budget": "warn"}.get(r.status, "warn")+ tone = {"max_rounds": "good", "failed": "bad"}.get(r.status, "warn") # why it stopped, not a verdictimg = (f'<button class="sheet" data-full="{r.sheet}" 'f'aria-label="enlarge {html.escape(r.slug)}"><img src="{r.sheet}" 'f'alt="rendered frames for {html.escape(r.slug)}" loading="lazy"></button>'@@ -147,7 +156,7 @@ def _panel(r: Run | None, side: str) -> str:{img}<div class="nums"><span class="pill {tone}">{html.escape(r.status)}</span>- <span class="n"><b>{_num(r.final)}</b> score</span>+ <span class="n"><b>{_num(r.picked)}</b> score</span><span class="n">{r.rounds} rounds</span><span class="n">${r.usd:.2f}</span><span class="n">{r.minutes} min</span>@@ -158,8 +167,8 @@ def _panel(r: Run | None, side: str) -> str:def _pair(i: int, brief: str, a: Run | None, b: Run | None, flip: bool) -> str:left, right = (b, a) if flip else (a, b)- da = "" if a is None or a.final is None else f'{a.final:.3f}'- db = "" if b is None or b.final is None else f'{b.final:.3f}'+ da = "" if a is None or a.picked is None else f'{a.picked:.3f}'+ db = "" if b is None or b.picked is None else f'{b.picked:.3f}'return f"""<section class="pair" id="p{i}" data-i="{i}" data-flip="{'1' if flip else '0'}"data-a="{da}" data-b="{db}"><header class="pairhead">diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex 58671c16..9965abf6 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -80,8 +80,10 @@ from bench.run_bench import ( # noqa: E402archive_attempt,build_spec,default_run_track,+ hand_over,select_prompts,)+from codeverse3d.addons import select # noqa: E402from codeverse3d.config import get_settings # noqa: E402from codeverse3d.contracts.artifacts import RenderSet # noqa: E402from codeverse3d.contracts.common import ENTRY_FILE, Track # noqa: E402@@ -265,20 +267,27 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptionwith (exclusive(run_ws.root, what=f"compare {spec.id}:{run_ws.root.parent.name}"),run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}")):rec = deps.run_track(spec, run_ws, resume)+ hand_over(run_ws, rec) # what `3dcode make` delivers: the picked round, packaged+ summary = select.summarise(run_ws.root, record=rec)res.gen_cost_usd = rec.total_usage.cost_usdres.tool_calls = rec.total_usage.tool_calls- res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), rec.final_score- res.harness_stop_reason = str(rec.extra.get("stop_reason") or "")+ res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), summary.picked_score+ res.harness_stop_reason = summary.stop_reasonres.harness_aborted_rounds = len(rec.extra.get("aborted_rounds") or [])- if not (run_ws.root / entry_of(spec)).is_file():- res.error = f"harness run produced no {entry_of(spec)} (status {rec.status.value}: {rec.error})"+ # the DELIVERED code is the picked round's deliverable/ (src/ + public/ at its commit);+ # a run with no judged round handed nothing over, and its working tree — the last+ # round — is what it produced+ delivered = run_ws.deliverable if (run_ws.deliverable / entry_of(spec)).is_file() else run_ws.root+ if not (delivered / entry_of(spec)).is_file():+ res.error = f"harness run produced no {entry_of(spec)} (stop {summary.stop_reason}: {rec.error})"return# the whole src/ tree: agents may split helpers into src/parts/*.py (the build wrapper puts src/ on sys.path)- shutil.copytree(run_ws.src, eval_ws.src, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))+ shutil.copytree(delivered / "src", eval_ws.src, dirs_exist_ok=True,+ ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))# a scene's Blender heroes live in public/assets/*.glb, preloaded by src/scene.js over http:# without them every hero clone is a 404 and the fixed evaluation judges an empty zone- if spec.track is Track.SCENE and (run_ws.root / "public").is_dir():- shutil.copytree(run_ws.root / "public", eval_ws.root / "public", dirs_exist_ok=True)+ if spec.track is Track.SCENE and (delivered / "public").is_dir():+ shutil.copytree(delivered / "public", eval_ws.root / "public", dirs_exist_ok=True)def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptions,diff --git a/eval/bench/complexity_report.py b/eval/bench/complexity_report.pyindex cba2b44c..00eb1dfb 100644--- a/eval/bench/complexity_report.py+++ b/eval/bench/complexity_report.py@@ -11,10 +11,12 @@ or down, and what does a complexity point cost?**python bench/complexity_report.py bench/out/complexity_v3 --battery bench/prompts/complexity_v3.yamlA "root" is a battery directory (``<battery>/runs/<slug>``), a runs directory,-or a single run directory — all three are detected. The complexity vector is-read from the run's recorded measurement when present and recomputed from-``artifacts/object.glb`` otherwise, so the whole historic corpus is usable.-Rows without a judgment or without geometry are skipped and counted.+or a single run directory — all three are detected. A run is the round+``codeverse3d.addons.select`` picks (a record it cannot read: the last round). The+complexity vector is read from that round's recorded measurement when present and+recomputed from its GLB (``artifacts/rNN/object.glb``, else ``artifacts/object.glb``)+otherwise, so the whole historic corpus is usable. Rows without a judgment or without+geometry are skipped and counted. There is no pass rate: a run is not passed or failed."""from __future__ import annotations@@ -31,6 +33,7 @@ from typing import Anyfor _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)+from codeverse3d.addons import select # noqa: E402from codeverse3d.addons.calibration import _ranks # noqa: E402 — tie-averaged ranks, one copyfrom codeverse3d.contracts.run import RunId # noqa: E402from codeverse3d.proc import read_json_or_none # noqa: E402@@ -67,21 +70,22 @@ class Row(dict):_read_json = read_json_or_none-def _vector_for(run: Path, best: dict[str, Any] | None) -> ComplexityVector | None:+def _vector_for(run: Path, rnd: dict[str, Any] | None) -> ComplexityVector | None:"""The run's recorded complexity vector, else one recomputed from its GLB.The historic corpus predates the vector, so recomputing is the normal pathfor anything recorded before 2026-08-24 — that is what makes the wholecorpus comparable."""- for meas in (best or {}).get("measurement"), (_read_json(run / "artifacts" / "measurement.json") or {}):+ for meas in (rnd or {}).get("measurement"), (_read_json(run / "artifacts" / "measurement.json") or {}):block = ((meas or {}).get("extra") or {}).get("complexity")if isinstance(block, dict):try:return ComplexityVector.model_validate(block)except Exception: # noqa: BLE001 - an older/foreign block must not stop the scanpass- glb = run / "artifacts" / "object.glb"+ kept = run / "artifacts" / f"r{int((rnd or {}).get('index') or 0):02d}" / "object.glb"+ glb = kept if kept.is_file() else run / "artifacts" / "object.glb"if not glb.is_file():return Nonefrom codeverse3d.spatial.complexity import complexity_of_glb@@ -92,15 +96,15 @@ def _vector_for(run: Path, best: dict[str, Any] | None) -> ComplexityVector | Noreturn None-def _best_round(record: dict[str, Any]) -> dict[str, Any] | None:+def _picked_round(run: Path, record: dict[str, Any]) -> dict[str, Any] | None:+ """The round ``addons/select`` picks, as its raw dict; the last round when the record+ will not load (the scan reads the historic corpus as-is)."""rounds = record.get("rounds") or []- if not rounds:- return None- idx = record.get("best_round")- for r in rounds:- if r.get("index") == idx:- return r- return rounds[-1]+ try:+ idx = select.summarise(run).picked_round+ except Exception: # noqa: BLE001 - one unreadable record must not stop the scan+ idx = None+ return next((r for r in rounds if r.get("index") == idx), rounds[-1] if rounds else None)def _minutes(record: dict[str, Any]) -> float:@@ -114,10 +118,10 @@ def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:record = _read_json(run / "record.json")if not record:return None- best = _best_round(record)- judgment = (best or {}).get("judgment") or {}+ rnd = _picked_round(run, record)+ judgment = (rnd or {}).get("judgment") or {}scores = judgment.get("scores") or {}- vec = _vector_for(run, best)+ vec = _vector_for(run, rnd)if vec is None or not scores:return Noneplan_parts = len((record.get("plan") or {}).get("parts") or [])@@ -132,9 +136,8 @@ def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:plan_parts=plan_parts,parts_per_plan_part=round(vec.part_count / plan_parts, 3) if plan_parts else None,gate_errors=sum(len([f for f in (g.get("findings") or []) if f.get("severity") == "error"])- for g in (best or {}).get("gates") or []),+ for g in (rnd or {}).get("gates") or []),overall=judgment.get("overall"),- passed=judgment.get("passed"),issues=len(judgment.get("issues") or []),cost_usd=round(float((record.get("total_usage") or {}).get("cost_usd") or 0.0), 4),minutes=_minutes(record),@@ -238,18 +241,17 @@ def _mean(vals: Iterable[float | None]) -> float | None:def scatter_table(rows: Sequence[Row]) -> str:"""Score vs complexity, bucketed by index — the scatter as a text table."""lines = [- "| complexity band | n | mean index | mean overall | pass % | detail | struct | fit | craft | $/run | min |",- "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|",+ "| complexity band | n | mean index | mean overall | detail | struct | fit | craft | $/run | min |",+ "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|",]for lo, hi in zip(BUCKETS, BUCKETS[1:], strict=False):b = [r for r in rows if isinstance(r.get("index"), (int, float)) and lo <= r["index"] < hi]if not b:continue- npass = sum(1 for r in b if r.get("passed"))lines.append(f"| {band_of((lo + hi) / 2)} {lo:.2f}–{min(hi, 1.0):.2f} | {len(b)} |"f"{_fmt(_mean(r['index'] for r in b))} |{_fmt(_mean(r['overall'] for r in b))} |"- f" {100 * npass / len(b):5.0f} |{_fmt(_mean(r['geometry_detail'] for r in b))} |"+ f"{_fmt(_mean(r['geometry_detail'] for r in b))} |"f"{_fmt(_mean(r['structure_plausibility'] for r in b))} |{_fmt(_mean(r['assembly_fit'] for r in b))} |"f"{_fmt(_mean(r['craftsmanship_no_artifacts'] for r in b))} |"f"{_fmt(_mean(r['cost_usd'] for r in b))} |{_fmt(_mean(r['minutes'] for r in b))} |"@@ -275,9 +277,6 @@ def dollars_per_point(rows: Sequence[Row]) -> dict[str, Any]:cost = sum(float(r.get("cost_usd") or 0.0) for r in rows)mins = sum(float(r.get("minutes") or 0.0) for r in rows)points = sum(100.0 * float(r.get("index") or 0.0) for r in rows)- passed = [r for r in rows if r.get("passed")]- p_cost = sum(float(r.get("cost_usd") or 0.0) for r in passed)- p_points = sum(100.0 * float(r.get("index") or 0.0) for r in passed)return {"runs": len(rows),"total_usd": round(cost, 3),@@ -285,8 +284,6 @@ def dollars_per_point(rows: Sequence[Row]) -> dict[str, Any]:"complexity_points": round(points, 1),"usd_per_point": round(cost / points, 4) if points else None,"minutes_per_point": round(mins / points, 3) if points else None,- "usd_per_point_passed_only": round(p_cost / p_points, 4) if p_points else None,- "passed": len(passed),}diff --git a/eval/bench/coupling_stats.py b/eval/bench/coupling_stats.pyindex e46d716d..35fa21a1 100644--- a/eval/bench/coupling_stats.py+++ b/eval/bench/coupling_stats.py@@ -31,10 +31,10 @@ from codeverse3d.spatial.joints_model import UrdfError, load_urdf # noqa: E402def _urdfs(root: Path) -> list[Path]:"""The BUILT ``robot.urdf`` of each run under ``root``, once per run.- A run holds the same file three times — ``src/`` (what the agent wrote),- ``artifacts/`` (what the build produced) and ``deliverable/`` (what finalise copied) —- so counting files instead of runs multiplies every mechanical number by three. The- artefact is the one the sweep actually posed. The walk is+ A run holds the same file several times — ``src/`` (what the agent wrote),+ ``artifacts/`` (the last build), ``artifacts/rNN/`` (every round's own copy) and+ ``deliverable/`` (the handed-over round) — so counting files instead of runs multiplies+ every mechanical number. The artefact is the one the last sweep actually posed. The walk is``flywheel.record.unique_files``, so a cell symlinked from another battery is one run."""return [p for p in unique_files(root, "robot.urdf") if p.parent.name == "artifacts"]diff --git a/eval/bench/h2h_glb.py b/eval/bench/h2h_glb.pyindex 4ad1545d..9c153840 100644--- a/eval/bench/h2h_glb.py+++ b/eval/bench/h2h_glb.py@@ -6,9 +6,10 @@ Protocol (one fixed judge, one renderer, one gate set, for BOTH sides):verbatim with an empty ``must_have`` (their run had no checklist, so ours gets none);each prompt's last tag is the gallery slug.2. THEIR GLB is ``<gallery>/<slug>/object.glb`` (built by their harness, never- rebuilt here). OUR GLB is the best round of the harness run under- ``<bench_out>/<lang>/runs/<id>/`` (``deliverable/object.glb``, else- ``artifacts/object.glb`` — ``finalise`` restores and rebuilds the best round).+ rebuilt here). OUR GLB is the round ``codeverse3d.addons.select`` picks from the+ harness run under ``<bench_out>/<lang>/runs/<id>/`` (``deliverable/object.glb``, else+ that round's own ``artifacts/rNN/object.glb``, else — a run recorded before rounds kept+ their own — ``artifacts/object.glb``).3. Both GLBs go through the same pipeline as ``bench/_fixed_eval.FixedEvaluator``:``measure_glb`` → ``check_connectivity`` → ``render_glb`` (OBJECT_VIEWS, settingssize, contact sheet) → ``VlmJudge(static_object_v1, gemini-3.1-pro-preview,@@ -47,10 +48,11 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolfrom bench._fixed_eval import RUBRIC # noqa: E402from bench.h2h_scene import sign_test # noqa: E402from bench.run_bench import Battery, BenchPrompt # noqa: E402+from codeverse3d.addons import select # noqa: E402JUDGE_MODEL = "gemini:gemini-3.1-pro-preview"N_SAMPLES = 2-TERMINAL = {"passed", "plateau", "budget", "failed"} # RunStatus values a finished run can hold+IN_PROGRESS = {"planning", "generating", "refining"} # RunStatus values of a run that has not stopped_H2H_TAGS = {"h2h", "brilliana"}@@ -120,10 +122,19 @@ def our_side(runs_dir: Path, item_id: str) -> Side:side.minutes = round((t1 - t0).total_seconds() / 60, 2)except (KeyError, TypeError, ValueError):pass- if status not in TERMINAL or rec.get("best_round") is None:- side.error = f"run not finished (status={status!r}, best_round={rec.get('best_round')})"+ if not status or status in IN_PROGRESS:+ side.error = f"run not finished (status={status!r})"return side- for cand in (ws / "deliverable" / "object.glb", ws / "artifacts" / "object.glb"):+ try:+ picked = select.summarise(ws).picked_round+ except Exception as e: # noqa: BLE001 - one unreadable record must not kill the battery+ side.error = f"unreadable record: {e}"+ return side+ if picked is None:+ side.error = f"no judged round to hand over (status={status!r})"+ return side+ for cand in (ws / "deliverable" / "object.glb", ws / "artifacts" / f"r{picked:02d}" / "object.glb",+ ws / "artifacts" / "object.glb"):if cand.is_file():side.glb = str(cand)return sidediff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.pyindex 2c7d26c6..e30a00f8 100644--- a/eval/bench/h2h_scene.py+++ b/eval/bench/h2h_scene.py@@ -10,8 +10,8 @@ Protocol— authored cameras (``view_cam_*``) first, then the overview rig (``view_overview_*``),capped at ``MAX_VIEWS``; zone/probe frames are never shown. Their own judge score,wall time and cost come from ``run_report.json`` / ``trajectories/*/score.json``.-* OURS: the best round of ``bench/out/h2h_scene_v1/runs/<id>/`` (``record.best_round`` →- ``rounds/rNN.json`` renders): the ``t=0`` views the scene judge saw (per-view ``judge``+* OURS: the round ``codeverse3d.addons.select`` picks from ``bench/out/h2h_scene_v1/runs/<id>/``+ (``rounds/rNN.json`` renders): the ``t=0`` views the scene judge saw (per-view ``judge``flag, authored cameras first), capped at ``MAX_VIEWS`` — stills only, so both sides arejudged blind to animation in the same way.* Judge: ONE ``VlmJudge(rubric=scene_v1, gemini-3.1-pro-preview, n_samples=2)`` fed a@@ -42,6 +42,7 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolsys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)from bench.run_bench import Battery, BenchPrompt # noqa: E402+from codeverse3d.addons import select # noqa: E402from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402from codeverse3d.contracts.common import Language, Track # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402@@ -81,7 +82,7 @@ class OurSide(BaseModel):defect_penalty: float = 0.0harness_score: float | None = Nonerounds: int = 0- best_round: int = 0+ picked_round: int = 0cost_usd: float = 0.0minutes: float = 0.0n_views: int = 0@@ -134,8 +135,12 @@ def our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:if not rec_path.is_file():return Nonerec = json.loads(rec_path.read_text())- best = int(rec.get("best_round") or 0)- rnd_path = run_dir / "rounds" / f"r{best:02d}.json"+ try:+ summary = select.summarise(run_dir)+ except Exception: # noqa: BLE001 - an unreadable record is a run we cannot compare+ return None+ picked = summary.picked_round if summary.picked_round is not None else 0+ rnd_path = run_dir / "rounds" / f"r{picked:02d}.json"if not rnd_path.is_file():return Nonernd = json.loads(rnd_path.read_text())@@ -150,9 +155,9 @@ def our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:if minutes is None and rec.get("started_at") and rec.get("finished_at"):t0, t1 = (datetime.fromisoformat(rec[k]) for k in ("started_at", "finished_at"))minutes = (t1 - t0).total_seconds() / 60- meta = {"harness_score": rec.get("final_score"), "rounds": len(rec.get("rounds") or []), "best_round": best,+ meta = {"harness_score": summary.picked_score, "rounds": len(rec.get("rounds") or []), "picked_round": picked,"cost_usd": float((rec.get("total_usage") or {}).get("cost_usd") or 0.0), "minutes": round(minutes or 0.0, 2),- "status": str(rec.get("status") or "")}+ "status": summary.stop_reason}return [RenderView(name=v.name, path=v.path, mode="shaded", width=v.width, height=v.height) for v in chosen], meta@@ -206,7 +211,7 @@ def sheet(row_id: str, theirs: list[Path], ours: list[Path], out_png: Path, thumx += t.width + 4return canvas- a, b = strip(theirs, f"{row_id} — THEIRS (authored cams + overview)"), strip(ours, f"{row_id} — OURS (best round, t=0 judge views)")+ a, b = strip(theirs, f"{row_id} — THEIRS (authored cams + overview)"), strip(ours, f"{row_id} — OURS (picked round, t=0 judge views)")out = Image.new("RGB", (max(a.width, b.width), a.height + b.height + 6), (60, 60, 60))out.paste(a, (0, 0))out.paste(b, (0, a.height + 6))diff --git a/eval/bench/judge_calib_graphics.py b/eval/bench/judge_calib_graphics.pyindex 351f9c57..123bc606 100644--- a/eval/bench/judge_calib_graphics.py+++ b/eval/bench/judge_calib_graphics.py@@ -12,7 +12,7 @@ person sees → revise) never ran for graphics. This script is that loop's meas* ``eye.json``: ``{run_dir: score}`` — a person's 0–1 reading of the SAME sheets ("would acurator screenshot it / does it look like the thing"). The ground truth this calibrates to.-For every run and every rubric it rebuilds the JudgeInput the loop used (spec, the best round's+For every run and every rubric it rebuilds the JudgeInput the loop used (spec, the picked round'sRenderSet and gates; no planner acceptance items, so the rubric is measured on its own) andjudges it with a fresh ``VlmJudge`` (n samples, pro). Output per rubric: ``<rubric>.jsonl`` (onerow per run: overall, per-criterion, defects present, caps) and one ``summary.md`` with@@ -29,36 +29,38 @@ from concurrent.futures import ThreadPoolExecutorfrom pathlib import Pathfrom typing import Any+from codeverse3d.addons import selectfrom codeverse3d.addons.calibration import spearmanfrom codeverse3d.contracts.artifacts import GateReport, RenderSetfrom codeverse3d.contracts.spec import Specfrom codeverse3d.judges.base import JudgeInput, plan_digestfrom codeverse3d.judges.vlm_judge import VlmJudge+from codeverse3d.record.record import RecordErrorfrom codeverse3d.tracks.graphics import frame_stats_textfrom codeverse3d.workspace import WorkspaceOUT_ROOT = Path(__file__).resolve().parent / "out"-def best_round(run: Path) -> dict[str, Any] | None:- best: dict[str, Any] | None = None- for f in sorted(run.glob("rounds/r*.json")):- r = json.loads(f.read_text())- j = r.get("judgment") or {}- if j.get("overall") is not None and (best is None or j["overall"] > (best.get("judgment") or {}).get("overall", -1)):- best = r- return best+def picked_round(run: Path) -> dict[str, Any] | None:+ """The round ``codeverse3d.addons.select`` picks, as its rounds/rNN.json (None: nothing judged)."""+ try:+ idx = select.pick(run)+ except RecordError:+ return None+ path = run / "rounds" / f"r{idx:02d}.json" if idx is not None else None+ return json.loads(path.read_text()) if path is not None and path.is_file() else Nonedef judge_one(run: Path, rubric: str, model: str, n: int) -> dict[str, Any]:spec = Spec.model_validate(json.loads((run / "spec.json").read_text()))- rnd = best_round(run)+ rnd = picked_round(run)if rnd is None:return {"run": str(run), "error": "no judged round"}renders = RenderSet.model_validate(rnd["renders"])gates = [GateReport.model_validate(g) for g in rnd.get("gates") or []]# what the in-run judge also saw: the plan digest and the measured frame metrics (the metrics- # file is the LAST build's; it is only quoted when the best round is the last one, else the+ # file is the LAST build's; it is only quoted when the picked round is the last one, else the# judge is told so — without it, pro called four moving effects "static" in the first pass)ws = Workspace(run)plan_path = run / "plan.json"diff --git a/eval/bench/paired_compare.py b/eval/bench/paired_compare.pyindex 44476ddc..a657be1a 100644--- a/eval/bench/paired_compare.py+++ b/eval/bench/paired_compare.py@@ -137,8 +137,10 @@ def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_ahs.append(h.score)os_.append(o.score)st.deltas[p] = round(h.score - o.score, 4)- hp.append(bool(h.passed))- op.append(bool(o.passed))+ if h.passed is not None: # a bench-run row carries no pass/fail (2026-09-22): no rate+ hp.append(h.passed)+ if o.passed is not None:+ op.append(o.passed)hb.append(h.build_ok)ob.append(o.build_ok)st.n = len(hs)@@ -153,8 +155,8 @@ def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_ast.losses = sum(d < 0 for d in deltas)st.ties = st.n - st.wins - st.lossesst.sign_p = _sign_test(deltas)["sign_p"] # type: ignore[assignment]- st.pass_rate_harness = round(sum(hp) / st.n, 4)- st.pass_rate_oneshot = round(sum(op) / st.n, 4)+ st.pass_rate_harness = round(sum(hp) / len(hp), 4) if hp else None+ st.pass_rate_oneshot = round(sum(op) / len(op), 4) if op else Nonest.build_ok_harness = round(sum(hb) / st.n, 4)st.build_ok_oneshot = round(sum(ob) / st.n, 4)if st.n >= 2:@@ -257,7 +259,7 @@ def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:"""A ``bench run`` battery read as cells of one arm.``compare_backends`` writes one journal with an ``arm`` column; ``bench run`` writes a- directory per arm with ``id`` / ``score_final``. Two of those directories are a paired+ directory per arm with ``id`` / ``score_picked`` (``score_final`` before 2026-09-22). Two of those directories are a pairedcomparison — same prompts, one thing different — and this lets the statistics below(paired CI, exact sign test, the "unsupported when the interval crosses zero" rule) bethe same for both shapes rather than recomputed by hand.@@ -268,11 +270,13 @@ def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:continueraw = json.loads(line)# a bench-run row records no build flag: a cell with a verdict was built and judged,- # one without (score None, passed None) was not — so `build_ok` here means "judged",- # and `gen_cost_usd` is the run's WHOLE cost (plan + loop judge), as the field says+ # one without (score None) was not — so `build_ok` here means "judged", and+ # `gen_cost_usd` is the run's WHOLE cost (plan + loop judge), as the field says. A+ # row written before 2026-09-22 still has its old `passed`; a new one has none.+ score = raw.get("score_picked", raw.get("score_final"))rows.append(CellResult(prompt_id=raw["id"], arm=arm, tier=raw.get("tier", ""),- score=raw.get("score_final"), status=raw.get("status", ""),- passed=raw.get("passed"), build_ok=raw.get("passed") is not None,+ score=score, status=raw.get("status", ""),+ passed=raw.get("passed"), build_ok=score is not None,gen_cost_usd=float(raw.get("cost_usd") or 0.0), kind="harness"))return rowsdiff --git a/eval/bench/report.py b/eval/bench/report.pyindex bc22878b..35ef6373 100644--- a/eval/bench/report.py+++ b/eval/bench/report.py@@ -1,7 +1,8 @@"""Aggregate a bench run: per-tier / per-category stats → markdown + HTML gallery.``build_report(out_dir)`` reads ``results.jsonl`` (or results.json), aggregates-mean/median/pass-rate/cost per tier and category, writes ``report.md`` and+baseline / picked-round mean + median, the delta and cost per tier and category (no pass+rate: a run is not passed or failed since 2026-09-22), writes ``report.md`` and``report.html`` — the same self-contained page as ``3dcode gallery build --embed``(``codeverse3d.addons.gallery``), one section per tier, with the stats tables under thesummary strip.@@ -35,10 +36,9 @@ class GroupStats(BaseModel):n_evaluated: int = Field(default=0, description="rows that actually ran (n minus provider outages)")infra_failed: int = Field(default=0, description="rows dropped: the provider, not the model, failed")baseline_mean: float | None- final_mean: float | None- final_median: float | None+ picked_mean: float | None+ picked_median: float | Nonedelta_mean: float | None- pass_rate: float | Nonecost_mean: floatminutes_mean: floaterrors: int@@ -73,15 +73,13 @@ def _stats(group: str, rs: list[BenchItemResult]) -> GroupStats:# min/run 12.8x. It is also not an `error`: that column must keep meaning "a real# crash", or a storm and a bug read the same in the report.ev = [r for r in rs if r.status != "infra_failed"]- finals = [r.score_final for r in ev if r.score_final is not None]+ picked = [r.score_picked for r in ev if r.score_picked is not None]bases = [r.score_baseline for r in ev if r.score_baseline is not None]- deltas = [r.score_final - r.score_baseline for r in ev if r.score_final is not None and r.score_baseline is not None]- passed = [r.passed for r in ev if r.passed is not None]+ deltas = [r.score_picked - r.score_baseline for r in ev if r.score_picked is not None and r.score_baseline is not None]return GroupStats(- group=group, n=len(rs), n_scored=len(finals), n_evaluated=len(ev), infra_failed=len(rs) - len(ev),- baseline_mean=_mean(bases), final_mean=_mean(finals),- final_median=round(statistics.median(finals), 4) if finals else None, delta_mean=_mean(deltas),- pass_rate=round(sum(passed) / len(passed), 4) if passed else None,+ group=group, n=len(rs), n_scored=len(picked), n_evaluated=len(ev), infra_failed=len(rs) - len(ev),+ baseline_mean=_mean(bases), picked_mean=_mean(picked),+ picked_median=round(statistics.median(picked), 4) if picked else None, delta_mean=_mean(deltas),cost_mean=round(statistics.fmean([r.cost_usd for r in ev]), 4) if ev else 0.0,minutes_mean=round(statistics.fmean([r.minutes for r in ev]), 2) if ev else 0.0,errors=sum(1 for r in ev if r.status in ("error", "failed") or r.errors),@@ -98,11 +96,11 @@ def _grouped(rs: list[BenchItemResult], key: str) -> list[GroupStats]:def _md_table(title: str, stats: list[GroupStats]) -> str:lines = [f"### {title}", "",- "| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors | outage |",- "|---|---|---|---|---|---|---|---|---|---|---|---|"]+ "| group | n | scored | baseline | picked | median | Δ | $/run | min/run | errors | outage |",+ "|---|---|---|---|---|---|---|---|---|---|---|"]for s in stats:- lines.append(f"| {s.group} | {s.n} | {s.n_scored} | {_fmt(s.baseline_mean)} | {_fmt(s.final_mean)} | "- f"{_fmt(s.final_median)} | {_fmt(s.delta_mean, '+.3f')} | {_fmt(s.pass_rate, '.0%')} | "+ lines.append(f"| {s.group} | {s.n} | {s.n_scored} | {_fmt(s.baseline_mean)} | {_fmt(s.picked_mean)} | "+ f"{_fmt(s.picked_median)} | {_fmt(s.delta_mean, '+.3f')} | "f"{s.cost_mean:.2f} | {s.minutes_mean:.1f} | {s.errors} | {s.infra_failed} |")return "\n".join(lines) + "\n"@@ -124,8 +122,8 @@ def _entry_for(r: BenchItemResult) -> RunEntry:return entryreturn RunEntry(battery=r.tier, slug=r.id, path=r.workspace or "", state="ok", title=r.id,- generator=r.generator, judge=r.judge, score=r.score_final, baseline_score=r.score_baseline,- passed=r.passed, rounds=r.rounds, cost_usd=r.cost_usd, minutes=r.minutes, status=r.status,+ generator=r.generator, judge=r.judge, score=r.score_picked, baseline_score=r.score_baseline,+ picked_round=r.picked_round, rounds=r.rounds, cost_usd=r.cost_usd, minutes=r.minutes, status=r.status,error=r.errors,)@@ -143,11 +141,11 @@ def build_report(out_dir: Path | str, *, title: str | None = None) -> BenchRepormd.append(_md_table("overall", [rep.overall] if rep.overall else []))md.append(_md_table("by tier", rep.by_tier))md.append(_md_table("by category", rep.by_category))- md += ["### per prompt", "", "| id | tier | category | baseline | final | passed | rounds | $ | min | status |",+ md += ["### per prompt", "", "| id | tier | category | baseline | picked | round | rounds | $ | min | stop |","|---|---|---|---|---|---|---|---|---|---|"]for r in results:- md.append(f"| {r.id} | {r.tier} | {r.category} | {_fmt(r.score_baseline)} | {_fmt(r.score_final)} | "- f"{'-' if r.passed is None else ('yes' if r.passed else 'no')} | {r.rounds} | {r.cost_usd:.2f} | "+ md.append(f"| {r.id} | {r.tier} | {r.category} | {_fmt(r.score_baseline)} | {_fmt(r.score_picked)} | "+ f"{'-' if r.picked_round is None else f'r{r.picked_round:02d}'} | {r.rounds} | {r.cost_usd:.2f} | "f"{r.minutes:.1f} | {r.status}{' ⚠' if r.errors else ''} |")rep.markdown = "\n".join(md) + "\n"(out / "report.md").write_text(rep.markdown)@@ -158,12 +156,12 @@ def build_report(out_dir: Path | str, *, title: str | None = None) -> BenchRepordef _html_table(stats: list[GroupStats]) -> str:rows = "".join(- f"<tr><td>{html.escape(s.group)}</td><td>{s.n}</td><td>{_fmt(s.baseline_mean)}</td><td>{_fmt(s.final_mean)}</td>"- f"<td>{_fmt(s.delta_mean, '+.3f')}</td><td>{_fmt(s.pass_rate, '.0%')}</td><td>{s.cost_mean:.2f}</td>"+ f"<tr><td>{html.escape(s.group)}</td><td>{s.n}</td><td>{_fmt(s.baseline_mean)}</td><td>{_fmt(s.picked_mean)}</td>"+ f"<td>{_fmt(s.delta_mean, '+.3f')}</td><td>{s.cost_mean:.2f}</td>"f"<td>{s.minutes_mean:.1f}</td><td>{s.errors}</td><td>{s.infra_failed}</td></tr>"for s in stats)- return ("<table><tr><th>group</th><th>n</th><th>baseline</th><th>final</th><th>Δ</th><th>pass</th><th>$/run</th>"+ return ("<table><tr><th>group</th><th>n</th><th>baseline</th><th>picked</th><th>Δ</th><th>$/run</th>"f"<th>min/run</th><th>errors</th><th>outage</th></tr>{rows}</table>")diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.pyindex 74cf555c..18c659b8 100644--- a/eval/bench/run_bench.py+++ b/eval/bench/run_bench.py@@ -14,13 +14,16 @@ Each prompt runs inside its own ``codeverse3d.cost.run_ledger``, exactly like a``3dcode make``: the batteries are where most runs come from, so without it thepriced per-call rows of a whole battery went to the per-process fallback log and``3dcode cost --runs-dir <out>/runs`` had to reconstruct them from trajectories.-The binding is context-local, so ``--parallel N`` keeps N ledgers apart.+The binding is context-local, so ``--parallel N`` keeps N ledgers apart. And like+``3dcode make``, the round ``codeverse3d.addons.select`` picks is packaged after the run+(``deliverable/`` + ``selection.json``); a result row is baseline → picked, no pass/fail."""from __future__ import annotationsimport csvimport json+import loggingimport timeimport tracebackfrom collections.abc import Callable, Sequence@@ -30,10 +33,11 @@ from pathlib import Pathfrom typing import Anyimport yaml-from pydantic import BaseModel, Field+from pydantic import AliasChoices, BaseModel, Fieldfrom bench._infra import is_infra_failurefrom bench._jsonl import read_jsonl, seal_for_append+from codeverse3d.addons import selectfrom codeverse3d.config import get_settingsfrom codeverse3d.contracts.common import Backends, Budget, Language, Trackfrom codeverse3d.contracts.run import RunRecord@@ -42,7 +46,9 @@ from codeverse3d.cost import run_ledgerfrom codeverse3d.proc import exclusivefrom codeverse3d.workspace import Workspace-RESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_final", "passed", "rounds", "cost_usd",+log = logging.getLogger(__name__)++RESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_picked", "picked_round", "rounds", "cost_usd","minutes", "status", "errors", "workspace", "generator", "judge")@@ -81,8 +87,9 @@ class BenchItemResult(BaseModel):tier: strcategory: str = ""score_baseline: float | None = None- score_final: float | None = None- passed: bool | None = None+ score_picked: float | None = Field(default=None, validation_alias=AliasChoices("score_picked", "score_final"),+ description="the picked round's score (rows before 2026-09-22: score_final)")+ picked_round: int | None = Nonerounds: int = 0cost_usd: float = 0.0minutes: float = 0.0@@ -168,8 +175,8 @@ def spec_for(battery: Battery, item: BenchPrompt, opts: BenchOptions) -> Spec:def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Workspace) -> BenchItemResult:- best = next((r for r in rec.rounds if r.index == rec.best_round), None)- # A run whose rounds all lost their verdict still reports a normal status (`plateau`+ s = select.summarise(ws.root, record=rec)+ # A run whose rounds all lost their verdict still reports a normal status (`max_rounds`# after three unjudged rounds), so a whole ARM can read as healthy and score nothing —# what a worktree without node_modules did on 2026-09-04: render_glb died, every round# skipped the judge, ten cells came back with score=None and status=plateau. Say it@@ -179,10 +186,10 @@ def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Woif unjudged and not error:error = f"no verdict in any of {len(rec.rounds)} round(s) — the judge was skipped every time"return BenchItemResult(- id=item.id, tier=item.tier, category=item.category, score_baseline=rec.baseline_score,- score_final=rec.final_score, passed=None if best is None or best.judgment is None else best.judgment.passed,+ id=item.id, tier=item.tier, category=item.category, score_baseline=s.baseline_score,+ score_picked=s.picked_score, picked_round=s.picked_round,rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd, minutes=round(minutes, 2),- status=rec.status.value, errors=error, workspace=str(ws.root),+ status=s.stop_reason, errors=error, workspace=str(ws.root),generator=rec.spec.backends.generator, judge=rec.spec.backends.judge,)@@ -228,6 +235,17 @@ def default_run_track(spec: Spec, ws: Workspace, resume: bool) -> RunRecord:return get_track(spec.track).run(spec, ws, resume=resume)+def hand_over(ws: Workspace, rec: RunRecord) -> None:+ """What ``3dcode make`` does after a run: package the round ``select`` picks (by score).+ A packaging failure costs the hand-over, never the scored row."""+ try:+ idx = select.pick(ws.root, record=rec)+ if idx is not None:+ select.package(ws.root, idx, method="score")+ except Exception as e: # noqa: BLE001 - the run and its scores stand without deliverable/+ log.warning("hand-over of %s failed: %s", ws.root, e)++def run_battery(battery_path: Path | str, out_dir: Path | str, opts: BenchOptions | None = None, *,run_fn: RunFn | None = None, on_result: Callable[[BenchItemResult], None] | None = None,@@ -270,6 +288,7 @@ def run_battery(# the run mutex is what keeps two cells off the same workspacewith exclusive(ws.root, what=f"bench {item.id}"), run_ledger(ws.root, run=item.id):rec = run_fn(spec, ws, resume)+ hand_over(ws, rec)except Exception as e: # one failing prompt must not kill the battery# A provider outage is not a result: an hour spent retrying a 503 is not model# latency and not a crash. compare_backends has classified this sincediff --git a/eval/tests/conftest.py b/eval/tests/conftest.pyindex 1cc37b90..e52dda6b 100644--- a/eval/tests/conftest.py+++ b/eval/tests/conftest.py@@ -41,7 +41,7 @@ from codeverse3d.contracts.artifacts import ( # noqa: E402Severity,)from codeverse3d.contracts.common import Usage # noqa: E402-from codeverse3d.contracts.run import RunRecord, RunStatus # noqa: E402+from codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402from codeverse3d.workspace import Workspace # noqa: E402@@ -131,7 +131,10 @@ def fake_run_track(score: float = 0.8):(ws.root / MODEL_FILE).write_text(GOOD.format(score=score))(ws.src / "parts").mkdir(exist_ok=True)(ws.src / "parts" / "legs.py").write_text("LEGS = 3\n")- rec = RunRecord(spec=spec, workspace=str(ws.root), status=RunStatus.MAX_ROUNDS, final_score=score - 0.1,+ loop_verdict = Judgment(rubric="static_object_v1", scores={}, overall=round(score - 0.1, 4), passed=False)+ rec = RunRecord(spec=spec, workspace=str(ws.root), status=RunStatus.MAX_ROUNDS,+ rounds=[RoundRecord(index=0, kind="baseline", judgment=loop_verdict,+ build=BuildResult(ok=True, language="blender"))],total_usage=Usage(cost_usd=0.9, tool_calls=12))ws.write_json(ws.record_path, rec)return recdiff --git a/eval/tests/test_bench.py b/eval/tests/test_bench.pyindex 70c55a10..7f48d846 100644--- a/eval/tests/test_bench.py+++ b/eval/tests/test_bench.py@@ -59,7 +59,10 @@ def test_run_battery_resume_and_report(tmp_path: Path):res = run_battery(battery, out, opts, run_fn=_fake_run_fn(scores, fail_ids={"veh_easy_toy_car"}))assert len(res) == 4by_id = {r.id: r for r in res}- assert by_id["furn_easy_stool"].score_final == 0.85 and by_id["furn_easy_stool"].passed is True+ assert by_id["furn_easy_stool"].score_picked == 0.85 and by_id["furn_easy_stool"].picked_round == 1+ # like `3dcode make`, the battery hands over the picked round+ assert (out / "runs" / "furn_easy_stool" / "deliverable" / "manifest.json").is_file()+ assert json.loads((out / "runs" / "furn_easy_stool" / "selection.json").read_text())["round"] == 1assert by_id["veh_easy_toy_car"].status == "error" and "boom" in by_id["veh_easy_toy_car"].errorsassert (out / "results.csv").is_file() and len(load_results(out)) == 4assert (out / "runs" / "furn_easy_stool" / "spec.json").is_file()@@ -77,7 +80,8 @@ def test_run_battery_resume_and_report(tmp_path: Path):rep = build_report(out)assert rep.n == 5 and (out / "report.md").is_file() and (out / "report.html").is_file()tiers = {g.group: g for g in rep.by_tier}- assert tiers["easy"].n == 2 and tiers["easy"].final_mean == pytest.approx(0.85) # toy car errored → unscored+ assert tiers["easy"].n == 2 and tiers["easy"].picked_mean == pytest.approx(0.85) # toy car errored → unscored+ assert "pass" not in rep.markdown.split("### per prompt")[0], "no pass rate: a run is not passed or failed"assert rep.overall is not None and rep.overall.errors == 1assert "| furn_easy_stool | easy |" in rep.markdownpage = (out / "report.html").read_text()@@ -150,8 +154,8 @@ def test_a_provider_outage_is_not_model_latency_and_not_an_error():from bench.report import _statsfrom bench.run_bench import BenchItemResult- clear = [BenchItemResult(id=f"p{i}", tier="easy", score_baseline=0.5, score_final=0.8,- passed=True, minutes=1.0, cost_usd=0.10, status="passed") for i in range(4)]+ clear = [BenchItemResult(id=f"p{i}", tier="easy", score_baseline=0.5, score_picked=0.8,+ minutes=1.0, cost_usd=0.10, status="max_rounds") for i in range(4)]storm = BenchItemResult(id="p_storm", tier="easy", minutes=60.0, cost_usd=0.03, status="infra_failed",errors="ModelError: Gemini API error 503: The model is overloaded. Please try again later.")@@ -164,7 +168,9 @@ def test_a_provider_outage_is_not_model_latency_and_not_an_error():assert with_storm.n == 5 and with_storm.n_evaluated == 4 and with_storm.infra_failed == 1assert with_storm.errors == 0, "a provider outage must not read as a crash"# the scored rates were already safe; they must stay so- assert with_storm.final_mean == base.final_mean and with_storm.pass_rate == base.pass_rate+ assert with_storm.picked_mean == base.picked_mean and with_storm.delta_mean == base.delta_mean+ # a row written before 2026-09-22 still reads: its score_final is the picked score+ assert BenchItemResult.model_validate({"id": "old", "tier": "easy", "score_final": 0.7, "passed": True}).score_picked == 0.7def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):diff --git a/eval/tests/test_bench_shared.py b/eval/tests/test_bench_shared.pyindex e7f7e41d..6df35ef7 100644--- a/eval/tests/test_bench_shared.py+++ b/eval/tests/test_bench_shared.py@@ -78,27 +78,32 @@ def test_the_harness_arm_looks_for_each_languages_own_entry_file():assert entry_of(SimpleNamespace(language=Language.GLSL_SHADER)) == "src/shader.frag"-def test_a_run_that_scored_nothing_says_so_in_its_row() -> None:+def test_a_run_that_scored_nothing_says_so_in_its_row(tmp_path: Path) -> None:"""A worktree without node_modules made render_glb die, every round skip the judge andten cells come back `status=plateau, score=None` — an arm that reads as healthy andmeasures nothing (2026-09-04). The row carries the reason now."""from bench.run_bench import result_from_record+ from codeverse3d.contracts.artifacts import Judgment+ from codeverse3d.contracts.common import Language, Track, Usage+ from codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus+ from codeverse3d.contracts.spec import Spec+ from codeverse3d.workspace import Workspaceitem = SimpleNamespace(id="cpl_umbrella", tier="hard", category="mechanism")- rounds = [SimpleNamespace(index=i, judgment=None) for i in range(3)]- rec = SimpleNamespace(- rounds=rounds, best_round=0, baseline_score=None, final_score=None, error="",- total_usage=SimpleNamespace(cost_usd=1.76), status=SimpleNamespace(value="plateau"),- spec=SimpleNamespace(backends=SimpleNamespace(generator="g", judge="j")))- ws = SimpleNamespace(root=Path("/tmp/ws"))+ spec = Spec(id="cpl_umbrella", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="an umbrella")+ rounds = [RoundRecord(index=i, kind="refine") for i in range(3)]+ rec = RunRecord(spec=spec, workspace=str(tmp_path), status=RunStatus.MAX_ROUNDS, rounds=rounds,+ total_usage=Usage(cost_usd=1.76))+ ws = Workspace(tmp_path)row = result_from_record(item, rec, 16.0, ws)- assert row.score_final is None and row.status == "plateau"+ assert row.score_picked is None and row.picked_round is None and row.status == "max_rounds"assert "no verdict in any of 3 round(s)" in row.errors- judged = SimpleNamespace(index=0, judgment=SimpleNamespace(passed=True))+ judged = RoundRecord(index=0, kind="baseline", judgment=Judgment(rubric="r", scores={}, overall=0.6, passed=False))rec.rounds = [judged, rounds[1]]- assert result_from_record(item, rec, 16.0, ws).errors == "" # one verdict is enough+ row = result_from_record(item, rec, 16.0, ws)+ assert row.errors == "" and (row.score_picked, row.picked_round) == (0.6, 0) # one verdict is enoughdef test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.pyindex 6c000213..bc9ab72f 100644--- a/eval/tests/test_infra_failures.py+++ b/eval/tests/test_infra_failures.py@@ -313,7 +313,7 @@ def test_the_ab_viewer_refuses_to_call_a_winner_it_cannot_support():from bench.ab_view import Run, verdictdef arm(name, *scores):- return [Run(slug=f"{name}{i}", arm=name, final=s, status="passed") for i, s in enumerate(scores)]+ return [Run(slug=f"{name}{i}", arm=name, picked=s, status="max_rounds") for i, s in enumerate(scores)]cases = [((0.7,), (0.9,), "Inconclusive"),@@ -326,7 +326,7 @@ def test_the_ab_viewer_refuses_to_call_a_winner_it_cannot_support():assert head.startswith(expected), head# a run that never scored must not be counted as an observation- a = arm("a", 0.5, 0.5) + [Run(slug="a9", arm="a", final=None, status="failed")]+ a = arm("a", 0.5, 0.5) + [Run(slug="a9", arm="a", picked=None, status="failed")]head, why = verdict(a, arm("b", 0.5))assert head.startswith("Inconclusive") and "1 of 1" in whydiff --git a/harness/codeverse3d/addons/gallery/index.py b/harness/codeverse3d/addons/gallery/index.pyindex ddcda0e2..2cf40feb 100644--- a/harness/codeverse3d/addons/gallery/index.py+++ b/harness/codeverse3d/addons/gallery/index.py@@ -133,8 +133,16 @@ def _articulation_sheet(ws: Workspace, picked: int | None) -> str:def entry_links(ws: Workspace, rec: RunRecord | None, picked: int | None) -> list[RunLink]:- """The working links of a card: workspace, record, code, sheet, GLB, track extras."""+ """The working links of a card: workspace, record, code, sheet, GLB, track extras — the+ artifact files of the handed-over round (``deliverable/``), else of the picked round+ (``artifacts/rNN/``), else whatever ``artifacts/`` holds (a run recorded before rounds+ kept their own)."""run = ws.root+ kept = [f"artifacts/r{picked:02d}"] if picked is not None else []++ def artifact(name: str, *extra: str) -> str:+ return _first_file(run, f"deliverable/{name}", *(f"{k}/{name}" for k in kept), f"artifacts/{name}", *extra)+links = [RunLink(label="workspace", rel="", kind="dir"),RunLink(label="record.json", rel="record.json")]for name in ("spec.json", "plan.json"):@@ -145,7 +153,7 @@ def entry_links(ws: Workspace, rec: RunRecord | None, picked: int | None) -> lissheet = picked_sheet(ws, rec, picked) if rec is not None else ""if sheet:links.append(RunLink(label="sheet", rel=sheet))- glb = _first_file(run, "deliverable/object.glb", "artifacts/object.glb")+ glb = artifact("object.glb")if glb:links.append(RunLink(label="glb", rel=glb, kind="viewer"))# the textured GLB is a link only when the texture pass SHIPPED — a stray file@@ -155,13 +163,13 @@ def entry_links(ws: Workspace, rec: RunRecord | None, picked: int | None) -> listextured = _first_file(run, "deliverable/object_textured.glb", "artifacts/object_textured.glb")if textured:links.append(RunLink(label="textured glb", rel=textured, kind="viewer"))- urdf = _first_file(run, "deliverable/robot.urdf", "artifacts/robot.urdf", "src/robot.urdf")+ urdf = artifact("robot.urdf", "src/robot.urdf")if urdf:links.append(RunLink(label="robot.urdf", rel=urdf))art = _articulation_sheet(ws, picked)if art:links.append(RunLink(label="articulation", rel=art))- gif = _first_file(run, "deliverable/preview.gif", "artifacts/preview.gif")+ gif = artifact("preview.gif")if gif:links.append(RunLink(label="preview.gif", rel=gif))if (run / "artifacts" / "frames").is_dir():diff --git a/harness/codeverse3d/cli/main.py b/harness/codeverse3d/cli/main.pyindex a971a556..36e85915 100644--- a/harness/codeverse3d/cli/main.py+++ b/harness/codeverse3d/cli/main.py@@ -124,7 +124,7 @@ def run_cmd(console.print(f"battery={battery} out={out_dir} generator={generator or 'default'} judge={judge or 'default'}")def _on(res) -> None:- console.print(f" [{res.status}] {res.id}: baseline={res.score_baseline} final={res.score_final} "+ console.print(f" [{res.status}] {res.id}: baseline={res.score_baseline} picked={res.score_picked} "f"rounds={res.rounds} ${res.cost_usd:.2f} {res.minutes:.1f}min" + (f" [red]{res.errors[:80]}[/red]" if res.errors else ""))results = run_bench.run_battery(battery, out_dir, opts, on_result=_on)diff --git a/harness/tests/gallery/test_index.py b/harness/tests/gallery/test_index.pyindex 9285ee03..c53a7cde 100644--- a/harness/tests/gallery/test_index.py+++ b/harness/tests/gallery/test_index.py@@ -41,6 +41,22 @@ def test_entry_fields_and_links(gallery_tree: dict[str, Path]):assert entry.key == "runs/wooden_chair_ab12cd34"+def test_a_card_links_the_picked_rounds_own_glb(tmp_path: Path):+ """The tree ends at the LAST round (2026-09-22), so artifacts/object.glb is not the card's+ round: until a round is handed over, the card links the picked round's artifacts/rNN/ copy."""+ from codeverse3d.addons import select+ from tests.flywheel_cli.conftest import make_fake_run++ ws, _ = make_fake_run(tmp_path / "runs", "kept_rounds", scores=(0.9, 0.6)) # the pick is r00+ (ws.round_artifacts(0)).mkdir(parents=True)+ (ws.round_artifacts(0) / "object.glb").write_bytes(b"glTF r00")+ links = {ln.label: ln.rel for ln in entry_for_dir("runs", ws.root).links}+ assert links["glb"] == "artifacts/r00/object.glb"+ select.package(ws.root, 0)+ links = {ln.label: ln.rel for ln in entry_for_dir("runs", ws.root).links}+ assert links["glb"] == "deliverable/object.glb"++def test_corrupt_record_does_not_raise(tmp_path: Path):run = tmp_path / "runs" / "boom"run.mkdir(parents=True)