Skip to content

Official Codebase of 3DCodeVerse

Python

eval: bench readers read select — baseline → picked, the harness arm delivers its picked round

gaoypeng599c2a486650
diff --git a/eval/bench/ab_gate_rates.py b/eval/bench/ab_gate_rates.py
index 444d2471..9bb25137 100644
--- a/eval/bench/ab_gate_rates.py
+++ b/eval/bench/ab_gate_rates.py
@@ -2,7 +2,8 @@
The judged score carries the planner's variance (docs/EVAL.md 8.1); these do not carry the
judge's. They are what c3d-part-contact and c3d-bbox-contract are FOR, so they are the
-primary readout and the score is the second.
+primary readout and the score is the second. Both are read off the round
+``codeverse3d.addons.select`` picks (the last round when nothing was judged).
"""
from __future__ import annotations
@@ -11,6 +12,11 @@ import re
import sys
from pathlib import Path
+for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
+ sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+
+from codeverse3d.addons import select # noqa: E402
+
if len(sys.argv) < 2:
raise SystemExit("usage: python bench/ab_gate_rates.py bench/out/<ab-run>")
OUT = Path(sys.argv[1])
@@ -21,10 +27,11 @@ def cell_stats(record: Path) -> dict | None:
rounds = rec.get("rounds") or []
if not rounds:
return None
- last = rounds[-1]
+ summary = select.summarise(record.parent)
+ rnd = next((r for r in rounds if r.get("index") == summary.picked_round), rounds[-1])
pairs = floats = islands = contract_n = 0
worst = 0.0
- for g in last.get("gates") or []:
+ for g in rnd.get("gates") or []:
for f in g.get("findings") or []:
sev, msg = str(f.get("severity", "")), str(f.get("message", ""))
if sev == "info":
@@ -42,7 +49,7 @@ def cell_stats(record: Path) -> dict | None:
contract_n += 1
return {"pairs": pairs, "worst_depth_mm": worst, "floating": floats,
"islands": islands, "contract_findings": contract_n,
- "score": rec.get("final_score"), "status": rec.get("status")}
+ "score": summary.picked_score, "status": summary.stop_reason}
rows: dict[str, dict[str, dict]] = {}
for arm in ("control", "variant"):
diff --git a/eval/bench/ab_view.py b/eval/bench/ab_view.py
index 3461837b..21715636 100644
--- a/eval/bench/ab_view.py
+++ b/eval/bench/ab_view.py
@@ -23,9 +23,15 @@ import html
import io
import json
import statistics
+import sys
from dataclasses import dataclass, field
from pathlib import Path
+for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
+ sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+
+from codeverse3d.addons import select # noqa: E402
+
SHEET_W = 1100
@@ -35,7 +41,7 @@ class Run:
arm: str
brief: str = ""
status: str = "?"
- final: float | None = None
+ picked: float | None = None
baseline: float | None = None
rounds: int = 0
usd: float = 0.0
@@ -73,8 +79,11 @@ def load_run(run_dir: Path, arm: str) -> Run:
d = json.loads(rec.read_text())
b = (d.get("extra") or {}).get("budget") or {}
r.brief = str((d.get("spec") or {}).get("prompt") or "")
- r.status = str(d.get("status") or "?")
- r.final, r.baseline = d.get("final_score"), d.get("baseline_score")
+ try: # the round addons/select picks; a record it cannot read keeps only its status
+ s = select.summarise(run_dir)
+ r.status, r.picked, r.baseline = s.stop_reason or "?", s.picked_score, s.baseline_score
+ except Exception: # noqa: BLE001 - one unreadable record must not blank the page
+ r.status = str(d.get("status") or "?")
r.rounds = len(d.get("rounds") or [])
r.usd, r.minutes = round(float(b.get("spent_usd") or 0), 2), round(float(b.get("elapsed_min") or 0))
r.error = str(d.get("error") or "")[:300]
@@ -97,8 +106,8 @@ MIN_RUNS_PER_ARM = 3
def verdict(a: list[Run], b: list[Run]) -> tuple[str, str]:
"""(headline, why) — refuses to call a winner the sample cannot support."""
- sa = [x.final for x in a if x.final is not None]
- sb = [x.final for x in b if x.final is not None]
+ sa = [x.picked for x in a if x.picked is not None]
+ sb = [x.picked for x in b if x.picked is not None]
if len(sa) < MIN_RUNS_PER_ARM or len(sb) < MIN_RUNS_PER_ARM:
return ("Inconclusive — not enough scored runs",
f"arm A scored {len(sa)} of {len(a)} runs, arm B {len(sb)} of {len(b)}; this needs "
@@ -122,7 +131,7 @@ def _num(v: object) -> str:
def _stats(rs: list[Run]) -> str:
- sc = [x.final for x in rs if x.final is not None]
+ sc = [x.picked for x in rs if x.picked is not None]
if not sc:
return "no scored run"
return (f"n={len(sc)}/{len(rs)} · mean {statistics.mean(sc):.3f} · "
@@ -135,7 +144,7 @@ def _panel(r: Run | None, side: str) -> str:
page can keep them hidden until the reader has actually looked."""
if r is None:
return f'<div class="panel empty" data-side="{side}"><p>this arm has no run for this brief</p></div>'
- tone = {"passed": "good", "failed": "bad", "budget": "warn"}.get(r.status, "warn")
+ tone = {"max_rounds": "good", "failed": "bad"}.get(r.status, "warn") # why it stopped, not a verdict
img = (f'<button class="sheet" data-full="{r.sheet}" '
f'aria-label="enlarge {html.escape(r.slug)}"><img src="{r.sheet}" '
f'alt="rendered frames for {html.escape(r.slug)}" loading="lazy"></button>'
@@ -147,7 +156,7 @@ def _panel(r: Run | None, side: str) -> str:
{img}
<div class="nums">
<span class="pill {tone}">{html.escape(r.status)}</span>
- <span class="n"><b>{_num(r.final)}</b> score</span>
+ <span class="n"><b>{_num(r.picked)}</b> score</span>
<span class="n">{r.rounds} rounds</span>
<span class="n">${r.usd:.2f}</span>
<span class="n">{r.minutes} min</span>
@@ -158,8 +167,8 @@ def _panel(r: Run | None, side: str) -> str:
def _pair(i: int, brief: str, a: Run | None, b: Run | None, flip: bool) -> str:
left, right = (b, a) if flip else (a, b)
- da = "" if a is None or a.final is None else f'{a.final:.3f}'
- db = "" if b is None or b.final is None else f'{b.final:.3f}'
+ da = "" if a is None or a.picked is None else f'{a.picked:.3f}'
+ db = "" if b is None or b.picked is None else f'{b.picked:.3f}'
return f"""<section class="pair" id="p{i}" data-i="{i}" data-flip="{'1' if flip else '0'}"
data-a="{da}" data-b="{db}">
<header class="pairhead">
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.py
index 58671c16..9965abf6 100644
--- a/eval/bench/compare_backends.py
+++ b/eval/bench/compare_backends.py
@@ -80,8 +80,10 @@ from bench.run_bench import ( # noqa: E402
archive_attempt,
build_spec,
default_run_track,
+ hand_over,
select_prompts,
)
+from codeverse3d.addons import select # noqa: E402
from codeverse3d.config import get_settings # noqa: E402
from codeverse3d.contracts.artifacts import RenderSet # noqa: E402
from codeverse3d.contracts.common import ENTRY_FILE, Track # noqa: E402
@@ -265,20 +267,27 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOption
with (exclusive(run_ws.root, what=f"compare {spec.id}:{run_ws.root.parent.name}"),
run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}")):
rec = deps.run_track(spec, run_ws, resume)
+ hand_over(run_ws, rec) # what `3dcode make` delivers: the picked round, packaged
+ summary = select.summarise(run_ws.root, record=rec)
res.gen_cost_usd = rec.total_usage.cost_usd
res.tool_calls = rec.total_usage.tool_calls
- res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), rec.final_score
- res.harness_stop_reason = str(rec.extra.get("stop_reason") or "")
+ res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), summary.picked_score
+ res.harness_stop_reason = summary.stop_reason
res.harness_aborted_rounds = len(rec.extra.get("aborted_rounds") or [])
- if not (run_ws.root / entry_of(spec)).is_file():
- res.error = f"harness run produced no {entry_of(spec)} (status {rec.status.value}: {rec.error})"
+ # the DELIVERED code is the picked round's deliverable/ (src/ + public/ at its commit);
+ # a run with no judged round handed nothing over, and its working tree — the last
+ # round — is what it produced
+ delivered = run_ws.deliverable if (run_ws.deliverable / entry_of(spec)).is_file() else run_ws.root
+ if not (delivered / entry_of(spec)).is_file():
+ res.error = f"harness run produced no {entry_of(spec)} (stop {summary.stop_reason}: {rec.error})"
return
# the whole src/ tree: agents may split helpers into src/parts/*.py (the build wrapper puts src/ on sys.path)
- shutil.copytree(run_ws.src, eval_ws.src, dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
+ shutil.copytree(delivered / "src", eval_ws.src, dirs_exist_ok=True,
+ ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
# a scene's Blender heroes live in public/assets/*.glb, preloaded by src/scene.js over http:
# without them every hero clone is a 404 and the fixed evaluation judges an empty zone
- if spec.track is Track.SCENE and (run_ws.root / "public").is_dir():
- shutil.copytree(run_ws.root / "public", eval_ws.root / "public", dirs_exist_ok=True)
+ if spec.track is Track.SCENE and (delivered / "public").is_dir():
+ shutil.copytree(delivered / "public", eval_ws.root / "public", dirs_exist_ok=True)
def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptions,
diff --git a/eval/bench/complexity_report.py b/eval/bench/complexity_report.py
index cba2b44c..00eb1dfb 100644
--- a/eval/bench/complexity_report.py
+++ b/eval/bench/complexity_report.py
@@ -11,10 +11,12 @@ or down, and what does a complexity point cost?**
python bench/complexity_report.py bench/out/complexity_v3 --battery bench/prompts/complexity_v3.yaml
A "root" is a battery directory (``<battery>/runs/<slug>``), a runs directory,
-or a single run directory — all three are detected. The complexity vector is
-read from the run's recorded measurement when present and recomputed from
-``artifacts/object.glb`` otherwise, so the whole historic corpus is usable.
-Rows without a judgment or without geometry are skipped and counted.
+or a single run directory — all three are detected. A run is the round
+``codeverse3d.addons.select`` picks (a record it cannot read: the last round). The
+complexity vector is read from that round's recorded measurement when present and
+recomputed from its GLB (``artifacts/rNN/object.glb``, else ``artifacts/object.glb``)
+otherwise, so the whole historic corpus is usable. Rows without a judgment or without
+geometry are skipped and counted. There is no pass rate: a run is not passed or failed.
"""
from __future__ import annotations
@@ -31,6 +33,7 @@ from typing import Any
for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resolve().parents[1]):
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
+from codeverse3d.addons import select # noqa: E402
from codeverse3d.addons.calibration import _ranks # noqa: E402 — tie-averaged ranks, one copy
from codeverse3d.contracts.run import RunId # noqa: E402
from codeverse3d.proc import read_json_or_none # noqa: E402
@@ -67,21 +70,22 @@ class Row(dict):
_read_json = read_json_or_none
-def _vector_for(run: Path, best: dict[str, Any] | None) -> ComplexityVector | None:
+def _vector_for(run: Path, rnd: dict[str, Any] | None) -> ComplexityVector | None:
"""The run's recorded complexity vector, else one recomputed from its GLB.
The historic corpus predates the vector, so recomputing is the normal path
for anything recorded before 2026-08-24 — that is what makes the whole
corpus comparable.
"""
- for meas in (best or {}).get("measurement"), (_read_json(run / "artifacts" / "measurement.json") or {}):
+ for meas in (rnd or {}).get("measurement"), (_read_json(run / "artifacts" / "measurement.json") or {}):
block = ((meas or {}).get("extra") or {}).get("complexity")
if isinstance(block, dict):
try:
return ComplexityVector.model_validate(block)
except Exception: # noqa: BLE001 - an older/foreign block must not stop the scan
pass
- glb = run / "artifacts" / "object.glb"
+ kept = run / "artifacts" / f"r{int((rnd or {}).get('index') or 0):02d}" / "object.glb"
+ glb = kept if kept.is_file() else run / "artifacts" / "object.glb"
if not glb.is_file():
return None
from codeverse3d.spatial.complexity import complexity_of_glb
@@ -92,15 +96,15 @@ def _vector_for(run: Path, best: dict[str, Any] | None) -> ComplexityVector | No
return None
-def _best_round(record: dict[str, Any]) -> dict[str, Any] | None:
+def _picked_round(run: Path, record: dict[str, Any]) -> dict[str, Any] | None:
+ """The round ``addons/select`` picks, as its raw dict; the last round when the record
+ will not load (the scan reads the historic corpus as-is)."""
rounds = record.get("rounds") or []
- if not rounds:
- return None
- idx = record.get("best_round")
- for r in rounds:
- if r.get("index") == idx:
- return r
- return rounds[-1]
+ try:
+ idx = select.summarise(run).picked_round
+ except Exception: # noqa: BLE001 - one unreadable record must not stop the scan
+ idx = None
+ return next((r for r in rounds if r.get("index") == idx), rounds[-1] if rounds else None)
def _minutes(record: dict[str, Any]) -> float:
@@ -114,10 +118,10 @@ def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:
record = _read_json(run / "record.json")
if not record:
return None
- best = _best_round(record)
- judgment = (best or {}).get("judgment") or {}
+ rnd = _picked_round(run, record)
+ judgment = (rnd or {}).get("judgment") or {}
scores = judgment.get("scores") or {}
- vec = _vector_for(run, best)
+ vec = _vector_for(run, rnd)
if vec is None or not scores:
return None
plan_parts = len((record.get("plan") or {}).get("parts") or [])
@@ -132,9 +136,8 @@ def row_for(run: Path, battery: str, slug: str | None = None) -> Row | None:
plan_parts=plan_parts,
parts_per_plan_part=round(vec.part_count / plan_parts, 3) if plan_parts else None,
gate_errors=sum(len([f for f in (g.get("findings") or []) if f.get("severity") == "error"])
- for g in (best or {}).get("gates") or []),
+ for g in (rnd or {}).get("gates") or []),
overall=judgment.get("overall"),
- passed=judgment.get("passed"),
issues=len(judgment.get("issues") or []),
cost_usd=round(float((record.get("total_usage") or {}).get("cost_usd") or 0.0), 4),
minutes=_minutes(record),
@@ -238,18 +241,17 @@ def _mean(vals: Iterable[float | None]) -> float | None:
def scatter_table(rows: Sequence[Row]) -> str:
"""Score vs complexity, bucketed by index — the scatter as a text table."""
lines = [
- "| complexity band | n | mean index | mean overall | pass % | detail | struct | fit | craft | $/run | min |",
- "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|",
+ "| complexity band | n | mean index | mean overall | detail | struct | fit | craft | $/run | min |",
+ "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|",
]
for lo, hi in zip(BUCKETS, BUCKETS[1:], strict=False):
b = [r for r in rows if isinstance(r.get("index"), (int, float)) and lo <= r["index"] < hi]
if not b:
continue
- npass = sum(1 for r in b if r.get("passed"))
lines.append(
f"| {band_of((lo + hi) / 2)} {lo:.2f}–{min(hi, 1.0):.2f} | {len(b)} |"
f"{_fmt(_mean(r['index'] for r in b))} |{_fmt(_mean(r['overall'] for r in b))} |"
- f" {100 * npass / len(b):5.0f} |{_fmt(_mean(r['geometry_detail'] for r in b))} |"
+ f"{_fmt(_mean(r['geometry_detail'] for r in b))} |"
f"{_fmt(_mean(r['structure_plausibility'] for r in b))} |{_fmt(_mean(r['assembly_fit'] for r in b))} |"
f"{_fmt(_mean(r['craftsmanship_no_artifacts'] for r in b))} |"
f"{_fmt(_mean(r['cost_usd'] for r in b))} |{_fmt(_mean(r['minutes'] for r in b))} |"
@@ -275,9 +277,6 @@ def dollars_per_point(rows: Sequence[Row]) -> dict[str, Any]:
cost = sum(float(r.get("cost_usd") or 0.0) for r in rows)
mins = sum(float(r.get("minutes") or 0.0) for r in rows)
points = sum(100.0 * float(r.get("index") or 0.0) for r in rows)
- passed = [r for r in rows if r.get("passed")]
- p_cost = sum(float(r.get("cost_usd") or 0.0) for r in passed)
- p_points = sum(100.0 * float(r.get("index") or 0.0) for r in passed)
return {
"runs": len(rows),
"total_usd": round(cost, 3),
@@ -285,8 +284,6 @@ def dollars_per_point(rows: Sequence[Row]) -> dict[str, Any]:
"complexity_points": round(points, 1),
"usd_per_point": round(cost / points, 4) if points else None,
"minutes_per_point": round(mins / points, 3) if points else None,
- "usd_per_point_passed_only": round(p_cost / p_points, 4) if p_points else None,
- "passed": len(passed),
}
diff --git a/eval/bench/coupling_stats.py b/eval/bench/coupling_stats.py
index e46d716d..35fa21a1 100644
--- a/eval/bench/coupling_stats.py
+++ b/eval/bench/coupling_stats.py
@@ -31,10 +31,10 @@ from codeverse3d.spatial.joints_model import UrdfError, load_urdf # noqa: E402
def _urdfs(root: Path) -> list[Path]:
"""The BUILT ``robot.urdf`` of each run under ``root``, once per run.
- A run holds the same file three times — ``src/`` (what the agent wrote),
- ``artifacts/`` (what the build produced) and ``deliverable/`` (what finalise copied) —
- so counting files instead of runs multiplies every mechanical number by three. The
- artefact is the one the sweep actually posed. The walk is
+ A run holds the same file several times — ``src/`` (what the agent wrote),
+ ``artifacts/`` (the last build), ``artifacts/rNN/`` (every round's own copy) and
+ ``deliverable/`` (the handed-over round) — so counting files instead of runs multiplies
+ every mechanical number. The artefact is the one the last sweep actually posed. The walk is
``flywheel.record.unique_files``, so a cell symlinked from another battery is one run."""
return [p for p in unique_files(root, "robot.urdf") if p.parent.name == "artifacts"]
diff --git a/eval/bench/h2h_glb.py b/eval/bench/h2h_glb.py
index 4ad1545d..9c153840 100644
--- a/eval/bench/h2h_glb.py
+++ b/eval/bench/h2h_glb.py
@@ -6,9 +6,10 @@ Protocol (one fixed judge, one renderer, one gate set, for BOTH sides):
verbatim with an empty ``must_have`` (their run had no checklist, so ours gets none);
each prompt's last tag is the gallery slug.
2. THEIR GLB is ``<gallery>/<slug>/object.glb`` (built by their harness, never
- rebuilt here). OUR GLB is the best round of the harness run under
- ``<bench_out>/<lang>/runs/<id>/`` (``deliverable/object.glb``, else
- ``artifacts/object.glb`` — ``finalise`` restores and rebuilds the best round).
+ rebuilt here). OUR GLB is the round ``codeverse3d.addons.select`` picks from the
+ harness run under ``<bench_out>/<lang>/runs/<id>/`` (``deliverable/object.glb``, else
+ that round's own ``artifacts/rNN/object.glb``, else — a run recorded before rounds kept
+ their own — ``artifacts/object.glb``).
3. Both GLBs go through the same pipeline as ``bench/_fixed_eval.FixedEvaluator``:
``measure_glb`` → ``check_connectivity`` → ``render_glb`` (OBJECT_VIEWS, settings
size, contact sheet) → ``VlmJudge(static_object_v1, gemini-3.1-pro-preview,
@@ -47,10 +48,11 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resol
from bench._fixed_eval import RUBRIC # noqa: E402
from bench.h2h_scene import sign_test # noqa: E402
from bench.run_bench import Battery, BenchPrompt # noqa: E402
+from codeverse3d.addons import select # noqa: E402
JUDGE_MODEL = "gemini:gemini-3.1-pro-preview"
N_SAMPLES = 2
-TERMINAL = {"passed", "plateau", "budget", "failed"} # RunStatus values a finished run can hold
+IN_PROGRESS = {"planning", "generating", "refining"} # RunStatus values of a run that has not stopped
_H2H_TAGS = {"h2h", "brilliana"}
@@ -120,10 +122,19 @@ def our_side(runs_dir: Path, item_id: str) -> Side:
side.minutes = round((t1 - t0).total_seconds() / 60, 2)
except (KeyError, TypeError, ValueError):
pass
- if status not in TERMINAL or rec.get("best_round") is None:
- side.error = f"run not finished (status={status!r}, best_round={rec.get('best_round')})"
+ if not status or status in IN_PROGRESS:
+ side.error = f"run not finished (status={status!r})"
return side
- for cand in (ws / "deliverable" / "object.glb", ws / "artifacts" / "object.glb"):
+ try:
+ picked = select.summarise(ws).picked_round
+ except Exception as e: # noqa: BLE001 - one unreadable record must not kill the battery
+ side.error = f"unreadable record: {e}"
+ return side
+ if picked is None:
+ side.error = f"no judged round to hand over (status={status!r})"
+ return side
+ for cand in (ws / "deliverable" / "object.glb", ws / "artifacts" / f"r{picked:02d}" / "object.glb",
+ ws / "artifacts" / "object.glb"):
if cand.is_file():
side.glb = str(cand)
return side
diff --git a/eval/bench/h2h_scene.py b/eval/bench/h2h_scene.py
index 2c7d26c6..e30a00f8 100644
--- a/eval/bench/h2h_scene.py
+++ b/eval/bench/h2h_scene.py
@@ -10,8 +10,8 @@ Protocol
— authored cameras (``view_cam_*``) first, then the overview rig (``view_overview_*``),
capped at ``MAX_VIEWS``; zone/probe frames are never shown. Their own judge score,
wall time and cost come from ``run_report.json`` / ``trajectories/*/score.json``.
-* OURS: the best round of ``bench/out/h2h_scene_v1/runs/<id>/`` (``record.best_round`` →
- ``rounds/rNN.json`` renders): the ``t=0`` views the scene judge saw (per-view ``judge``
+* OURS: the round ``codeverse3d.addons.select`` picks from ``bench/out/h2h_scene_v1/runs/<id>/``
+ (``rounds/rNN.json`` renders): the ``t=0`` views the scene judge saw (per-view ``judge``
flag, authored cameras first), capped at ``MAX_VIEWS`` — stills only, so both sides are
judged blind to animation in the same way.
* Judge: ONE ``VlmJudge(rubric=scene_v1, gemini-3.1-pro-preview, n_samples=2)`` fed a
@@ -42,6 +42,7 @@ for _p in (Path(__file__).resolve().parents[2] / "harness", Path(__file__).resol
sys.path.insert(0, str(_p)) # this tree's codeverse3d (harness/) + the `bench` package (eval/)
from bench.run_bench import Battery, BenchPrompt # noqa: E402
+from codeverse3d.addons import select # noqa: E402
from codeverse3d.contracts.artifacts import RenderSet, RenderView # noqa: E402
from codeverse3d.contracts.common import Language, Track # noqa: E402
from codeverse3d.contracts.spec import Spec # noqa: E402
@@ -81,7 +82,7 @@ class OurSide(BaseModel):
defect_penalty: float = 0.0
harness_score: float | None = None
rounds: int = 0
- best_round: int = 0
+ picked_round: int = 0
cost_usd: float = 0.0
minutes: float = 0.0
n_views: int = 0
@@ -134,8 +135,12 @@ def our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:
if not rec_path.is_file():
return None
rec = json.loads(rec_path.read_text())
- best = int(rec.get("best_round") or 0)
- rnd_path = run_dir / "rounds" / f"r{best:02d}.json"
+ try:
+ summary = select.summarise(run_dir)
+ except Exception: # noqa: BLE001 - an unreadable record is a run we cannot compare
+ return None
+ picked = summary.picked_round if summary.picked_round is not None else 0
+ rnd_path = run_dir / "rounds" / f"r{picked:02d}.json"
if not rnd_path.is_file():
return None
rnd = json.loads(rnd_path.read_text())
@@ -150,9 +155,9 @@ def our_frames(run_dir: Path) -> tuple[list[RenderView], dict[str, Any]] | None:
if minutes is None and rec.get("started_at") and rec.get("finished_at"):
t0, t1 = (datetime.fromisoformat(rec[k]) for k in ("started_at", "finished_at"))
minutes = (t1 - t0).total_seconds() / 60
- meta = {"harness_score": rec.get("final_score"), "rounds": len(rec.get("rounds") or []), "best_round": best,
+ meta = {"harness_score": summary.picked_score, "rounds": len(rec.get("rounds") or []), "picked_round": picked,
"cost_usd": float((rec.get("total_usage") or {}).get("cost_usd") or 0.0), "minutes": round(minutes or 0.0, 2),
- "status": str(rec.get("status") or "")}
+ "status": summary.stop_reason}
return [RenderView(name=v.name, path=v.path, mode="shaded", width=v.width, height=v.height) for v in chosen], meta
@@ -206,7 +211,7 @@ def sheet(row_id: str, theirs: list[Path], ours: list[Path], out_png: Path, thum
x += t.width + 4
return canvas
- a, b = strip(theirs, f"{row_id} — THEIRS (authored cams + overview)"), strip(ours, f"{row_id} — OURS (best round, t=0 judge views)")
+ a, b = strip(theirs, f"{row_id} — THEIRS (authored cams + overview)"), strip(ours, f"{row_id} — OURS (picked round, t=0 judge views)")
out = Image.new("RGB", (max(a.width, b.width), a.height + b.height + 6), (60, 60, 60))
out.paste(a, (0, 0))
out.paste(b, (0, a.height + 6))
diff --git a/eval/bench/judge_calib_graphics.py b/eval/bench/judge_calib_graphics.py
index 351f9c57..123bc606 100644
--- a/eval/bench/judge_calib_graphics.py
+++ b/eval/bench/judge_calib_graphics.py
@@ -12,7 +12,7 @@ person sees → revise) never ran for graphics. This script is that loop's meas
* ``eye.json``: ``{run_dir: score}`` — a person's 0–1 reading of the SAME sheets ("would a
curator screenshot it / does it look like the thing"). The ground truth this calibrates to.
-For every run and every rubric it rebuilds the JudgeInput the loop used (spec, the best round's
+For every run and every rubric it rebuilds the JudgeInput the loop used (spec, the picked round's
RenderSet and gates; no planner acceptance items, so the rubric is measured on its own) and
judges it with a fresh ``VlmJudge`` (n samples, pro). Output per rubric: ``<rubric>.jsonl`` (one
row per run: overall, per-criterion, defects present, caps) and one ``summary.md`` with
@@ -29,36 +29,38 @@ from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any
+from codeverse3d.addons import select
from codeverse3d.addons.calibration import spearman
from codeverse3d.contracts.artifacts import GateReport, RenderSet
from codeverse3d.contracts.spec import Spec
from codeverse3d.judges.base import JudgeInput, plan_digest
from codeverse3d.judges.vlm_judge import VlmJudge
+from codeverse3d.record.record import RecordError
from codeverse3d.tracks.graphics import frame_stats_text
from codeverse3d.workspace import Workspace
OUT_ROOT = Path(__file__).resolve().parent / "out"
-def best_round(run: Path) -> dict[str, Any] | None:
- best: dict[str, Any] | None = None
- for f in sorted(run.glob("rounds/r*.json")):
- r = json.loads(f.read_text())
- j = r.get("judgment") or {}
- if j.get("overall") is not None and (best is None or j["overall"] > (best.get("judgment") or {}).get("overall", -1)):
- best = r
- return best
+def picked_round(run: Path) -> dict[str, Any] | None:
+ """The round ``codeverse3d.addons.select`` picks, as its rounds/rNN.json (None: nothing judged)."""
+ try:
+ idx = select.pick(run)
+ except RecordError:
+ return None
+ path = run / "rounds" / f"r{idx:02d}.json" if idx is not None else None
+ return json.loads(path.read_text()) if path is not None and path.is_file() else None
def judge_one(run: Path, rubric: str, model: str, n: int) -> dict[str, Any]:
spec = Spec.model_validate(json.loads((run / "spec.json").read_text()))
- rnd = best_round(run)
+ rnd = picked_round(run)
if rnd is None:
return {"run": str(run), "error": "no judged round"}
renders = RenderSet.model_validate(rnd["renders"])
gates = [GateReport.model_validate(g) for g in rnd.get("gates") or []]
# what the in-run judge also saw: the plan digest and the measured frame metrics (the metrics
- # file is the LAST build's; it is only quoted when the best round is the last one, else the
+ # file is the LAST build's; it is only quoted when the picked round is the last one, else the
# judge is told so — without it, pro called four moving effects "static" in the first pass)
ws = Workspace(run)
plan_path = run / "plan.json"
diff --git a/eval/bench/paired_compare.py b/eval/bench/paired_compare.py
index 44476ddc..a657be1a 100644
--- a/eval/bench/paired_compare.py
+++ b/eval/bench/paired_compare.py
@@ -137,8 +137,10 @@ def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_a
hs.append(h.score)
os_.append(o.score)
st.deltas[p] = round(h.score - o.score, 4)
- hp.append(bool(h.passed))
- op.append(bool(o.passed))
+ if h.passed is not None: # a bench-run row carries no pass/fail (2026-09-22): no rate
+ hp.append(h.passed)
+ if o.passed is not None:
+ op.append(o.passed)
hb.append(h.build_ok)
ob.append(o.build_ok)
st.n = len(hs)
@@ -153,8 +155,8 @@ def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_a
st.losses = sum(d < 0 for d in deltas)
st.ties = st.n - st.wins - st.losses
st.sign_p = _sign_test(deltas)["sign_p"] # type: ignore[assignment]
- st.pass_rate_harness = round(sum(hp) / st.n, 4)
- st.pass_rate_oneshot = round(sum(op) / st.n, 4)
+ st.pass_rate_harness = round(sum(hp) / len(hp), 4) if hp else None
+ st.pass_rate_oneshot = round(sum(op) / len(op), 4) if op else None
st.build_ok_harness = round(sum(hb) / st.n, 4)
st.build_ok_oneshot = round(sum(ob) / st.n, 4)
if st.n >= 2:
@@ -257,7 +259,7 @@ def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:
"""A ``bench run`` battery read as cells of one arm.
``compare_backends`` writes one journal with an ``arm`` column; ``bench run`` writes a
- directory per arm with ``id`` / ``score_final``. Two of those directories are a paired
+ directory per arm with ``id`` / ``score_picked`` (``score_final`` before 2026-09-22). Two of those directories are a paired
comparison — same prompts, one thing different — and this lets the statistics below
(paired CI, exact sign test, the "unsupported when the interval crosses zero" rule) be
the same for both shapes rather than recomputed by hand.
@@ -268,11 +270,13 @@ def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:
continue
raw = json.loads(line)
# a bench-run row records no build flag: a cell with a verdict was built and judged,
- # one without (score None, passed None) was not — so `build_ok` here means "judged",
- # and `gen_cost_usd` is the run's WHOLE cost (plan + loop judge), as the field says
+ # one without (score None) was not — so `build_ok` here means "judged", and
+ # `gen_cost_usd` is the run's WHOLE cost (plan + loop judge), as the field says. A
+ # row written before 2026-09-22 still has its old `passed`; a new one has none.
+ score = raw.get("score_picked", raw.get("score_final"))
rows.append(CellResult(prompt_id=raw["id"], arm=arm, tier=raw.get("tier", ""),
- score=raw.get("score_final"), status=raw.get("status", ""),
- passed=raw.get("passed"), build_ok=raw.get("passed") is not None,
+ score=score, status=raw.get("status", ""),
+ passed=raw.get("passed"), build_ok=score is not None,
gen_cost_usd=float(raw.get("cost_usd") or 0.0), kind="harness"))
return rows
diff --git a/eval/bench/report.py b/eval/bench/report.py
index bc22878b..35ef6373 100644
--- a/eval/bench/report.py
+++ b/eval/bench/report.py
@@ -1,7 +1,8 @@
"""Aggregate a bench run: per-tier / per-category stats → markdown + HTML gallery.
``build_report(out_dir)`` reads ``results.jsonl`` (or results.json), aggregates
-mean/median/pass-rate/cost per tier and category, writes ``report.md`` and
+baseline / picked-round mean + median, the delta and cost per tier and category (no pass
+rate: a run is not passed or failed since 2026-09-22), writes ``report.md`` and
``report.html`` — the same self-contained page as ``3dcode gallery build --embed``
(``codeverse3d.addons.gallery``), one section per tier, with the stats tables under the
summary strip.
@@ -35,10 +36,9 @@ class GroupStats(BaseModel):
n_evaluated: int = Field(default=0, description="rows that actually ran (n minus provider outages)")
infra_failed: int = Field(default=0, description="rows dropped: the provider, not the model, failed")
baseline_mean: float | None
- final_mean: float | None
- final_median: float | None
+ picked_mean: float | None
+ picked_median: float | None
delta_mean: float | None
- pass_rate: float | None
cost_mean: float
minutes_mean: float
errors: int
@@ -73,15 +73,13 @@ def _stats(group: str, rs: list[BenchItemResult]) -> GroupStats:
# min/run 12.8x. It is also not an `error`: that column must keep meaning "a real
# crash", or a storm and a bug read the same in the report.
ev = [r for r in rs if r.status != "infra_failed"]
- finals = [r.score_final for r in ev if r.score_final is not None]
+ picked = [r.score_picked for r in ev if r.score_picked is not None]
bases = [r.score_baseline for r in ev if r.score_baseline is not None]
- deltas = [r.score_final - r.score_baseline for r in ev if r.score_final is not None and r.score_baseline is not None]
- passed = [r.passed for r in ev if r.passed is not None]
+ deltas = [r.score_picked - r.score_baseline for r in ev if r.score_picked is not None and r.score_baseline is not None]
return GroupStats(
- group=group, n=len(rs), n_scored=len(finals), n_evaluated=len(ev), infra_failed=len(rs) - len(ev),
- baseline_mean=_mean(bases), final_mean=_mean(finals),
- final_median=round(statistics.median(finals), 4) if finals else None, delta_mean=_mean(deltas),
- pass_rate=round(sum(passed) / len(passed), 4) if passed else None,
+ group=group, n=len(rs), n_scored=len(picked), n_evaluated=len(ev), infra_failed=len(rs) - len(ev),
+ baseline_mean=_mean(bases), picked_mean=_mean(picked),
+ picked_median=round(statistics.median(picked), 4) if picked else None, delta_mean=_mean(deltas),
cost_mean=round(statistics.fmean([r.cost_usd for r in ev]), 4) if ev else 0.0,
minutes_mean=round(statistics.fmean([r.minutes for r in ev]), 2) if ev else 0.0,
errors=sum(1 for r in ev if r.status in ("error", "failed") or r.errors),
@@ -98,11 +96,11 @@ def _grouped(rs: list[BenchItemResult], key: str) -> list[GroupStats]:
def _md_table(title: str, stats: list[GroupStats]) -> str:
lines = [f"### {title}", "",
- "| group | n | scored | baseline | final | median | Δ | pass | $/run | min/run | errors | outage |",
- "|---|---|---|---|---|---|---|---|---|---|---|---|"]
+ "| group | n | scored | baseline | picked | median | Δ | $/run | min/run | errors | outage |",
+ "|---|---|---|---|---|---|---|---|---|---|---|"]
for s in stats:
- lines.append(f"| {s.group} | {s.n} | {s.n_scored} | {_fmt(s.baseline_mean)} | {_fmt(s.final_mean)} | "
- f"{_fmt(s.final_median)} | {_fmt(s.delta_mean, '+.3f')} | {_fmt(s.pass_rate, '.0%')} | "
+ lines.append(f"| {s.group} | {s.n} | {s.n_scored} | {_fmt(s.baseline_mean)} | {_fmt(s.picked_mean)} | "
+ f"{_fmt(s.picked_median)} | {_fmt(s.delta_mean, '+.3f')} | "
f"{s.cost_mean:.2f} | {s.minutes_mean:.1f} | {s.errors} | {s.infra_failed} |")
return "\n".join(lines) + "\n"
@@ -124,8 +122,8 @@ def _entry_for(r: BenchItemResult) -> RunEntry:
return entry
return RunEntry(
battery=r.tier, slug=r.id, path=r.workspace or "", state="ok", title=r.id,
- generator=r.generator, judge=r.judge, score=r.score_final, baseline_score=r.score_baseline,
- passed=r.passed, rounds=r.rounds, cost_usd=r.cost_usd, minutes=r.minutes, status=r.status,
+ generator=r.generator, judge=r.judge, score=r.score_picked, baseline_score=r.score_baseline,
+ picked_round=r.picked_round, rounds=r.rounds, cost_usd=r.cost_usd, minutes=r.minutes, status=r.status,
error=r.errors,
)
@@ -143,11 +141,11 @@ def build_report(out_dir: Path | str, *, title: str | None = None) -> BenchRepor
md.append(_md_table("overall", [rep.overall] if rep.overall else []))
md.append(_md_table("by tier", rep.by_tier))
md.append(_md_table("by category", rep.by_category))
- md += ["### per prompt", "", "| id | tier | category | baseline | final | passed | rounds | $ | min | status |",
+ md += ["### per prompt", "", "| id | tier | category | baseline | picked | round | rounds | $ | min | stop |",
"|---|---|---|---|---|---|---|---|---|---|"]
for r in results:
- md.append(f"| {r.id} | {r.tier} | {r.category} | {_fmt(r.score_baseline)} | {_fmt(r.score_final)} | "
- f"{'-' if r.passed is None else ('yes' if r.passed else 'no')} | {r.rounds} | {r.cost_usd:.2f} | "
+ md.append(f"| {r.id} | {r.tier} | {r.category} | {_fmt(r.score_baseline)} | {_fmt(r.score_picked)} | "
+ f"{'-' if r.picked_round is None else f'r{r.picked_round:02d}'} | {r.rounds} | {r.cost_usd:.2f} | "
f"{r.minutes:.1f} | {r.status}{' ⚠' if r.errors else ''} |")
rep.markdown = "\n".join(md) + "\n"
(out / "report.md").write_text(rep.markdown)
@@ -158,12 +156,12 @@ def build_report(out_dir: Path | str, *, title: str | None = None) -> BenchRepor
def _html_table(stats: list[GroupStats]) -> str:
rows = "".join(
- f"<tr><td>{html.escape(s.group)}</td><td>{s.n}</td><td>{_fmt(s.baseline_mean)}</td><td>{_fmt(s.final_mean)}</td>"
- f"<td>{_fmt(s.delta_mean, '+.3f')}</td><td>{_fmt(s.pass_rate, '.0%')}</td><td>{s.cost_mean:.2f}</td>"
+ f"<tr><td>{html.escape(s.group)}</td><td>{s.n}</td><td>{_fmt(s.baseline_mean)}</td><td>{_fmt(s.picked_mean)}</td>"
+ f"<td>{_fmt(s.delta_mean, '+.3f')}</td><td>{s.cost_mean:.2f}</td>"
f"<td>{s.minutes_mean:.1f}</td><td>{s.errors}</td><td>{s.infra_failed}</td></tr>"
for s in stats
)
- return ("<table><tr><th>group</th><th>n</th><th>baseline</th><th>final</th><th>Δ</th><th>pass</th><th>$/run</th>"
+ return ("<table><tr><th>group</th><th>n</th><th>baseline</th><th>picked</th><th>Δ</th><th>$/run</th>"
f"<th>min/run</th><th>errors</th><th>outage</th></tr>{rows}</table>")
diff --git a/eval/bench/run_bench.py b/eval/bench/run_bench.py
index 74cf555c..18c659b8 100644
--- a/eval/bench/run_bench.py
+++ b/eval/bench/run_bench.py
@@ -14,13 +14,16 @@ Each prompt runs inside its own ``codeverse3d.cost.run_ledger``, exactly like a
``3dcode make``: the batteries are where most runs come from, so without it the
priced per-call rows of a whole battery went to the per-process fallback log and
``3dcode cost --runs-dir <out>/runs`` had to reconstruct them from trajectories.
-The binding is context-local, so ``--parallel N`` keeps N ledgers apart.
+The binding is context-local, so ``--parallel N`` keeps N ledgers apart. And like
+``3dcode make``, the round ``codeverse3d.addons.select`` picks is packaged after the run
+(``deliverable/`` + ``selection.json``); a result row is baseline → picked, no pass/fail.
"""
from __future__ import annotations
import csv
import json
+import logging
import time
import traceback
from collections.abc import Callable, Sequence
@@ -30,10 +33,11 @@ from pathlib import Path
from typing import Any
import yaml
-from pydantic import BaseModel, Field
+from pydantic import AliasChoices, BaseModel, Field
from bench._infra import is_infra_failure
from bench._jsonl import read_jsonl, seal_for_append
+from codeverse3d.addons import select
from codeverse3d.config import get_settings
from codeverse3d.contracts.common import Backends, Budget, Language, Track
from codeverse3d.contracts.run import RunRecord
@@ -42,7 +46,9 @@ from codeverse3d.cost import run_ledger
from codeverse3d.proc import exclusive
from codeverse3d.workspace import Workspace
-RESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_final", "passed", "rounds", "cost_usd",
+log = logging.getLogger(__name__)
+
+RESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_picked", "picked_round", "rounds", "cost_usd",
"minutes", "status", "errors", "workspace", "generator", "judge")
@@ -81,8 +87,9 @@ class BenchItemResult(BaseModel):
tier: str
category: str = ""
score_baseline: float | None = None
- score_final: float | None = None
- passed: bool | None = None
+ score_picked: float | None = Field(default=None, validation_alias=AliasChoices("score_picked", "score_final"),
+ description="the picked round's score (rows before 2026-09-22: score_final)")
+ picked_round: int | None = None
rounds: int = 0
cost_usd: float = 0.0
minutes: float = 0.0
@@ -168,8 +175,8 @@ def spec_for(battery: Battery, item: BenchPrompt, opts: BenchOptions) -> Spec:
def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Workspace) -> BenchItemResult:
- best = next((r for r in rec.rounds if r.index == rec.best_round), None)
- # A run whose rounds all lost their verdict still reports a normal status (`plateau`
+ s = select.summarise(ws.root, record=rec)
+ # A run whose rounds all lost their verdict still reports a normal status (`max_rounds`
# after three unjudged rounds), so a whole ARM can read as healthy and score nothing —
# what a worktree without node_modules did on 2026-09-04: render_glb died, every round
# skipped the judge, ten cells came back with score=None and status=plateau. Say it
@@ -179,10 +186,10 @@ def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Wo
if unjudged and not error:
error = f"no verdict in any of {len(rec.rounds)} round(s) — the judge was skipped every time"
return BenchItemResult(
- id=item.id, tier=item.tier, category=item.category, score_baseline=rec.baseline_score,
- score_final=rec.final_score, passed=None if best is None or best.judgment is None else best.judgment.passed,
+ id=item.id, tier=item.tier, category=item.category, score_baseline=s.baseline_score,
+ score_picked=s.picked_score, picked_round=s.picked_round,
rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd, minutes=round(minutes, 2),
- status=rec.status.value, errors=error, workspace=str(ws.root),
+ status=s.stop_reason, errors=error, workspace=str(ws.root),
generator=rec.spec.backends.generator, judge=rec.spec.backends.judge,
)
@@ -228,6 +235,17 @@ def default_run_track(spec: Spec, ws: Workspace, resume: bool) -> RunRecord:
return get_track(spec.track).run(spec, ws, resume=resume)
+def hand_over(ws: Workspace, rec: RunRecord) -> None:
+ """What ``3dcode make`` does after a run: package the round ``select`` picks (by score).
+ A packaging failure costs the hand-over, never the scored row."""
+ try:
+ idx = select.pick(ws.root, record=rec)
+ if idx is not None:
+ select.package(ws.root, idx, method="score")
+ except Exception as e: # noqa: BLE001 - the run and its scores stand without deliverable/
+ log.warning("hand-over of %s failed: %s", ws.root, e)
+
+
def run_battery(
battery_path: Path | str, out_dir: Path | str, opts: BenchOptions | None = None, *,
run_fn: RunFn | None = None, on_result: Callable[[BenchItemResult], None] | None = None,
@@ -270,6 +288,7 @@ def run_battery(
# the run mutex is what keeps two cells off the same workspace
with exclusive(ws.root, what=f"bench {item.id}"), run_ledger(ws.root, run=item.id):
rec = run_fn(spec, ws, resume)
+ hand_over(ws, rec)
except Exception as e: # one failing prompt must not kill the battery
# A provider outage is not a result: an hour spent retrying a 503 is not model
# latency and not a crash. compare_backends has classified this since
diff --git a/eval/tests/conftest.py b/eval/tests/conftest.py
index 1cc37b90..e52dda6b 100644
--- a/eval/tests/conftest.py
+++ b/eval/tests/conftest.py
@@ -41,7 +41,7 @@ from codeverse3d.contracts.artifacts import ( # noqa: E402
Severity,
)
from codeverse3d.contracts.common import Usage # noqa: E402
-from codeverse3d.contracts.run import RunRecord, RunStatus # noqa: E402
+from codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus # noqa: E402
from codeverse3d.contracts.spec import Spec # noqa: E402
from codeverse3d.workspace import Workspace # noqa: E402
@@ -131,7 +131,10 @@ def fake_run_track(score: float = 0.8):
(ws.root / MODEL_FILE).write_text(GOOD.format(score=score))
(ws.src / "parts").mkdir(exist_ok=True)
(ws.src / "parts" / "legs.py").write_text("LEGS = 3\n")
- rec = RunRecord(spec=spec, workspace=str(ws.root), status=RunStatus.MAX_ROUNDS, final_score=score - 0.1,
+ loop_verdict = Judgment(rubric="static_object_v1", scores={}, overall=round(score - 0.1, 4), passed=False)
+ rec = RunRecord(spec=spec, workspace=str(ws.root), status=RunStatus.MAX_ROUNDS,
+ rounds=[RoundRecord(index=0, kind="baseline", judgment=loop_verdict,
+ build=BuildResult(ok=True, language="blender"))],
total_usage=Usage(cost_usd=0.9, tool_calls=12))
ws.write_json(ws.record_path, rec)
return rec
diff --git a/eval/tests/test_bench.py b/eval/tests/test_bench.py
index 70c55a10..7f48d846 100644
--- a/eval/tests/test_bench.py
+++ b/eval/tests/test_bench.py
@@ -59,7 +59,10 @@ def test_run_battery_resume_and_report(tmp_path: Path):
res = run_battery(battery, out, opts, run_fn=_fake_run_fn(scores, fail_ids={"veh_easy_toy_car"}))
assert len(res) == 4
by_id = {r.id: r for r in res}
- assert by_id["furn_easy_stool"].score_final == 0.85 and by_id["furn_easy_stool"].passed is True
+ assert by_id["furn_easy_stool"].score_picked == 0.85 and by_id["furn_easy_stool"].picked_round == 1
+ # like `3dcode make`, the battery hands over the picked round
+ assert (out / "runs" / "furn_easy_stool" / "deliverable" / "manifest.json").is_file()
+ assert json.loads((out / "runs" / "furn_easy_stool" / "selection.json").read_text())["round"] == 1
assert by_id["veh_easy_toy_car"].status == "error" and "boom" in by_id["veh_easy_toy_car"].errors
assert (out / "results.csv").is_file() and len(load_results(out)) == 4
assert (out / "runs" / "furn_easy_stool" / "spec.json").is_file()
@@ -77,7 +80,8 @@ def test_run_battery_resume_and_report(tmp_path: Path):
rep = build_report(out)
assert rep.n == 5 and (out / "report.md").is_file() and (out / "report.html").is_file()
tiers = {g.group: g for g in rep.by_tier}
- assert tiers["easy"].n == 2 and tiers["easy"].final_mean == pytest.approx(0.85) # toy car errored → unscored
+ assert tiers["easy"].n == 2 and tiers["easy"].picked_mean == pytest.approx(0.85) # toy car errored → unscored
+ assert "pass" not in rep.markdown.split("### per prompt")[0], "no pass rate: a run is not passed or failed"
assert rep.overall is not None and rep.overall.errors == 1
assert "| furn_easy_stool | easy |" in rep.markdown
page = (out / "report.html").read_text()
@@ -150,8 +154,8 @@ def test_a_provider_outage_is_not_model_latency_and_not_an_error():
from bench.report import _stats
from bench.run_bench import BenchItemResult
- clear = [BenchItemResult(id=f"p{i}", tier="easy", score_baseline=0.5, score_final=0.8,
- passed=True, minutes=1.0, cost_usd=0.10, status="passed") for i in range(4)]
+ clear = [BenchItemResult(id=f"p{i}", tier="easy", score_baseline=0.5, score_picked=0.8,
+ minutes=1.0, cost_usd=0.10, status="max_rounds") for i in range(4)]
storm = BenchItemResult(
id="p_storm", tier="easy", minutes=60.0, cost_usd=0.03, status="infra_failed",
errors="ModelError: Gemini API error 503: The model is overloaded. Please try again later.")
@@ -164,7 +168,9 @@ def test_a_provider_outage_is_not_model_latency_and_not_an_error():
assert with_storm.n == 5 and with_storm.n_evaluated == 4 and with_storm.infra_failed == 1
assert with_storm.errors == 0, "a provider outage must not read as a crash"
# the scored rates were already safe; they must stay so
- assert with_storm.final_mean == base.final_mean and with_storm.pass_rate == base.pass_rate
+ assert with_storm.picked_mean == base.picked_mean and with_storm.delta_mean == base.delta_mean
+ # a row written before 2026-09-22 still reads: its score_final is the picked score
+ assert BenchItemResult.model_validate({"id": "old", "tier": "easy", "score_final": 0.7, "passed": True}).score_picked == 0.7
def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):
diff --git a/eval/tests/test_bench_shared.py b/eval/tests/test_bench_shared.py
index e7f7e41d..6df35ef7 100644
--- a/eval/tests/test_bench_shared.py
+++ b/eval/tests/test_bench_shared.py
@@ -78,27 +78,32 @@ def test_the_harness_arm_looks_for_each_languages_own_entry_file():
assert entry_of(SimpleNamespace(language=Language.GLSL_SHADER)) == "src/shader.frag"
-def test_a_run_that_scored_nothing_says_so_in_its_row() -> None:
+def test_a_run_that_scored_nothing_says_so_in_its_row(tmp_path: Path) -> None:
"""A worktree without node_modules made render_glb die, every round skip the judge and
ten cells come back `status=plateau, score=None` — an arm that reads as healthy and
measures nothing (2026-09-04). The row carries the reason now."""
from bench.run_bench import result_from_record
+ from codeverse3d.contracts.artifacts import Judgment
+ from codeverse3d.contracts.common import Language, Track, Usage
+ from codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus
+ from codeverse3d.contracts.spec import Spec
+ from codeverse3d.workspace import Workspace
item = SimpleNamespace(id="cpl_umbrella", tier="hard", category="mechanism")
- rounds = [SimpleNamespace(index=i, judgment=None) for i in range(3)]
- rec = SimpleNamespace(
- rounds=rounds, best_round=0, baseline_score=None, final_score=None, error="",
- total_usage=SimpleNamespace(cost_usd=1.76), status=SimpleNamespace(value="plateau"),
- spec=SimpleNamespace(backends=SimpleNamespace(generator="g", judge="j")))
- ws = SimpleNamespace(root=Path("/tmp/ws"))
+ spec = Spec(id="cpl_umbrella", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="an umbrella")
+ rounds = [RoundRecord(index=i, kind="refine") for i in range(3)]
+ rec = RunRecord(spec=spec, workspace=str(tmp_path), status=RunStatus.MAX_ROUNDS, rounds=rounds,
+ total_usage=Usage(cost_usd=1.76))
+ ws = Workspace(tmp_path)
row = result_from_record(item, rec, 16.0, ws)
- assert row.score_final is None and row.status == "plateau"
+ assert row.score_picked is None and row.picked_round is None and row.status == "max_rounds"
assert "no verdict in any of 3 round(s)" in row.errors
- judged = SimpleNamespace(index=0, judgment=SimpleNamespace(passed=True))
+ judged = RoundRecord(index=0, kind="baseline", judgment=Judgment(rubric="r", scores={}, overall=0.6, passed=False))
rec.rounds = [judged, rounds[1]]
- assert result_from_record(item, rec, 16.0, ws).errors == "" # one verdict is enough
+ row = result_from_record(item, rec, 16.0, ws)
+ assert row.errors == "" and (row.score_picked, row.picked_round) == (0.6, 0) # one verdict is enough
def test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:
diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.py
index 6c000213..bc9ab72f 100644
--- a/eval/tests/test_infra_failures.py
+++ b/eval/tests/test_infra_failures.py
@@ -313,7 +313,7 @@ def test_the_ab_viewer_refuses_to_call_a_winner_it_cannot_support():
from bench.ab_view import Run, verdict
def arm(name, *scores):
- return [Run(slug=f"{name}{i}", arm=name, final=s, status="passed") for i, s in enumerate(scores)]
+ return [Run(slug=f"{name}{i}", arm=name, picked=s, status="max_rounds") for i, s in enumerate(scores)]
cases = [
((0.7,), (0.9,), "Inconclusive"),
@@ -326,7 +326,7 @@ def test_the_ab_viewer_refuses_to_call_a_winner_it_cannot_support():
assert head.startswith(expected), head
# a run that never scored must not be counted as an observation
- a = arm("a", 0.5, 0.5) + [Run(slug="a9", arm="a", final=None, status="failed")]
+ a = arm("a", 0.5, 0.5) + [Run(slug="a9", arm="a", picked=None, status="failed")]
head, why = verdict(a, arm("b", 0.5))
assert head.startswith("Inconclusive") and "1 of 1" in why
diff --git a/harness/codeverse3d/addons/gallery/index.py b/harness/codeverse3d/addons/gallery/index.py
index ddcda0e2..2cf40feb 100644
--- a/harness/codeverse3d/addons/gallery/index.py
+++ b/harness/codeverse3d/addons/gallery/index.py
@@ -133,8 +133,16 @@ def _articulation_sheet(ws: Workspace, picked: int | None) -> str:
def entry_links(ws: Workspace, rec: RunRecord | None, picked: int | None) -> list[RunLink]:
- """The working links of a card: workspace, record, code, sheet, GLB, track extras."""
+ """The working links of a card: workspace, record, code, sheet, GLB, track extras — the
+ artifact files of the handed-over round (``deliverable/``), else of the picked round
+ (``artifacts/rNN/``), else whatever ``artifacts/`` holds (a run recorded before rounds
+ kept their own)."""
run = ws.root
+ kept = [f"artifacts/r{picked:02d}"] if picked is not None else []
+
+ def artifact(name: str, *extra: str) -> str:
+ return _first_file(run, f"deliverable/{name}", *(f"{k}/{name}" for k in kept), f"artifacts/{name}", *extra)
+
links = [RunLink(label="workspace", rel="", kind="dir"),
RunLink(label="record.json", rel="record.json")]
for name in ("spec.json", "plan.json"):
@@ -145,7 +153,7 @@ def entry_links(ws: Workspace, rec: RunRecord | None, picked: int | None) -> lis
sheet = picked_sheet(ws, rec, picked) if rec is not None else ""
if sheet:
links.append(RunLink(label="sheet", rel=sheet))
- glb = _first_file(run, "deliverable/object.glb", "artifacts/object.glb")
+ glb = artifact("object.glb")
if glb:
links.append(RunLink(label="glb", rel=glb, kind="viewer"))
# the textured GLB is a link only when the texture pass SHIPPED — a stray file
@@ -155,13 +163,13 @@ def entry_links(ws: Workspace, rec: RunRecord | None, picked: int | None) -> lis
textured = _first_file(run, "deliverable/object_textured.glb", "artifacts/object_textured.glb")
if textured:
links.append(RunLink(label="textured glb", rel=textured, kind="viewer"))
- urdf = _first_file(run, "deliverable/robot.urdf", "artifacts/robot.urdf", "src/robot.urdf")
+ urdf = artifact("robot.urdf", "src/robot.urdf")
if urdf:
links.append(RunLink(label="robot.urdf", rel=urdf))
art = _articulation_sheet(ws, picked)
if art:
links.append(RunLink(label="articulation", rel=art))
- gif = _first_file(run, "deliverable/preview.gif", "artifacts/preview.gif")
+ gif = artifact("preview.gif")
if gif:
links.append(RunLink(label="preview.gif", rel=gif))
if (run / "artifacts" / "frames").is_dir():
diff --git a/harness/codeverse3d/cli/main.py b/harness/codeverse3d/cli/main.py
index a971a556..36e85915 100644
--- a/harness/codeverse3d/cli/main.py
+++ b/harness/codeverse3d/cli/main.py
@@ -124,7 +124,7 @@ def run_cmd(
console.print(f"battery={battery} out={out_dir} generator={generator or 'default'} judge={judge or 'default'}")
def _on(res) -> None:
- console.print(f" [{res.status}] {res.id}: baseline={res.score_baseline} final={res.score_final} "
+ console.print(f" [{res.status}] {res.id}: baseline={res.score_baseline} picked={res.score_picked} "
f"rounds={res.rounds} ${res.cost_usd:.2f} {res.minutes:.1f}min" + (f" [red]{res.errors[:80]}[/red]" if res.errors else ""))
results = run_bench.run_battery(battery, out_dir, opts, on_result=_on)
diff --git a/harness/tests/gallery/test_index.py b/harness/tests/gallery/test_index.py
index 9285ee03..c53a7cde 100644
--- a/harness/tests/gallery/test_index.py
+++ b/harness/tests/gallery/test_index.py
@@ -41,6 +41,22 @@ def test_entry_fields_and_links(gallery_tree: dict[str, Path]):
assert entry.key == "runs/wooden_chair_ab12cd34"
+def test_a_card_links_the_picked_rounds_own_glb(tmp_path: Path):
+ """The tree ends at the LAST round (2026-09-22), so artifacts/object.glb is not the card's
+ round: until a round is handed over, the card links the picked round's artifacts/rNN/ copy."""
+ from codeverse3d.addons import select
+ from tests.flywheel_cli.conftest import make_fake_run
+
+ ws, _ = make_fake_run(tmp_path / "runs", "kept_rounds", scores=(0.9, 0.6)) # the pick is r00
+ (ws.round_artifacts(0)).mkdir(parents=True)
+ (ws.round_artifacts(0) / "object.glb").write_bytes(b"glTF r00")
+ links = {ln.label: ln.rel for ln in entry_for_dir("runs", ws.root).links}
+ assert links["glb"] == "artifacts/r00/object.glb"
+ select.package(ws.root, 0)
+ links = {ln.label: ln.rel for ln in entry_for_dir("runs", ws.root).links}
+ assert links["glb"] == "deliverable/object.glb"
+
+
def test_corrupt_record_does_not_raise(tmp_path: Path):
run = tmp_path / "runs" / "boom"
run.mkdir(parents=True)