Skip to content

Official Codebase of 3DCodeVerse

Python

bench: an arm that scored nothing must not read as healthy

ziyao7d0015e09f33
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.py
index d5422ff2..b6c071e4 100644
--- a/harness/bench/run_bench.py
+++ b/harness/bench/run_bench.py
@@ -169,11 +169,20 @@ def spec_for(battery: Battery, item: BenchPrompt, opts: BenchOptions) -> Spec:
def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Workspace) -> BenchItemResult:
best = next((r for r in rec.rounds if r.index == rec.best_round), None)
+ # A run whose rounds all lost their verdict still reports a normal status (`plateau`
+ # after three unjudged rounds), so a whole ARM can read as healthy and score nothing —
+ # what a worktree without node_modules did on 2026-09-04: render_glb died, every round
+ # skipped the judge, ten cells came back with score=None and status=plateau. Say it
+ # where the row is read.
+ unjudged = bool(rec.rounds) and all(r.judgment is None for r in rec.rounds)
+ error = rec.error
+ if unjudged and not error:
+ error = f"no verdict in any of {len(rec.rounds)} round(s) — the judge was skipped every time"
return BenchItemResult(
id=item.id, tier=item.tier, category=item.category, score_baseline=rec.baseline_score,
score_final=rec.final_score, passed=None if best is None or best.judgment is None else best.judgment.passed,
rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd, minutes=round(minutes, 2),
- status=rec.status.value, errors=rec.error, workspace=str(ws.root),
+ status=rec.status.value, errors=error, workspace=str(ws.root),
generator=rec.spec.backends.generator, judge=rec.spec.backends.judge,
)
diff --git a/harness/docs/RUNBOOK.md b/harness/docs/RUNBOOK.md
index 1dc3fc42..931b063a 100644
--- a/harness/docs/RUNBOOK.md
+++ b/harness/docs/RUNBOOK.md
@@ -169,6 +169,13 @@ unless you type `--host` yourself; it never serves a path outside the declared r
python -m codeverse.judges.calibration runs/<slug> [runs/<slug2> …] --model gemini:gemini-3.1-pro-preview --n 3 --out out/
# re-judges recorded rounds; writes calibration_<model>.md/.json (never touches runs/)
```
+**A new worktree needs `runtime_js/node_modules` before it can run a battery.** Without it
+`render_glb` dies on every round, the judge is skipped for want of renders, and the cells
+come back `status=plateau` with `score=None` — an arm that reads as healthy and measures
+nothing (2026-09-04, the mimic-off arm). `npm ci` in `runtime_js/`, or symlink the
+directory from a worktree that has it (the `package.json` is the same file). The row now
+says `no verdict in any of N round(s)` when this happens.
+
A run that crashed outside its own handling leaves `record.json` with `status=failed`;
`3dcv resume` retries from the last completed stage/round (cached plan/skeleton/scene
stages are reused — this also recovers from Gemini 503 storms). Ctrl-C is safe.
diff --git a/harness/tests/compare_bench/test_bench_shared.py b/harness/tests/compare_bench/test_bench_shared.py
index 8c749ccd..2da7e503 100644
--- a/harness/tests/compare_bench/test_bench_shared.py
+++ b/harness/tests/compare_bench/test_bench_shared.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from pathlib import Path
from types import SimpleNamespace
import bench.compare_backends as cb
@@ -73,3 +74,26 @@ def test_the_harness_arm_looks_for_each_languages_own_entry_file():
assert {lang.value for lang, e in ENTRY_FILE.items() if e != "src/model.py"} == {
"threejs", "scene_threejs", "glsl_shader", "opengl_python"}
assert entry_of(SimpleNamespace(language=Language.GLSL_SHADER)) == "src/shader.frag"
+
+
+def test_a_run_that_scored_nothing_says_so_in_its_row() -> None:
+ """A worktree without node_modules made render_glb die, every round skip the judge and
+ ten cells come back `status=plateau, score=None` — an arm that reads as healthy and
+ measures nothing (2026-09-04). The row carries the reason now."""
+ from bench.run_bench import result_from_record
+
+ item = SimpleNamespace(id="cpl_umbrella", tier="hard", category="mechanism")
+ rounds = [SimpleNamespace(index=i, judgment=None) for i in range(3)]
+ rec = SimpleNamespace(
+ rounds=rounds, best_round=0, baseline_score=None, final_score=None, error="",
+ total_usage=SimpleNamespace(cost_usd=1.76), status=SimpleNamespace(value="plateau"),
+ spec=SimpleNamespace(backends=SimpleNamespace(generator="g", judge="j")))
+ ws = SimpleNamespace(root=Path("/tmp/ws"))
+
+ row = result_from_record(item, rec, 16.0, ws)
+ assert row.score_final is None and row.status == "plateau"
+ assert "no verdict in any of 3 round(s)" in row.errors
+
+ judged = SimpleNamespace(index=0, judgment=SimpleNamespace(passed=True))
+ rec.rounds = [judged, rounds[1]]
+ assert result_from_record(item, rec, 16.0, ws).errors == "" # one verdict is enough