Official Codebase of 3DCodeVerse
Python
bench: an arm that scored nothing must not read as healthy
ziyao7d0015e09f33
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.pyindex d5422ff2..b6c071e4 100644--- a/harness/bench/run_bench.py+++ b/harness/bench/run_bench.py@@ -169,11 +169,20 @@ def spec_for(battery: Battery, item: BenchPrompt, opts: BenchOptions) -> Spec:def result_from_record(item: BenchPrompt, rec: RunRecord, minutes: float, ws: Workspace) -> BenchItemResult:best = next((r for r in rec.rounds if r.index == rec.best_round), None)+ # A run whose rounds all lost their verdict still reports a normal status (`plateau`+ # after three unjudged rounds), so a whole ARM can read as healthy and score nothing —+ # what a worktree without node_modules did on 2026-09-04: render_glb died, every round+ # skipped the judge, ten cells came back with score=None and status=plateau. Say it+ # where the row is read.+ unjudged = bool(rec.rounds) and all(r.judgment is None for r in rec.rounds)+ error = rec.error+ if unjudged and not error:+ error = f"no verdict in any of {len(rec.rounds)} round(s) — the judge was skipped every time"return BenchItemResult(id=item.id, tier=item.tier, category=item.category, score_baseline=rec.baseline_score,score_final=rec.final_score, passed=None if best is None or best.judgment is None else best.judgment.passed,rounds=len(rec.rounds), cost_usd=rec.total_usage.cost_usd, minutes=round(minutes, 2),- status=rec.status.value, errors=rec.error, workspace=str(ws.root),+ status=rec.status.value, errors=error, workspace=str(ws.root),generator=rec.spec.backends.generator, judge=rec.spec.backends.judge,)diff --git a/harness/docs/RUNBOOK.md b/harness/docs/RUNBOOK.mdindex 1dc3fc42..931b063a 100644--- a/harness/docs/RUNBOOK.md+++ b/harness/docs/RUNBOOK.md@@ -169,6 +169,13 @@ unless you type `--host` yourself; it never serves a path outside the declared rpython -m codeverse.judges.calibration runs/<slug> [runs/<slug2> …] --model gemini:gemini-3.1-pro-preview --n 3 --out out/# re-judges recorded rounds; writes calibration_<model>.md/.json (never touches runs/)```+**A new worktree needs `runtime_js/node_modules` before it can run a battery.** Without it+`render_glb` dies on every round, the judge is skipped for want of renders, and the cells+come back `status=plateau` with `score=None` — an arm that reads as healthy and measures+nothing (2026-09-04, the mimic-off arm). `npm ci` in `runtime_js/`, or symlink the+directory from a worktree that has it (the `package.json` is the same file). The row now+says `no verdict in any of N round(s)` when this happens.+A run that crashed outside its own handling leaves `record.json` with `status=failed`;`3dcv resume` retries from the last completed stage/round (cached plan/skeleton/scenestages are reused — this also recovers from Gemini 503 storms). Ctrl-C is safe.diff --git a/harness/tests/compare_bench/test_bench_shared.py b/harness/tests/compare_bench/test_bench_shared.pyindex 8c749ccd..2da7e503 100644--- a/harness/tests/compare_bench/test_bench_shared.py+++ b/harness/tests/compare_bench/test_bench_shared.py@@ -2,6 +2,7 @@from __future__ import annotations+from pathlib import Pathfrom types import SimpleNamespaceimport bench.compare_backends as cb@@ -73,3 +74,26 @@ def test_the_harness_arm_looks_for_each_languages_own_entry_file():assert {lang.value for lang, e in ENTRY_FILE.items() if e != "src/model.py"} == {"threejs", "scene_threejs", "glsl_shader", "opengl_python"}assert entry_of(SimpleNamespace(language=Language.GLSL_SHADER)) == "src/shader.frag"+++def test_a_run_that_scored_nothing_says_so_in_its_row() -> None:+ """A worktree without node_modules made render_glb die, every round skip the judge and+ ten cells come back `status=plateau, score=None` — an arm that reads as healthy and+ measures nothing (2026-09-04). The row carries the reason now."""+ from bench.run_bench import result_from_record++ item = SimpleNamespace(id="cpl_umbrella", tier="hard", category="mechanism")+ rounds = [SimpleNamespace(index=i, judgment=None) for i in range(3)]+ rec = SimpleNamespace(+ rounds=rounds, best_round=0, baseline_score=None, final_score=None, error="",+ total_usage=SimpleNamespace(cost_usd=1.76), status=SimpleNamespace(value="plateau"),+ spec=SimpleNamespace(backends=SimpleNamespace(generator="g", judge="j")))+ ws = SimpleNamespace(root=Path("/tmp/ws"))++ row = result_from_record(item, rec, 16.0, ws)+ assert row.score_final is None and row.status == "plateau"+ assert "no verdict in any of 3 round(s)" in row.errors++ judged = SimpleNamespace(index=0, judgment=SimpleNamespace(passed=True))+ rec.rounds = [judged, rounds[1]]+ assert result_from_record(item, rec, 16.0, ws).errors == "" # one verdict is enough