Official Codebase of 3DCodeVerse
Python
bench: a scene arm that never answered is not scored on the scaffold's example scene.js
gaoypeng229dd9a6de8a
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex bad39f46..22111245 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -185,6 +185,16 @@ def _fresh_ws(path: Path) -> Workspace:return Workspace(path).create()+def _render_scaffold(deps: CompareDeps, spec: Spec, eval_ws: Workspace) -> None:+ """The scene render host's scaffold (package.json, the harness-owned lib) WITHOUT its example+ ``src/scene.js``. The example used to stay on disk, so an arm whose generation failed was evaluated+ on it: all 10 claude-code:sonnet scene cells timed out with no answer and scored the scaffold's+ ~0.03 as if it were theirs (p3x_scenes_v2, 2026-09-23). With the entry gone, a failed generation+ is a missing file — no_code, or infra_failed when the provider is to blame."""+ deps.evaluator.runtime(spec.language).skeleton(eval_ws, None)+ (eval_ws.root / entry_of(spec)).unlink(missing_ok=True)++def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptions,deps: CompareDeps, res: CellResult) -> None:backend = deps.oneshot_backend(arm.target)@@ -193,7 +203,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts# the answer is ONE src/scene.js; the render host still needs the workspace scaffold# (package.json, the harness-owned lib the example imports). The example's own# scene.js is overwritten by the answer below — the model never sees any of it.- deps.evaluator.runtime(spec.language).skeleton(eval_ws, None)+ _render_scaffold(deps, spec, eval_ws)max_attempts = 1 + (opts.repair_attempts if arm.kind == "oneshot+repair" else 0)for attempt in range(max_attempts):gen_dir = cell / "gen" / f"attempt{attempt}"@@ -318,7 +328,7 @@ def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts:from bench._bare_agent import run_bare_agentif spec.track is Track.SCENE: # the render host's scaffold, exactly as a one-shot scene cell gets it- deps.evaluator.runtime(spec.language).skeleton(eval_ws, None)+ _render_scaffold(deps, spec, eval_ws)result = run_bare_agent(spec, arm.target, cell, eval_ws, minutes=opts.max_minutes)res.gen_cost_usd = result.usage.cost_usdres.tool_calls = result.tool_callsdiff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.pyindex 9af1f1dc..83b37bd9 100644--- a/eval/tests/test_infra_failures.py+++ b/eval/tests/test_infra_failures.py@@ -425,3 +425,27 @@ def test_bare_agent_provider_failure_with_a_partial_file_is_not_a_quality_zero(telse:assert (result.status,result.score)==('build_failed',0.0)assert evaluator.evaluated+++def test_the_scene_scaffold_leaves_no_example_entry_to_score(tmp_path):+ """A scene one-shot that never answered was scored on the scaffold's example scene.js (all 10+ claude-code:sonnet cells, 2026-09-23). The scaffold keeps its lib and drops the entry file."""+ from bench.compare_backends import CompareDeps, _render_scaffold+ from codeverse3d.contracts.common import Language, Track+ from codeverse3d.contracts.spec import Spec+ from codeverse3d.workspace import Workspace++ class Runtime:+ def skeleton(self, ws, plan):+ (ws.root / "src" / "lib").mkdir(parents=True, exist_ok=True)+ (ws.root / "src" / "lib" / "env.js").write_text("export {}\n")+ (ws.root / "src" / "scene.js").write_text("export async function createScene(){}\n")++ class Evaluator:+ def runtime(self, language):+ return Runtime()++ spec = Spec(id="t/harbour", track=Track.SCENE, language=Language.SCENE_THREEJS, prompt="a harbour")+ ws = Workspace(tmp_path / "eval").create()+ _render_scaffold(CompareDeps(Evaluator()), spec, ws)+ assert not (ws.root / "src" / "scene.js").exists() and (ws.root / "src" / "lib" / "env.js").is_file()