Skip to content

Official Codebase of 3DCodeVerse

Python

bench: a scene arm that never answered is not scored on the scaffold's example scene.js

gaoypeng229dd9a6de8a
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.py
index bad39f46..22111245 100644
--- a/eval/bench/compare_backends.py
+++ b/eval/bench/compare_backends.py
@@ -185,6 +185,16 @@ def _fresh_ws(path: Path) -> Workspace:
return Workspace(path).create()
+def _render_scaffold(deps: CompareDeps, spec: Spec, eval_ws: Workspace) -> None:
+ """The scene render host's scaffold (package.json, the harness-owned lib) WITHOUT its example
+ ``src/scene.js``. The example used to stay on disk, so an arm whose generation failed was evaluated
+ on it: all 10 claude-code:sonnet scene cells timed out with no answer and scored the scaffold's
+ ~0.03 as if it were theirs (p3x_scenes_v2, 2026-09-23). With the entry gone, a failed generation
+ is a missing file — no_code, or infra_failed when the provider is to blame."""
+ deps.evaluator.runtime(spec.language).skeleton(eval_ws, None)
+ (eval_ws.root / entry_of(spec)).unlink(missing_ok=True)
+
+
def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptions,
deps: CompareDeps, res: CellResult) -> None:
backend = deps.oneshot_backend(arm.target)
@@ -193,7 +203,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts
# the answer is ONE src/scene.js; the render host still needs the workspace scaffold
# (package.json, the harness-owned lib the example imports). The example's own
# scene.js is overwritten by the answer below — the model never sees any of it.
- deps.evaluator.runtime(spec.language).skeleton(eval_ws, None)
+ _render_scaffold(deps, spec, eval_ws)
max_attempts = 1 + (opts.repair_attempts if arm.kind == "oneshot+repair" else 0)
for attempt in range(max_attempts):
gen_dir = cell / "gen" / f"attempt{attempt}"
@@ -318,7 +328,7 @@ def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts:
from bench._bare_agent import run_bare_agent
if spec.track is Track.SCENE: # the render host's scaffold, exactly as a one-shot scene cell gets it
- deps.evaluator.runtime(spec.language).skeleton(eval_ws, None)
+ _render_scaffold(deps, spec, eval_ws)
result = run_bare_agent(spec, arm.target, cell, eval_ws, minutes=opts.max_minutes)
res.gen_cost_usd = result.usage.cost_usd
res.tool_calls = result.tool_calls
diff --git a/eval/tests/test_infra_failures.py b/eval/tests/test_infra_failures.py
index 9af1f1dc..83b37bd9 100644
--- a/eval/tests/test_infra_failures.py
+++ b/eval/tests/test_infra_failures.py
@@ -425,3 +425,27 @@ def test_bare_agent_provider_failure_with_a_partial_file_is_not_a_quality_zero(t
else:
assert (result.status,result.score)==('build_failed',0.0)
assert evaluator.evaluated
+
+
+def test_the_scene_scaffold_leaves_no_example_entry_to_score(tmp_path):
+ """A scene one-shot that never answered was scored on the scaffold's example scene.js (all 10
+ claude-code:sonnet cells, 2026-09-23). The scaffold keeps its lib and drops the entry file."""
+ from bench.compare_backends import CompareDeps, _render_scaffold
+ from codeverse3d.contracts.common import Language, Track
+ from codeverse3d.contracts.spec import Spec
+ from codeverse3d.workspace import Workspace
+
+ class Runtime:
+ def skeleton(self, ws, plan):
+ (ws.root / "src" / "lib").mkdir(parents=True, exist_ok=True)
+ (ws.root / "src" / "lib" / "env.js").write_text("export {}\n")
+ (ws.root / "src" / "scene.js").write_text("export async function createScene(){}\n")
+
+ class Evaluator:
+ def runtime(self, language):
+ return Runtime()
+
+ spec = Spec(id="t/harbour", track=Track.SCENE, language=Language.SCENE_THREEJS, prompt="a harbour")
+ ws = Workspace(tmp_path / "eval").create()
+ _render_scaffold(CompareDeps(Evaluator()), spec, ws)
+ assert not (ws.root / "src" / "scene.js").exists() and (ws.root / "src" / "lib" / "env.js").is_file()