Skip to content

Official Codebase of 3DCodeVerse

Python

eval: an import arm, tokens on every cell, a three.js one-shot contract, the design-mode battery, gpt-6-luna and opus-5.5 prices

gaoypengb18e3842a4dd
diff --git a/eval/bench/_bare_agent.py b/eval/bench/_bare_agent.py
index 4c989f77..69f6cd5d 100644
--- a/eval/bench/_bare_agent.py
+++ b/eval/bench/_bare_agent.py
@@ -40,6 +40,11 @@ def _tools_note(language: Language) -> str:
return (f"`{GLSLANG}` validates GLSL; python3 has moderngl, numpy and Pillow, so you can write your own "
"offscreen renderer under tmp/ (prepend the uniforms above, draw a fullscreen quad, save PNGs at a "
"few times) and open the PNGs to look at them.")
+ if language is Language.THREEJS:
+ return (f"node is installed; `{node_modules_dir()}` contains three (r182) and puppeteer with a cached headless "
+ "Chrome (set NODE_PATH to it, or import by absolute path). You may write throwaway scripts under tmp/ "
+ "that import src/object.js, call build(THREE), print its bounding box, and render it to PNGs in "
+ "headless Chrome, then open the PNGs to look at them.")
if language is Language.SCENE_THREEJS:
return (f"node is installed; `{node_modules_dir()}` contains three (r182) and puppeteer with a cached headless "
"Chrome (set NODE_PATH to it, or import by absolute path). You may write your own throwaway "
diff --git a/eval/bench/_compare_report.py b/eval/bench/_compare_report.py
index 8dd6c0fb..8b91cf96 100644
--- a/eval/bench/_compare_report.py
+++ b/eval/bench/_compare_report.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import html
import json
+import re
import shutil
import statistics
from pathlib import Path
@@ -48,6 +49,9 @@ class CellResult(BaseModel):
tris: int | None = None
gen_cost_usd: float = Field(default=0.0, description="generation cost (harness: whole run incl. its loop judge)")
judge_cost_usd: float = Field(default=0.0, description="fixed-judge cost for this cell")
+ gen_input_tokens: int = Field(default=0, description="generation input tokens (uncached + cached), whole run")
+ gen_output_tokens: int = Field(default=0, description="generation output tokens incl. thinking, whole run")
+ gen_cached_tokens: int = Field(default=0, description="the cache-read share of gen_input_tokens")
gen_seconds: float = 0.0
wall_s: float = Field(default=0.0, description="the cell's clock, provider waits included: what flag_degraded reads")
minutes: float | None = Field(default=None, description=(
@@ -303,7 +307,8 @@ def compare_html(out: Path, rows: list[CellResult], pairs: list[PairRow], meta:
continue
img = ""
if r.sheet and Path(r.sheet).is_file():
- dst = assets / f"{pid}__{a.replace(':', '_').replace('+', '_')}{Path(r.sheet).suffix}"
+ safe = re.sub(r"[^A-Za-z0-9._-]+", "_", a) # an import arm's name carries a path
+ dst = assets / f"{pid}__{safe}{Path(r.sheet).suffix}"
shutil.copy2(r.sheet, dst)
img = f"<img src='report_assets/{dst.name}' loading='lazy'>"
cls = "ok" if r.passed else "bad"
diff --git a/eval/bench/_oneshot.py b/eval/bench/_oneshot.py
index 0459ac19..87a64285 100644
--- a/eval/bench/_oneshot.py
+++ b/eval/bench/_oneshot.py
@@ -81,6 +81,7 @@ MODEL_FILE = "src/model.py"
URDF_FILE = "src/robot.urdf"
SHADER_FILE = ENTRY_FILE[Language.GLSL_SHADER] # src/shader.frag
SCENE_FILE = ENTRY_FILE[Language.SCENE_THREEJS] # src/scene.js
+OBJECT_FILE = ENTRY_FILE[Language.THREEJS] # src/object.js
DEFAULT_TIMEOUT_S = 900.0
IDLE_GRACE_S = 600.0
@@ -201,8 +202,26 @@ _OUTPUT_RULE_SCENE = (f"You have NO tools in this session: you cannot write file
f"```js fenced code block and nothing else (no prose before or after, no partial snippets).")
+#: The three.js static-object minimal contract: the build(THREE) shape, frame and units of
+#: codeverse3d/prompts/threejs/contract.md — no plan, no parts files, no cookbook, no gates table.
+_OBJECT_CONTRACT_BODY = """You write ONE file, `{object_file}`, an ES module for three.js r182:
+ export function build(THREE) → THREE.Group (the whole object, returned as one root Group)
+Imports: only `three` and `three/addons/*` (BufferGeometryUtils, RoundedBoxGeometry …). No renderer, scene,
+camera, lights, DOM, textures, images, files or network — procedural geometry and MeshStandardMaterial only.
+Y is up, +Z is the front, +X the right; metres at real-world size; the lowest point sits at y = 0 and the
+footprint is centred on the Y axis. Name the main part Groups (PascalCase, unique).
+The harness imports the module in node, calls build(THREE), exports a GLB, renders it from 14 views and
+judges it: the object must be recognisable and complete, parts must touch (nothing floating, no deep
+interpenetration), proportions plausible."""
+
+_OUTPUT_RULE_OBJECT = (f"You have NO tools in this session: you cannot write files, run node or render anything — "
+ f"the code must appear in your reply. Reply with the COMPLETE contents of `{OBJECT_FILE}` as ONE "
+ f"```js fenced code block and nothing else (no prose before or after, no partial snippets).")
+
+
#: the one file a single-file language answers with (URDF answers with two, see files_for)
-_ENTRY = {Language.BLENDER: MODEL_FILE, Language.GLSL_SHADER: SHADER_FILE, Language.SCENE_THREEJS: SCENE_FILE}
+_ENTRY = {Language.BLENDER: MODEL_FILE, Language.GLSL_SHADER: SHADER_FILE, Language.SCENE_THREEJS: SCENE_FILE,
+ Language.THREEJS: OBJECT_FILE}
def files_for(language: Language) -> list[str]:
@@ -223,12 +242,14 @@ def minimal_contract(language: Language = Language.BLENDER) -> str:
return _GLSL_CONTRACT_BODY.format(shader_file=SHADER_FILE)
if language is Language.SCENE_THREEJS:
return _SCENE_CONTRACT_BODY.format(scene_file=SCENE_FILE)
+ if language is Language.THREEJS:
+ return _OBJECT_CONTRACT_BODY.format(object_file=OBJECT_FILE)
return _CONTRACT_BODY.format(model_file=MODEL_FILE, frame=frame_doc(LANGUAGE_FRAME["blender"]))
def output_rule(language: Language = Language.BLENDER) -> str:
return {Language.URDF_BLENDER: _OUTPUT_RULE_URDF, Language.GLSL_SHADER: _OUTPUT_RULE_GLSL,
- Language.SCENE_THREEJS: _OUTPUT_RULE_SCENE}.get(language, _OUTPUT_RULE)
+ Language.SCENE_THREEJS: _OUTPUT_RULE_SCENE, Language.THREEJS: _OUTPUT_RULE_OBJECT}.get(language, _OUTPUT_RULE)
def oneshot_prompt(spec: Spec) -> str:
@@ -236,7 +257,8 @@ def oneshot_prompt(spec: Spec) -> str:
c = spec.constraints
lead = {Language.URDF_BLENDER: "Model this ARTICULATED object as raw bpy link meshes plus a hand-written URDF",
Language.GLSL_SHADER: "Write this as a Shadertoy-style fragment shader",
- Language.SCENE_THREEJS: "Build this scene in raw three.js"}.get(spec.language, "Model this object in raw bpy")
+ Language.SCENE_THREEJS: "Build this scene in raw three.js",
+ Language.THREEJS: "Model this object in raw three.js"}.get(spec.language, "Model this object in raw bpy")
lines = [f"{lead}: {spec.prompt.strip()}"]
if c.dimensions_m:
lines.append("Dimensions (m): " + ", ".join(f"{k}={v:g}" for k, v in c.dimensions_m.items()))
@@ -293,11 +315,11 @@ def _strip_hallucinated_tool_xml(text: str) -> str:
if not bodies:
return text
body = max(bodies, key=len)
- return body if any(k in body for k in ("import bpy", "createScene", "mainImage", "void main")) else text
+ return body if any(k in body for k in ("import bpy", "createScene", "mainImage", "void main", "function build")) else text
#: what a bare (unfenced) answer must contain to be read as the file at all
-_BARE_MARKERS = {SCENE_FILE: ("createScene",), SHADER_FILE: ("mainImage", "void main(")}
+_BARE_MARKERS = {SCENE_FILE: ("createScene",), SHADER_FILE: ("mainImage", "void main("), OBJECT_FILE: ("function build",)}
def extract_model_file(text: str, rel: str = MODEL_FILE) -> str:
diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.py
index d94ce3c4..debde89c 100644
--- a/eval/bench/compare_backends.py
+++ b/eval/bench/compare_backends.py
@@ -81,7 +81,7 @@ from bench.run_bench import ( # noqa: E402
from codeverse3d.addons import select # noqa: E402
from codeverse3d.config import get_settings # noqa: E402
from codeverse3d.contracts.artifacts import RenderSet # noqa: E402
-from codeverse3d.contracts.common import ENTRY_FILE, Track # noqa: E402
+from codeverse3d.contracts.common import ENTRY_FILE, Track, Usage # noqa: E402
from codeverse3d.contracts.run import RunRecord, RunStatus # noqa: E402
from codeverse3d.contracts.spec import Spec # noqa: E402
from codeverse3d.cost import run_ledger # noqa: E402
@@ -93,7 +93,7 @@ from codeverse3d.workspace import Workspace # noqa: E402
log = logging.getLogger(__name__)
-ArmKind = Literal["harness", "oneshot", "oneshot+repair", "agent"]
+ArmKind = Literal["harness", "oneshot", "oneshot+repair", "agent", "import"]
# ----------------------------------------------------------------------------- arms / options
@@ -109,10 +109,13 @@ class Arm(BaseModel):
def parse_arm(text: str) -> Arm:
kind, sep, target = text.strip().partition(":")
- if not sep or not target or kind not in ("harness", "oneshot", "oneshot+repair", "agent"):
+ if not sep or not target or kind not in ("harness", "oneshot", "oneshot+repair", "agent", "import"):
raise ValueError(f"bad arm {text!r}: expected harness:<generator-id> | oneshot:<target> | oneshot+repair:<target>"
- " | agent:<generator-id>")
- if kind == "agent":
+ " | agent:<generator-id> | import:<directory>")
+ if kind == "import":
+ if not Path(target).expanduser().is_dir():
+ raise ValueError(f"import arm: {target!r} is not a directory")
+ elif kind == "agent":
from codeverse3d.agents.registry import parse_agent_id
parse_agent_id(target) # validates the vendor id early
elif kind != "harness":
@@ -150,7 +153,8 @@ class CompareOptions(BaseModel):
def spec_for(battery: Battery, item: BenchPrompt, arm: Arm, opts: CompareOptions) -> Spec:
- generator = arm.target if arm.kind in ("harness", "agent") else f"single-shot:{arm.target}"
+ generator = arm.target if arm.kind in ("harness", "agent") else (
+ "single-shot:gemini:gemini-3.8-flash" if arm.kind == "import" else f"single-shot:{arm.target}")
backends = get_settings().backends(generator=generator, judge=opts.loop_judge, planner=opts.planner)
return build_spec(battery, item, backends=backends, rounds=opts.rounds,
max_minutes=opts.max_minutes, tag0="compare", extra_tags=(arm.kind,))
@@ -195,6 +199,30 @@ def _render_scaffold(deps: CompareDeps, spec: Spec, eval_ws: Workspace) -> None:
(eval_ws.root / entry_of(spec)).unlink(missing_ok=True)
+def _add_usage(res: CellResult, usage: Usage) -> None:
+ """Token accounting beside the dollars: subscription CLIs bill no money, so tokens are the comparable unit."""
+ res.gen_input_tokens += int(usage.input_tokens or 0)
+ res.gen_output_tokens += int(usage.output_tokens or 0) + int(usage.thoughts_tokens or 0)
+ res.gen_cached_tokens += int(usage.cached_tokens or 0)
+
+
+def _import_answer(arm: Arm, spec: Spec, eval_ws: Workspace, deps: CompareDeps, res: CellResult) -> None:
+ """``import:<directory>``: score a file some OTHER tool already produced — no model is asked. The file
+ is ``<directory>/<prompt id>.<ext>`` (the battery's id is the file stem), copied as the language's entry file
+ (e.g. Claude Design's three.js ``build(THREE)`` modules → ``src/object.js``)."""
+ root = Path(arm.target).expanduser()
+ stem = spec.id.split("/")[-1]
+ hits = sorted(p for p in root.glob(f"{stem}.*") if p.is_file())
+ if not hits:
+ res.error = f"import arm: no {stem}.* in {root}"
+ return
+ if spec.track is Track.SCENE:
+ _render_scaffold(deps, spec, eval_ws)
+ dst = eval_ws.root / entry_of(spec)
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(hits[0], dst)
+
+
def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptions,
deps: CompareDeps, res: CellResult) -> None:
backend = deps.oneshot_backend(arm.target)
@@ -220,6 +248,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts
res.attempts = attempt + 1
res.gen_cost_usd += gen.usage.cost_usd
res.tool_calls += gen.tool_calls
+ _add_usage(res, gen.usage)
res.gen_seconds += gen.duration_s
res.minutes = round((res.minutes or 0.0) + max(0.0, gen.duration_s - lost.lost_s()) / 60, 2)
if not gen.ok:
@@ -291,6 +320,7 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOption
summary = select.summarise(run_ws.root, record=rec)
res.gen_cost_usd = rec.total_usage.cost_usd
res.tool_calls = rec.total_usage.tool_calls
+ _add_usage(res, rec.total_usage)
res.minutes = round(rec.minutes or 0.0, 2)
res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), summary.picked_score
res.harness_stop_reason = summary.stop_reason
@@ -332,6 +362,7 @@ def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts:
result = run_bare_agent(spec, arm.target, cell, eval_ws, minutes=opts.max_minutes)
res.gen_cost_usd = result.usage.cost_usd
res.tool_calls = result.tool_calls
+ _add_usage(res, result.usage)
res.gen_seconds = result.duration_s
res.minutes = round(max(0.0, result.duration_s - result.provider_wait_s) / 60, 2)
res.harness_status = result.exit_reason
@@ -370,6 +401,8 @@ def run_cell(battery: Battery, item: BenchPrompt, arm: Arm, out: Path, opts: Com
with run_ledger(cell, run=f"{item.id}:{arm.slug}"):
if arm.kind == "harness":
_run_harness(spec, cell, eval_ws, opts, deps, res)
+ elif arm.kind == "import":
+ _import_answer(arm, spec, eval_ws, deps, res)
elif arm.kind == "agent":
_run_bare_agent(arm, spec, cell, eval_ws, opts, deps, res)
else:
diff --git a/eval/bench/prompts/h2h_designmode_v1.yaml b/eval/bench/prompts/h2h_designmode_v1.yaml
new file mode 100644
index 00000000..d0de6f97
--- /dev/null
+++ b/eval/bench/prompts/h2h_designmode_v1.yaml
@@ -0,0 +1,670 @@
+# h2h_designmode_v1 — generated 2026-09-25 from the design-mode repo (commit 465e8ad); edit here, not by hand-copying.
+name: h2h_designmode_v1
+track: static_object
+language: threejs
+description: 100 object briefs from github.com/jameson475/3d_batch_with_designmode (Claude Design outputs in its
+ code/), verbatim, no must_have — every arm gets exactly the brief Claude Design got. tier = the repo's detail
+ level (simple→easy, medium→medium, detailed→hard). ids are the repo's file stems, so `import:<repo>/code` scores
+ Claude Design's own build(THREE) modules as src/object.js.
+prompts:
+- id: 01_desk_lamp
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: An articulated desk lamp with a weighted round base, two arm segments joined by hinges, and a cone shade
+ with a glowing bulb.
+- id: 02_wooden_chair
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A simple Scandinavian wooden dining chair with four tapered legs, a slatted backrest, and a light oak
+ finish.
+- id: 03_ceramic_mug
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A glazed ceramic coffee mug with a curved handle, a thick rim, and dark coffee inside.
+- id: 04_potted_succulent
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A small terracotta pot holding an echeveria succulent with rosette leaves arranged in a golden-angle spiral.
+- id: 05_retro_rocket
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A 1950s retro rocket ship with a pointed nose cone, porthole window, three swept fins, and a flame exhaust.
+- id: 06_lighthouse
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A red-and-white striped lighthouse on a rocky base, with a gallery railing and a glowing lantern room.
+- id: 07_bonsai_tree
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A bonsai tree in a shallow rectangular tray, with a recursively branching trunk and clustered foliage
+ pads.
+- id: 08_toy_robot
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A boxy tin toy robot with antenna, glowing eyes, chest dial panel, jointed arms with claw hands, and wind-up
+ key.
+- id: 09_spiral_staircase
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A cast-iron spiral staircase with 20 wedge-shaped treads around a central pole and a helical handrail.
+- id: 10_hot_air_balloon
+ tier: hard
+ category: batch_01
+ tags:
+ - detailed
+ prompt: A hot air balloon with a teardrop envelope in alternating colored gores, suspension ropes, a burner, and
+ a wicker basket.
+- id: 11_vintage_radio
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A 1940s tabletop tube radio with a rounded-top walnut cabinet, a fabric speaker grille, a glowing tuning
+ dial, and two knobs.
+- id: 12_windmill
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A Dutch windmill with an octagonal tapered tower, a thatched cap, four lattice sails, and a wraparound
+ balcony.
+- id: 13_sailboat
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A small wooden sloop sailboat with a curved hull, a single mast, a triangular mainsail and jib, and a
+ boom.
+- id: 14_acoustic_guitar
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: An acoustic guitar with a figure-eight spruce body, a sound hole with rosette, a long fretted neck, a
+ headstock with tuning pegs, and six strings.
+- id: 15_park_bench
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A classic park bench with cast-iron scrolled side frames, wooden slat seat and curved backrest, and armrests.
+- id: 16_mushroom_cluster
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A cluster of five fly-agaric mushrooms of different sizes growing from a mossy log, red caps with white
+ spots.
+- id: 17_castle_tower
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A medieval stone castle tower with crenellated battlements, arrow-slit windows, a wooden door, and a flag
+ on a pole.
+- id: 18_hourglass
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: An antique hourglass with a turned wooden frame, three spindle posts, a glass double bulb, and sand mid-flow.
+- id: 19_satellite
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A communications satellite with a gold-foil bus, two deployable solar panel wings, a parabolic dish antenna,
+ and thruster nozzles.
+- id: 20_treasure_chest
+ tier: hard
+ category: batch_02
+ tags:
+ - detailed
+ prompt: A pirate treasure chest with a half-cylinder lid propped open, iron bands and rivets, a padlock hasp,
+ and gold coins spilling out.
+- id: 21_bicycle
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A classic city bicycle with spoked wheels, a diamond steel frame, a leather saddle, curved handlebars,
+ and a front basket.
+- id: 22_grand_piano
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A glossy black grand piano with a curved case, a propped-open lid, 52 white and 36 black keys, three legs,
+ and pedals.
+- id: 23_teapot
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A round porcelain teapot with blue bands, a curved spout, a loop handle, and a lid with a knob, on a saucer.
+- id: 24_film_camera
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A vintage 35mm rangefinder film camera with a leatherette body, chrome top plate, a lens barrel with focus
+ ring, dials, and a shutter button.
+- id: 25_desk_fan
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A retro oscillating desk fan with a round wire cage, four curved blades, a motor housing, a tilting neck,
+ and a heavy base.
+- id: 26_wine_set
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A dark green wine bottle with a cork and label beside a stemmed wine glass half-filled with red wine.
+- id: 27_anvil
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A blacksmith's anvil with a tapered horn, a flat face with a hardy hole, a waisted body, mounted on a
+ tree-stump block with a hammer resting on it.
+- id: 28_fire_hydrant
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A red cast-iron fire hydrant with a domed bonnet, a pentagonal operating nut, two side hose outlets with
+ chained caps, and a flanged base bolted to the sidewalk.
+- id: 29_umbrella
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: An open umbrella with an eight-panel canopy in alternating colors, visible ribs, a finial tip, and a curved
+ wooden J-handle, tilted as if held.
+- id: 30_pagoda
+ tier: hard
+ category: batch_03
+ tags:
+ - detailed
+ prompt: A five-tiered East Asian pagoda on a stone plinth, each tier with red columns and upturned curved eaves,
+ topped by a bronze spire with rings.
+- id: 31_steam_locomotive
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A toy-scale steam locomotive with a cylindrical boiler, smokestack, steam dome, cab with windows, cowcatcher,
+ and six red driving wheels linked by coupling rods.
+- id: 32_desk_globe
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A desk globe tilted at 23.5 degrees on a brass meridian ring and a turned wooden stand, with procedurally
+ generated continents.
+- id: 33_bookshelf
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A wooden bookshelf with four shelves filled with books of varied heights, thicknesses and colors, some
+ leaning, plus a small plant on top.
+- id: 34_candelabra
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: An ornate silver five-arm candelabra with S-curved arms, drip cups, lit white candles with flames, and
+ a turned base.
+- id: 35_birdhouse
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A painted wooden birdhouse on a post, with a pitched shingled roof, a round entrance hole, a perch dowel,
+ and a small bird sitting on the roof.
+- id: 36_snowman
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A three-ball snowman with a carrot nose, coal eyes and buttons, stick arms, a knitted striped scarf, and
+ a black top hat, standing on a snowy mound.
+- id: 37_telescope
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A brass refracting telescope on a wooden tripod with an altitude mount, a finder scope, an eyepiece, and
+ a dew shield, pointed at the sky.
+- id: 38_wooden_barrel
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: An oak wine barrel built from individual bulging staves held by four iron hoops, with a round head and
+ a bung hole, lying on a wooden cradle.
+- id: 39_wind_chime
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A hanging wind chime with a round wooden top disk, six aluminum tubes of graded lengths on strings, a
+ central striker, and a sail.
+- id: 40_oil_lantern
+ tier: hard
+ category: batch_04
+ tags:
+ - detailed
+ prompt: A hurricane oil lantern with a red-painted fuel tank, a glass globe with a glowing flame, a wire guard,
+ a vented cap, and a bail handle.
+- id: 41_office_chair
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: An ergonomic office swivel chair with a five-star base on casters, a gas-lift column, a padded seat, a
+ curved mesh backrest, and armrests.
+- id: 42_knight_helmet
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: 'A medieval knight''s great helm on a display stand: a steel bucket helmet with a riveted cross reinforcement,
+ eye slits, breathing holes, and a red plume.'
+- id: 43_microscope
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: A laboratory compound microscope with a curved arm, an angled eyepiece tube, a rotating nosepiece with
+ three objectives, a mechanical stage with slide clips, coarse and fine focus knobs, and a base lamp.
+- id: 44_cupcake
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: A cupcake in a pleated paper liner topped with a swirled pink frosting spiral, rainbow sprinkles, and
+ a cherry with a stem.
+- id: 45_traffic_cones
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: Three orange traffic cones with white reflective bands on square black bases, one knocked over, next to
+ a striped road barrier on A-frame legs.
+- id: 46_chess_set
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: A corner of a wooden chessboard with a Staunton-style king, queen, rook, bishop, and pawn in ivory and
+ ebony, all turned on a lathe.
+- id: 47_wheelbarrow
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: A garden wheelbarrow with a deep green steel tray, a single pneumatic front wheel, tubular handles with
+ rubber grips, rear legs, and a load of soil.
+- id: 48_jellyfish
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: A translucent glowing jellyfish with a scalloped bell, frilly oral arms, and long trailing tentacles that
+ wave in sine curves.
+- id: 49_grandfather_clock
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: A tall mahogany grandfather clock with an arched hood, a clock face with Roman-numeral ticks and hands,
+ a glass-front trunk showing a brass pendulum and weights, and a plinth base.
+- id: 50_ferris_wheel
+ tier: hard
+ category: batch_05
+ tags:
+ - detailed
+ prompt: A carnival Ferris wheel with a double-rim spoked wheel, twelve colorful gondolas hanging level, an A-frame
+ support tower, and light bulbs along the rim.
+- id: 51_apple
+ tier: easy
+ category: batch_06
+ tags:
+ - simple
+ prompt: A red apple.
+- id: 52_donut
+ tier: easy
+ category: batch_06
+ tags:
+ - simple
+ prompt: A donut with pink icing and sprinkles.
+- id: 53_coffee_table
+ tier: medium
+ category: batch_06
+ tags:
+ - medium
+ prompt: A mid-century modern coffee table with a walnut oval top and splayed legs.
+- id: 54_rubber_duck
+ tier: easy
+ category: batch_06
+ tags:
+ - simple
+ prompt: A rubber duck.
+- id: 55_pencil
+ tier: easy
+ category: batch_06
+ tags:
+ - simple
+ prompt: A yellow pencil.
+- id: 56_alarm_clock
+ tier: medium
+ category: batch_06
+ tags:
+ - medium
+ prompt: A classic twin-bell alarm clock with a hammer between the bells and little feet.
+- id: 57_saguaro_cactus
+ tier: medium
+ category: batch_06
+ tags:
+ - medium
+ prompt: A saguaro cactus with two arms in a clay pot, with ribbed stems and spines.
+- id: 58_laptop
+ tier: medium
+ category: batch_06
+ tags:
+ - medium
+ prompt: An open laptop with a keyboard, trackpad, and a glowing screen.
+- id: 59_electric_guitar
+ tier: hard
+ category: batch_06
+ tags:
+ - detailed
+ prompt: A sunburst electric guitar with an offset double-cutaway solid body, a white pickguard, three single-coil
+ pickups, volume and tone knobs, a whammy bar, a maple neck with dot inlays, and a six-in-line headstock.
+- id: 60_space_shuttle
+ tier: hard
+ category: batch_06
+ tags:
+ - detailed
+ prompt: 'A space shuttle orbiter stacked for launch: the white orbiter with black heat-shield underside and delta
+ wings, mounted on a large orange external fuel tank flanked by two white solid rocket boosters, standing on
+ a launch platform.'
+- id: 61_tree
+ tier: easy
+ category: batch_07
+ tags:
+ - simple
+ prompt: A tree.
+- id: 62_house
+ tier: easy
+ category: batch_07
+ tags:
+ - simple
+ prompt: A small house.
+- id: 63_old_key
+ tier: easy
+ category: batch_07
+ tags:
+ - simple
+ prompt: An old key.
+- id: 64_street_lamp
+ tier: medium
+ category: batch_07
+ tags:
+ - medium
+ prompt: A Victorian cast-iron street lamp with a lantern head and a fluted post.
+- id: 65_sofa
+ tier: medium
+ category: batch_07
+ tags:
+ - medium
+ prompt: A three-seat fabric sofa with rounded arms, seat cushions, and throw pillows.
+- id: 66_fishbowl
+ tier: medium
+ category: batch_07
+ tags:
+ - medium
+ prompt: A round glass fishbowl with water, gravel, a plant, and a goldfish.
+- id: 67_viking_ship
+ tier: hard
+ category: batch_07
+ tags:
+ - detailed
+ prompt: A Viking longship with an overlapping clinker-built plank hull, a carved dragon-head prow and curled stern,
+ a single mast with a red-and-white striped square sail, a row of painted round shields along each gunwale, and
+ oars.
+- id: 68_drum_kit
+ tier: hard
+ category: batch_07
+ tags:
+ - detailed
+ prompt: 'A five-piece rock drum kit: a bass drum with a logo head and pedal, a snare on a stand, two rack toms
+ mounted on the bass drum, a floor tom on legs, a hi-hat, and two cymbals on boom stands, all with sparkle-red
+ wraps and chrome hoops.'
+- id: 69_mailbox
+ tier: easy
+ category: batch_07
+ tags:
+ - simple
+ prompt: A mailbox.
+- id: 70_skateboard
+ tier: medium
+ category: batch_07
+ tags:
+ - medium
+ prompt: A skateboard with a concave deck, kicktails, trucks, and colorful wheels.
+- id: 71_star
+ tier: easy
+ category: batch_08
+ tags:
+ - simple
+ prompt: A gold star.
+- id: 72_gem
+ tier: easy
+ category: batch_08
+ tags:
+ - simple
+ prompt: A blue gemstone.
+- id: 73_bucket
+ tier: easy
+ category: batch_08
+ tags:
+ - simple
+ prompt: A metal bucket.
+- id: 74_toaster
+ tier: medium
+ category: batch_08
+ tags:
+ - medium
+ prompt: A retro chrome two-slice toaster with toast popping out.
+- id: 75_headphones
+ tier: medium
+ category: batch_08
+ tags:
+ - medium
+ prompt: Over-ear headphones with a padded headband and cushioned ear cups.
+- id: 76_excavator
+ tier: hard
+ category: batch_08
+ tags:
+ - detailed
+ prompt: A yellow hydraulic excavator on steel crawler tracks, with a rotating cab with windows, a counterweight,
+ and an articulated boom, stick, and toothed bucket driven by hydraulic cylinders, digging pose.
+- id: 77_carousel
+ tier: hard
+ category: batch_08
+ tags:
+ - detailed
+ prompt: A fairground carousel with a striped conical canopy with scalloped valance, a central mirrored column,
+ twisted brass poles, eight horses at alternating heights, a round wooden platform, and bulbs around the canopy
+ edge.
+- id: 78_ice_cream
+ tier: easy
+ category: batch_08
+ tags:
+ - simple
+ prompt: An ice cream cone.
+- id: 79_picnic_table
+ tier: medium
+ category: batch_08
+ tags:
+ - medium
+ prompt: A wooden picnic table with attached benches and a checkered tablecloth.
+- id: 80_trophy
+ tier: medium
+ category: batch_08
+ tags:
+ - medium
+ prompt: A gold trophy cup with two handles on a marble base.
+- id: 81_cloud
+ tier: easy
+ category: batch_09
+ tags:
+ - simple
+ prompt: A cloud.
+- id: 82_mushroom_house
+ tier: medium
+ category: batch_09
+ tags:
+ - medium
+ prompt: A fairy-tale mushroom house with a round door, windows, and a chimney.
+- id: 83_biplane
+ tier: hard
+ category: batch_09
+ tags:
+ - detailed
+ prompt: A 1930s biplane with fabric-covered upper and lower wings joined by struts and bracing wires, an open
+ cockpit with a windscreen, a radial engine with a two-blade wooden propeller, a tail with rudder and elevators,
+ and fixed landing gear with spoked wheels.
+- id: 84_vase
+ tier: easy
+ category: batch_09
+ tags:
+ - simple
+ prompt: A vase.
+- id: 85_watering_can
+ tier: medium
+ category: batch_09
+ tags:
+ - medium
+ prompt: A green metal watering can with a long spout and a sprinkler rose.
+- id: 86_stone_bridge
+ tier: hard
+ category: batch_09
+ tags:
+ - detailed
+ prompt: A medieval stone arch bridge with three semicircular arches of individual voussoir stones, cutwater piers,
+ a paved deck, and low parapet walls, spanning a stream.
+- id: 87_toy_blocks
+ tier: easy
+ category: batch_09
+ tags:
+ - simple
+ prompt: Toy blocks.
+- id: 88_sewing_machine
+ tier: hard
+ category: batch_09
+ tags:
+ - detailed
+ prompt: An antique black cast-iron sewing machine with gold decals, a curved arm and head, a hand wheel, a needle
+ bar and presser foot above a chrome bed plate, a thread spool on a pin, mounted on a wooden base.
+- id: 89_bell
+ tier: easy
+ category: batch_09
+ tags:
+ - simple
+ prompt: A bell.
+- id: 90_kite
+ tier: easy
+ category: batch_09
+ tags:
+ - simple
+ prompt: A kite.
+- id: 91_snail
+ tier: medium
+ category: batch_10
+ tags:
+ - medium
+ prompt: A garden snail with a spiral shell and eye stalks.
+- id: 92_robot_vacuum
+ tier: easy
+ category: batch_10
+ tags:
+ - simple
+ prompt: A robot vacuum.
+- id: 93_water_tower
+ tier: medium
+ category: batch_10
+ tags:
+ - medium
+ prompt: A rooftop wooden water tower on a steel stand with a conical roof and a ladder.
+- id: 94_typewriter
+ tier: hard
+ category: batch_10
+ tags:
+ - detailed
+ prompt: 'A 1920s manual typewriter in black enamel: four staggered rows of round glass-topped keys with chrome
+ rims, a semicircular basket of typebars, a platen roller with knobs and a sheet of paper, a carriage return
+ lever, and gold lettering.'
+- id: 95_igloo
+ tier: easy
+ category: batch_10
+ tags:
+ - simple
+ prompt: An igloo.
+- id: 96_tent
+ tier: easy
+ category: batch_10
+ tags:
+ - simple
+ prompt: A camping tent.
+- id: 97_submarine
+ tier: medium
+ category: batch_10
+ tags:
+ - medium
+ prompt: A yellow cartoon submarine with portholes, a periscope, and a propeller.
+- id: 98_treehouse
+ tier: hard
+ category: batch_10
+ tags:
+ - detailed
+ prompt: 'A treehouse built into a large oak: a thick trunk with branching limbs and leafy canopy clusters, a plank
+ platform with railings resting on the branches, a small cabin with a shingled roof and a window, a rope ladder
+ hanging to the ground, and a tire swing on another branch.'
+- id: 99_pumpkin
+ tier: easy
+ category: batch_10
+ tags:
+ - simple
+ prompt: A pumpkin.
+- id: 100_gramophone
+ tier: hard
+ category: batch_10
+ tags:
+ - detailed
+ prompt: 'An antique gramophone: a square mahogany cabinet with a side winding crank, a turntable with a black
+ shellac record showing grooves and a red label, a curved tonearm with a soundbox and needle, and a large flared
+ brass horn.'
diff --git a/eval/tests/test_import_arm.py b/eval/tests/test_import_arm.py
new file mode 100644
index 00000000..db187aa4
--- /dev/null
+++ b/eval/tests/test_import_arm.py
@@ -0,0 +1,76 @@
+"""The import arm scores files another tool produced (no model asked); every arm records its tokens (offline)."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[1]
+if str(REPO) not in sys.path:
+ sys.path.insert(0, str(REPO))
+
+from bench.compare_backends import CompareDeps, CompareOptions, parse_arm, run_cell # noqa: E402
+from bench.run_bench import Battery # noqa: E402
+from codeverse3d.contracts.common import Usage # noqa: E402
+from tests.conftest import BATTERY, GOOD, FakeBackend, FakeEvaluator # noqa: E402
+
+
+def test_import_arm_needs_an_existing_directory(tmp_path):
+ assert parse_arm(f"import:{tmp_path}").kind == "import"
+ with pytest.raises(ValueError):
+ parse_arm(f"import:{tmp_path / 'missing'}")
+
+
+def test_import_arm_scores_the_file_named_after_the_prompt(tmp_path):
+ battery = Battery.load(BATTERY)
+ item = battery.prompts[0]
+ src = tmp_path / "design"
+ src.mkdir()
+ (src / f"{item.id}.py").write_text(GOOD.format(score=0.66))
+ ev = FakeEvaluator()
+ r = run_cell(battery, item, parse_arm(f"import:{src}"), tmp_path / "out",
+ CompareOptions(judge="gemini:x", loop_judge="gemini:x"), CompareDeps(ev))
+ assert (r.status, r.score) == ("scored", 0.66) and len(ev.evaluated) == 1 and ev.evaluated[0].endswith(item.id)
+ assert r.gen_cost_usd == 0 and r.gen_input_tokens == 0
+
+
+def test_import_arm_without_the_file_is_no_code(tmp_path):
+ battery = Battery.load(BATTERY)
+ src = tmp_path / "design"
+ src.mkdir()
+ r = run_cell(battery, battery.prompts[0], parse_arm(f"import:{src}"), tmp_path / "out",
+ CompareOptions(judge="gemini:x", loop_judge="gemini:x"), CompareDeps(FakeEvaluator()))
+ assert (r.status, r.score) == ("no_code", 0.0) and "no " in r.error
+
+
+def test_a_one_shot_cell_records_its_tokens(tmp_path):
+ battery = Battery.load(BATTERY)
+
+ class Counted(FakeBackend):
+ def generate(self, prompt, *, out_dir, timeout_s=0, label=""):
+ res = super().generate(prompt, out_dir=out_dir, timeout_s=timeout_s, label=label)
+ res.usage = Usage(input_tokens=1200, cached_tokens=200, output_tokens=800, thoughts_tokens=300, cost_usd=0.01)
+ return res
+
+ backend = Counted(["```python\n" + GOOD.format(score=0.5) + "```"])
+ r = run_cell(battery, battery.prompts[0], parse_arm("oneshot:gemini:gemini-3.8-flash"), tmp_path,
+ CompareOptions(judge="gemini:x", loop_judge="gemini:x"),
+ CompareDeps(FakeEvaluator(), oneshot_backend=lambda t: backend))
+ assert (r.gen_input_tokens, r.gen_output_tokens, r.gen_cached_tokens) == (1200, 1100, 200)
+
+
+def test_the_report_names_an_import_arms_assets_without_its_path(tmp_path):
+
+ battery = Battery.load(BATTERY)
+ item = battery.prompts[0]
+ src = tmp_path / "some" / "design"
+ src.mkdir(parents=True)
+ (src / f"{item.id}.py").write_text(GOOD.format(score=0.7))
+ from bench.compare_backends import run_matrix
+ run_matrix(BATTERY, tmp_path / "out", [parse_arm(f"import:{src}")],
+ CompareOptions(judge="gemini:x", loop_judge="gemini:x", ids=[item.id], pairwise=False),
+ CompareDeps(FakeEvaluator()))
+ assets = list((tmp_path / "out" / "report_assets").glob("*"))
+ assert assets and all("/" not in a.name and a.is_file() for a in assets)
diff --git a/eval/tests/test_oneshot.py b/eval/tests/test_oneshot.py
index 26de3746..0a527cfe 100644
--- a/eval/tests/test_oneshot.py
+++ b/eval/tests/test_oneshot.py
@@ -120,8 +120,10 @@ def test_single_file_answers_are_extracted_under_their_own_entry():
glsl = "void mainImage(out vec4 o, in vec2 p) { o = vec4(p / iResolution.xy, 0.5, 1.0); }"
assert extract_files(f"```glsl\n{glsl}\n```", Language.GLSL_SHADER) == {SHADER_FILE: glsl + "\n"}
assert files_for(Language.SCENE_THREEJS) == [SCENE_FILE] and files_for(Language.GLSL_SHADER) == [SHADER_FILE]
+ obj = "export function build(THREE) { return new THREE.Group(); }"
+ assert extract_files(f"```js\n{obj}\n```", Language.THREEJS) == {"src/object.js": obj + "\n"}
with pytest.raises(ValueError, match="no one-shot contract"):
- files_for(Language.THREEJS)
+ files_for(Language.CADQUERY)
def test_a_hallucinated_write_tool_of_a_scene_module_is_still_read():
diff --git a/harness/codeverse3d/models/pricing.py b/harness/codeverse3d/models/pricing.py
index 32c3e600..2eccf484 100644
--- a/harness/codeverse3d/models/pricing.py
+++ b/harness/codeverse3d/models/pricing.py
@@ -111,6 +111,9 @@ PRICES: dict[tuple[str, str], Price] = {
("anthropic", "claude-fable-5"): Price(10.00, 50.00, 1.00, cache_write=12.50,
note="1h cache write 20.00 (not modelled)"),
("anthropic", "claude-mythos-5"): Price(10.00, 50.00, 1.00, cache_write=12.50, note="limited availability"),
+ # Claude Opus 5.5: cheaper than Opus 5 (claude-api skill model table, cached 2026-06-24); cache write = the
+ # standard 1.25x input for the 5-minute TTL, as every other Anthropic row.
+ ("anthropic", "claude-opus-5-5"): Price(4.00, 20.00, 0.20, cache_write=5.00, checked="2026-09-25"),
("anthropic", "claude-opus-5"): Price(5.00, 25.00, 0.50, cache_write=6.25,
note="fast mode is 10.00/50.00 (not modelled)"),
# the 1M-context variant of opus-5, and the id the DEFAULT claude-code arm serves.
@@ -146,6 +149,12 @@ PRICES: dict[tuple[str, str], Price] = {
("anthropic", "claude-haiku-3-5"): Price(0.80, 4.00, 0.08, cache_write=1.00,
note="retired except on Bedrock / Google Cloud"),
# ---------------------------------------------------------------- openai
+ ("openai", "gpt-6-luna"): Price(0.10, 0.50, 0.01,
+ long_context=LongContext(272_000, 0.20, 0.75, 0.02),
+ checked="2026-09-25",
+ note="Standard rates: developers.openai.com/api/docs/models/gpt-6-luna; "
+ ">272k input uses 2x input/cache and 1.5x output. "
+ "CLI aggregate usage is a notional estimate, not per-request billing."),
("openai", "gpt-6-astra"): Price(10.00, 50.00, 1.00, cache_write=12.50,
long_context=LongContext(272_000, 20.00, 75.00, 2.00),
checked="2026-09-23",