Official Codebase of 3DCodeVerse
Python
eval: an import arm, tokens on every cell, a three.js one-shot contract, the design-mode battery, gpt-6-luna and opus-5.5 prices
gaoypengb18e3842a4dd
diff --git a/eval/bench/_bare_agent.py b/eval/bench/_bare_agent.pyindex 4c989f77..69f6cd5d 100644--- a/eval/bench/_bare_agent.py+++ b/eval/bench/_bare_agent.py@@ -40,6 +40,11 @@ def _tools_note(language: Language) -> str:return (f"`{GLSLANG}` validates GLSL; python3 has moderngl, numpy and Pillow, so you can write your own ""offscreen renderer under tmp/ (prepend the uniforms above, draw a fullscreen quad, save PNGs at a ""few times) and open the PNGs to look at them.")+ if language is Language.THREEJS:+ return (f"node is installed; `{node_modules_dir()}` contains three (r182) and puppeteer with a cached headless "+ "Chrome (set NODE_PATH to it, or import by absolute path). You may write throwaway scripts under tmp/ "+ "that import src/object.js, call build(THREE), print its bounding box, and render it to PNGs in "+ "headless Chrome, then open the PNGs to look at them.")if language is Language.SCENE_THREEJS:return (f"node is installed; `{node_modules_dir()}` contains three (r182) and puppeteer with a cached headless ""Chrome (set NODE_PATH to it, or import by absolute path). You may write your own throwaway "diff --git a/eval/bench/_compare_report.py b/eval/bench/_compare_report.pyindex 8dd6c0fb..8b91cf96 100644--- a/eval/bench/_compare_report.py+++ b/eval/bench/_compare_report.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport htmlimport json+import reimport shutilimport statisticsfrom pathlib import Path@@ -48,6 +49,9 @@ class CellResult(BaseModel):tris: int | None = Nonegen_cost_usd: float = Field(default=0.0, description="generation cost (harness: whole run incl. its loop judge)")judge_cost_usd: float = Field(default=0.0, description="fixed-judge cost for this cell")+ gen_input_tokens: int = Field(default=0, description="generation input tokens (uncached + cached), whole run")+ gen_output_tokens: int = Field(default=0, description="generation output tokens incl. thinking, whole run")+ gen_cached_tokens: int = Field(default=0, description="the cache-read share of gen_input_tokens")gen_seconds: float = 0.0wall_s: float = Field(default=0.0, description="the cell's clock, provider waits included: what flag_degraded reads")minutes: float | None = Field(default=None, description=(@@ -303,7 +307,8 @@ def compare_html(out: Path, rows: list[CellResult], pairs: list[PairRow], meta:continueimg = ""if r.sheet and Path(r.sheet).is_file():- dst = assets / f"{pid}__{a.replace(':', '_').replace('+', '_')}{Path(r.sheet).suffix}"+ safe = re.sub(r"[^A-Za-z0-9._-]+", "_", a) # an import arm's name carries a path+ dst = assets / f"{pid}__{safe}{Path(r.sheet).suffix}"shutil.copy2(r.sheet, dst)img = f"<img src='report_assets/{dst.name}' loading='lazy'>"cls = "ok" if r.passed else "bad"diff --git a/eval/bench/_oneshot.py b/eval/bench/_oneshot.pyindex 0459ac19..87a64285 100644--- a/eval/bench/_oneshot.py+++ b/eval/bench/_oneshot.py@@ -81,6 +81,7 @@ MODEL_FILE = "src/model.py"URDF_FILE = "src/robot.urdf"SHADER_FILE = ENTRY_FILE[Language.GLSL_SHADER] # src/shader.fragSCENE_FILE = ENTRY_FILE[Language.SCENE_THREEJS] # src/scene.js+OBJECT_FILE = ENTRY_FILE[Language.THREEJS] # src/object.jsDEFAULT_TIMEOUT_S = 900.0IDLE_GRACE_S = 600.0@@ -201,8 +202,26 @@ _OUTPUT_RULE_SCENE = (f"You have NO tools in this session: you cannot write filef"```js fenced code block and nothing else (no prose before or after, no partial snippets).")+#: The three.js static-object minimal contract: the build(THREE) shape, frame and units of+#: codeverse3d/prompts/threejs/contract.md — no plan, no parts files, no cookbook, no gates table.+_OBJECT_CONTRACT_BODY = """You write ONE file, `{object_file}`, an ES module for three.js r182:+ export function build(THREE) → THREE.Group (the whole object, returned as one root Group)+Imports: only `three` and `three/addons/*` (BufferGeometryUtils, RoundedBoxGeometry …). No renderer, scene,+camera, lights, DOM, textures, images, files or network — procedural geometry and MeshStandardMaterial only.+Y is up, +Z is the front, +X the right; metres at real-world size; the lowest point sits at y = 0 and the+footprint is centred on the Y axis. Name the main part Groups (PascalCase, unique).+The harness imports the module in node, calls build(THREE), exports a GLB, renders it from 14 views and+judges it: the object must be recognisable and complete, parts must touch (nothing floating, no deep+interpenetration), proportions plausible."""++_OUTPUT_RULE_OBJECT = (f"You have NO tools in this session: you cannot write files, run node or render anything — "+ f"the code must appear in your reply. Reply with the COMPLETE contents of `{OBJECT_FILE}` as ONE "+ f"```js fenced code block and nothing else (no prose before or after, no partial snippets).")++#: the one file a single-file language answers with (URDF answers with two, see files_for)-_ENTRY = {Language.BLENDER: MODEL_FILE, Language.GLSL_SHADER: SHADER_FILE, Language.SCENE_THREEJS: SCENE_FILE}+_ENTRY = {Language.BLENDER: MODEL_FILE, Language.GLSL_SHADER: SHADER_FILE, Language.SCENE_THREEJS: SCENE_FILE,+ Language.THREEJS: OBJECT_FILE}def files_for(language: Language) -> list[str]:@@ -223,12 +242,14 @@ def minimal_contract(language: Language = Language.BLENDER) -> str:return _GLSL_CONTRACT_BODY.format(shader_file=SHADER_FILE)if language is Language.SCENE_THREEJS:return _SCENE_CONTRACT_BODY.format(scene_file=SCENE_FILE)+ if language is Language.THREEJS:+ return _OBJECT_CONTRACT_BODY.format(object_file=OBJECT_FILE)return _CONTRACT_BODY.format(model_file=MODEL_FILE, frame=frame_doc(LANGUAGE_FRAME["blender"]))def output_rule(language: Language = Language.BLENDER) -> str:return {Language.URDF_BLENDER: _OUTPUT_RULE_URDF, Language.GLSL_SHADER: _OUTPUT_RULE_GLSL,- Language.SCENE_THREEJS: _OUTPUT_RULE_SCENE}.get(language, _OUTPUT_RULE)+ Language.SCENE_THREEJS: _OUTPUT_RULE_SCENE, Language.THREEJS: _OUTPUT_RULE_OBJECT}.get(language, _OUTPUT_RULE)def oneshot_prompt(spec: Spec) -> str:@@ -236,7 +257,8 @@ def oneshot_prompt(spec: Spec) -> str:c = spec.constraintslead = {Language.URDF_BLENDER: "Model this ARTICULATED object as raw bpy link meshes plus a hand-written URDF",Language.GLSL_SHADER: "Write this as a Shadertoy-style fragment shader",- Language.SCENE_THREEJS: "Build this scene in raw three.js"}.get(spec.language, "Model this object in raw bpy")+ Language.SCENE_THREEJS: "Build this scene in raw three.js",+ Language.THREEJS: "Model this object in raw three.js"}.get(spec.language, "Model this object in raw bpy")lines = [f"{lead}: {spec.prompt.strip()}"]if c.dimensions_m:lines.append("Dimensions (m): " + ", ".join(f"{k}={v:g}" for k, v in c.dimensions_m.items()))@@ -293,11 +315,11 @@ def _strip_hallucinated_tool_xml(text: str) -> str:if not bodies:return textbody = max(bodies, key=len)- return body if any(k in body for k in ("import bpy", "createScene", "mainImage", "void main")) else text+ return body if any(k in body for k in ("import bpy", "createScene", "mainImage", "void main", "function build")) else text#: what a bare (unfenced) answer must contain to be read as the file at all-_BARE_MARKERS = {SCENE_FILE: ("createScene",), SHADER_FILE: ("mainImage", "void main(")}+_BARE_MARKERS = {SCENE_FILE: ("createScene",), SHADER_FILE: ("mainImage", "void main("), OBJECT_FILE: ("function build",)}def extract_model_file(text: str, rel: str = MODEL_FILE) -> str:diff --git a/eval/bench/compare_backends.py b/eval/bench/compare_backends.pyindex d94ce3c4..debde89c 100644--- a/eval/bench/compare_backends.py+++ b/eval/bench/compare_backends.py@@ -81,7 +81,7 @@ from bench.run_bench import ( # noqa: E402from codeverse3d.addons import select # noqa: E402from codeverse3d.config import get_settings # noqa: E402from codeverse3d.contracts.artifacts import RenderSet # noqa: E402-from codeverse3d.contracts.common import ENTRY_FILE, Track # noqa: E402+from codeverse3d.contracts.common import ENTRY_FILE, Track, Usage # noqa: E402from codeverse3d.contracts.run import RunRecord, RunStatus # noqa: E402from codeverse3d.contracts.spec import Spec # noqa: E402from codeverse3d.cost import run_ledger # noqa: E402@@ -93,7 +93,7 @@ from codeverse3d.workspace import Workspace # noqa: E402log = logging.getLogger(__name__)-ArmKind = Literal["harness", "oneshot", "oneshot+repair", "agent"]+ArmKind = Literal["harness", "oneshot", "oneshot+repair", "agent", "import"]# ----------------------------------------------------------------------------- arms / options@@ -109,10 +109,13 @@ class Arm(BaseModel):def parse_arm(text: str) -> Arm:kind, sep, target = text.strip().partition(":")- if not sep or not target or kind not in ("harness", "oneshot", "oneshot+repair", "agent"):+ if not sep or not target or kind not in ("harness", "oneshot", "oneshot+repair", "agent", "import"):raise ValueError(f"bad arm {text!r}: expected harness:<generator-id> | oneshot:<target> | oneshot+repair:<target>"- " | agent:<generator-id>")- if kind == "agent":+ " | agent:<generator-id> | import:<directory>")+ if kind == "import":+ if not Path(target).expanduser().is_dir():+ raise ValueError(f"import arm: {target!r} is not a directory")+ elif kind == "agent":from codeverse3d.agents.registry import parse_agent_idparse_agent_id(target) # validates the vendor id earlyelif kind != "harness":@@ -150,7 +153,8 @@ class CompareOptions(BaseModel):def spec_for(battery: Battery, item: BenchPrompt, arm: Arm, opts: CompareOptions) -> Spec:- generator = arm.target if arm.kind in ("harness", "agent") else f"single-shot:{arm.target}"+ generator = arm.target if arm.kind in ("harness", "agent") else (+ "single-shot:gemini:gemini-3.8-flash" if arm.kind == "import" else f"single-shot:{arm.target}")backends = get_settings().backends(generator=generator, judge=opts.loop_judge, planner=opts.planner)return build_spec(battery, item, backends=backends, rounds=opts.rounds,max_minutes=opts.max_minutes, tag0="compare", extra_tags=(arm.kind,))@@ -195,6 +199,30 @@ def _render_scaffold(deps: CompareDeps, spec: Spec, eval_ws: Workspace) -> None:(eval_ws.root / entry_of(spec)).unlink(missing_ok=True)+def _add_usage(res: CellResult, usage: Usage) -> None:+ """Token accounting beside the dollars: subscription CLIs bill no money, so tokens are the comparable unit."""+ res.gen_input_tokens += int(usage.input_tokens or 0)+ res.gen_output_tokens += int(usage.output_tokens or 0) + int(usage.thoughts_tokens or 0)+ res.gen_cached_tokens += int(usage.cached_tokens or 0)+++def _import_answer(arm: Arm, spec: Spec, eval_ws: Workspace, deps: CompareDeps, res: CellResult) -> None:+ """``import:<directory>``: score a file some OTHER tool already produced — no model is asked. The file+ is ``<directory>/<prompt id>.<ext>`` (the battery's id is the file stem), copied as the language's entry file+ (e.g. Claude Design's three.js ``build(THREE)`` modules → ``src/object.js``)."""+ root = Path(arm.target).expanduser()+ stem = spec.id.split("/")[-1]+ hits = sorted(p for p in root.glob(f"{stem}.*") if p.is_file())+ if not hits:+ res.error = f"import arm: no {stem}.* in {root}"+ return+ if spec.track is Track.SCENE:+ _render_scaffold(deps, spec, eval_ws)+ dst = eval_ws.root / entry_of(spec)+ dst.parent.mkdir(parents=True, exist_ok=True)+ shutil.copy2(hits[0], dst)++def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptions,deps: CompareDeps, res: CellResult) -> None:backend = deps.oneshot_backend(arm.target)@@ -220,6 +248,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, optsres.attempts = attempt + 1res.gen_cost_usd += gen.usage.cost_usdres.tool_calls += gen.tool_calls+ _add_usage(res, gen.usage)res.gen_seconds += gen.duration_sres.minutes = round((res.minutes or 0.0) + max(0.0, gen.duration_s - lost.lost_s()) / 60, 2)if not gen.ok:@@ -291,6 +320,7 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, opts: CompareOptionsummary = select.summarise(run_ws.root, record=rec)res.gen_cost_usd = rec.total_usage.cost_usdres.tool_calls = rec.total_usage.tool_calls+ _add_usage(res, rec.total_usage)res.minutes = round(rec.minutes or 0.0, 2)res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), summary.picked_scoreres.harness_stop_reason = summary.stop_reason@@ -332,6 +362,7 @@ def _run_bare_agent(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts:result = run_bare_agent(spec, arm.target, cell, eval_ws, minutes=opts.max_minutes)res.gen_cost_usd = result.usage.cost_usdres.tool_calls = result.tool_calls+ _add_usage(res, result.usage)res.gen_seconds = result.duration_sres.minutes = round(max(0.0, result.duration_s - result.provider_wait_s) / 60, 2)res.harness_status = result.exit_reason@@ -370,6 +401,8 @@ def run_cell(battery: Battery, item: BenchPrompt, arm: Arm, out: Path, opts: Comwith run_ledger(cell, run=f"{item.id}:{arm.slug}"):if arm.kind == "harness":_run_harness(spec, cell, eval_ws, opts, deps, res)+ elif arm.kind == "import":+ _import_answer(arm, spec, eval_ws, deps, res)elif arm.kind == "agent":_run_bare_agent(arm, spec, cell, eval_ws, opts, deps, res)else:diff --git a/eval/bench/prompts/h2h_designmode_v1.yaml b/eval/bench/prompts/h2h_designmode_v1.yamlnew file mode 100644index 00000000..d0de6f97--- /dev/null+++ b/eval/bench/prompts/h2h_designmode_v1.yaml@@ -0,0 +1,670 @@+# h2h_designmode_v1 — generated 2026-09-25 from the design-mode repo (commit 465e8ad); edit here, not by hand-copying.+name: h2h_designmode_v1+track: static_object+language: threejs+description: 100 object briefs from github.com/jameson475/3d_batch_with_designmode (Claude Design outputs in its+ code/), verbatim, no must_have — every arm gets exactly the brief Claude Design got. tier = the repo's detail+ level (simple→easy, medium→medium, detailed→hard). ids are the repo's file stems, so `import:<repo>/code` scores+ Claude Design's own build(THREE) modules as src/object.js.+prompts:+- id: 01_desk_lamp+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: An articulated desk lamp with a weighted round base, two arm segments joined by hinges, and a cone shade+ with a glowing bulb.+- id: 02_wooden_chair+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A simple Scandinavian wooden dining chair with four tapered legs, a slatted backrest, and a light oak+ finish.+- id: 03_ceramic_mug+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A glazed ceramic coffee mug with a curved handle, a thick rim, and dark coffee inside.+- id: 04_potted_succulent+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A small terracotta pot holding an echeveria succulent with rosette leaves arranged in a golden-angle spiral.+- id: 05_retro_rocket+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A 1950s retro rocket ship with a pointed nose cone, porthole window, three swept fins, and a flame exhaust.+- id: 06_lighthouse+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A red-and-white striped lighthouse on a rocky base, with a gallery railing and a glowing lantern room.+- id: 07_bonsai_tree+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A bonsai tree in a shallow rectangular tray, with a recursively branching trunk and clustered foliage+ pads.+- id: 08_toy_robot+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A boxy tin toy robot with antenna, glowing eyes, chest dial panel, jointed arms with claw hands, and wind-up+ key.+- id: 09_spiral_staircase+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A cast-iron spiral staircase with 20 wedge-shaped treads around a central pole and a helical handrail.+- id: 10_hot_air_balloon+ tier: hard+ category: batch_01+ tags:+ - detailed+ prompt: A hot air balloon with a teardrop envelope in alternating colored gores, suspension ropes, a burner, and+ a wicker basket.+- id: 11_vintage_radio+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A 1940s tabletop tube radio with a rounded-top walnut cabinet, a fabric speaker grille, a glowing tuning+ dial, and two knobs.+- id: 12_windmill+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A Dutch windmill with an octagonal tapered tower, a thatched cap, four lattice sails, and a wraparound+ balcony.+- id: 13_sailboat+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A small wooden sloop sailboat with a curved hull, a single mast, a triangular mainsail and jib, and a+ boom.+- id: 14_acoustic_guitar+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: An acoustic guitar with a figure-eight spruce body, a sound hole with rosette, a long fretted neck, a+ headstock with tuning pegs, and six strings.+- id: 15_park_bench+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A classic park bench with cast-iron scrolled side frames, wooden slat seat and curved backrest, and armrests.+- id: 16_mushroom_cluster+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A cluster of five fly-agaric mushrooms of different sizes growing from a mossy log, red caps with white+ spots.+- id: 17_castle_tower+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A medieval stone castle tower with crenellated battlements, arrow-slit windows, a wooden door, and a flag+ on a pole.+- id: 18_hourglass+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: An antique hourglass with a turned wooden frame, three spindle posts, a glass double bulb, and sand mid-flow.+- id: 19_satellite+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A communications satellite with a gold-foil bus, two deployable solar panel wings, a parabolic dish antenna,+ and thruster nozzles.+- id: 20_treasure_chest+ tier: hard+ category: batch_02+ tags:+ - detailed+ prompt: A pirate treasure chest with a half-cylinder lid propped open, iron bands and rivets, a padlock hasp,+ and gold coins spilling out.+- id: 21_bicycle+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A classic city bicycle with spoked wheels, a diamond steel frame, a leather saddle, curved handlebars,+ and a front basket.+- id: 22_grand_piano+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A glossy black grand piano with a curved case, a propped-open lid, 52 white and 36 black keys, three legs,+ and pedals.+- id: 23_teapot+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A round porcelain teapot with blue bands, a curved spout, a loop handle, and a lid with a knob, on a saucer.+- id: 24_film_camera+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A vintage 35mm rangefinder film camera with a leatherette body, chrome top plate, a lens barrel with focus+ ring, dials, and a shutter button.+- id: 25_desk_fan+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A retro oscillating desk fan with a round wire cage, four curved blades, a motor housing, a tilting neck,+ and a heavy base.+- id: 26_wine_set+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A dark green wine bottle with a cork and label beside a stemmed wine glass half-filled with red wine.+- id: 27_anvil+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A blacksmith's anvil with a tapered horn, a flat face with a hardy hole, a waisted body, mounted on a+ tree-stump block with a hammer resting on it.+- id: 28_fire_hydrant+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A red cast-iron fire hydrant with a domed bonnet, a pentagonal operating nut, two side hose outlets with+ chained caps, and a flanged base bolted to the sidewalk.+- id: 29_umbrella+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: An open umbrella with an eight-panel canopy in alternating colors, visible ribs, a finial tip, and a curved+ wooden J-handle, tilted as if held.+- id: 30_pagoda+ tier: hard+ category: batch_03+ tags:+ - detailed+ prompt: A five-tiered East Asian pagoda on a stone plinth, each tier with red columns and upturned curved eaves,+ topped by a bronze spire with rings.+- id: 31_steam_locomotive+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A toy-scale steam locomotive with a cylindrical boiler, smokestack, steam dome, cab with windows, cowcatcher,+ and six red driving wheels linked by coupling rods.+- id: 32_desk_globe+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A desk globe tilted at 23.5 degrees on a brass meridian ring and a turned wooden stand, with procedurally+ generated continents.+- id: 33_bookshelf+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A wooden bookshelf with four shelves filled with books of varied heights, thicknesses and colors, some+ leaning, plus a small plant on top.+- id: 34_candelabra+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: An ornate silver five-arm candelabra with S-curved arms, drip cups, lit white candles with flames, and+ a turned base.+- id: 35_birdhouse+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A painted wooden birdhouse on a post, with a pitched shingled roof, a round entrance hole, a perch dowel,+ and a small bird sitting on the roof.+- id: 36_snowman+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A three-ball snowman with a carrot nose, coal eyes and buttons, stick arms, a knitted striped scarf, and+ a black top hat, standing on a snowy mound.+- id: 37_telescope+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A brass refracting telescope on a wooden tripod with an altitude mount, a finder scope, an eyepiece, and+ a dew shield, pointed at the sky.+- id: 38_wooden_barrel+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: An oak wine barrel built from individual bulging staves held by four iron hoops, with a round head and+ a bung hole, lying on a wooden cradle.+- id: 39_wind_chime+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A hanging wind chime with a round wooden top disk, six aluminum tubes of graded lengths on strings, a+ central striker, and a sail.+- id: 40_oil_lantern+ tier: hard+ category: batch_04+ tags:+ - detailed+ prompt: A hurricane oil lantern with a red-painted fuel tank, a glass globe with a glowing flame, a wire guard,+ a vented cap, and a bail handle.+- id: 41_office_chair+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: An ergonomic office swivel chair with a five-star base on casters, a gas-lift column, a padded seat, a+ curved mesh backrest, and armrests.+- id: 42_knight_helmet+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: 'A medieval knight''s great helm on a display stand: a steel bucket helmet with a riveted cross reinforcement,+ eye slits, breathing holes, and a red plume.'+- id: 43_microscope+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: A laboratory compound microscope with a curved arm, an angled eyepiece tube, a rotating nosepiece with+ three objectives, a mechanical stage with slide clips, coarse and fine focus knobs, and a base lamp.+- id: 44_cupcake+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: A cupcake in a pleated paper liner topped with a swirled pink frosting spiral, rainbow sprinkles, and+ a cherry with a stem.+- id: 45_traffic_cones+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: Three orange traffic cones with white reflective bands on square black bases, one knocked over, next to+ a striped road barrier on A-frame legs.+- id: 46_chess_set+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: A corner of a wooden chessboard with a Staunton-style king, queen, rook, bishop, and pawn in ivory and+ ebony, all turned on a lathe.+- id: 47_wheelbarrow+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: A garden wheelbarrow with a deep green steel tray, a single pneumatic front wheel, tubular handles with+ rubber grips, rear legs, and a load of soil.+- id: 48_jellyfish+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: A translucent glowing jellyfish with a scalloped bell, frilly oral arms, and long trailing tentacles that+ wave in sine curves.+- id: 49_grandfather_clock+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: A tall mahogany grandfather clock with an arched hood, a clock face with Roman-numeral ticks and hands,+ a glass-front trunk showing a brass pendulum and weights, and a plinth base.+- id: 50_ferris_wheel+ tier: hard+ category: batch_05+ tags:+ - detailed+ prompt: A carnival Ferris wheel with a double-rim spoked wheel, twelve colorful gondolas hanging level, an A-frame+ support tower, and light bulbs along the rim.+- id: 51_apple+ tier: easy+ category: batch_06+ tags:+ - simple+ prompt: A red apple.+- id: 52_donut+ tier: easy+ category: batch_06+ tags:+ - simple+ prompt: A donut with pink icing and sprinkles.+- id: 53_coffee_table+ tier: medium+ category: batch_06+ tags:+ - medium+ prompt: A mid-century modern coffee table with a walnut oval top and splayed legs.+- id: 54_rubber_duck+ tier: easy+ category: batch_06+ tags:+ - simple+ prompt: A rubber duck.+- id: 55_pencil+ tier: easy+ category: batch_06+ tags:+ - simple+ prompt: A yellow pencil.+- id: 56_alarm_clock+ tier: medium+ category: batch_06+ tags:+ - medium+ prompt: A classic twin-bell alarm clock with a hammer between the bells and little feet.+- id: 57_saguaro_cactus+ tier: medium+ category: batch_06+ tags:+ - medium+ prompt: A saguaro cactus with two arms in a clay pot, with ribbed stems and spines.+- id: 58_laptop+ tier: medium+ category: batch_06+ tags:+ - medium+ prompt: An open laptop with a keyboard, trackpad, and a glowing screen.+- id: 59_electric_guitar+ tier: hard+ category: batch_06+ tags:+ - detailed+ prompt: A sunburst electric guitar with an offset double-cutaway solid body, a white pickguard, three single-coil+ pickups, volume and tone knobs, a whammy bar, a maple neck with dot inlays, and a six-in-line headstock.+- id: 60_space_shuttle+ tier: hard+ category: batch_06+ tags:+ - detailed+ prompt: 'A space shuttle orbiter stacked for launch: the white orbiter with black heat-shield underside and delta+ wings, mounted on a large orange external fuel tank flanked by two white solid rocket boosters, standing on+ a launch platform.'+- id: 61_tree+ tier: easy+ category: batch_07+ tags:+ - simple+ prompt: A tree.+- id: 62_house+ tier: easy+ category: batch_07+ tags:+ - simple+ prompt: A small house.+- id: 63_old_key+ tier: easy+ category: batch_07+ tags:+ - simple+ prompt: An old key.+- id: 64_street_lamp+ tier: medium+ category: batch_07+ tags:+ - medium+ prompt: A Victorian cast-iron street lamp with a lantern head and a fluted post.+- id: 65_sofa+ tier: medium+ category: batch_07+ tags:+ - medium+ prompt: A three-seat fabric sofa with rounded arms, seat cushions, and throw pillows.+- id: 66_fishbowl+ tier: medium+ category: batch_07+ tags:+ - medium+ prompt: A round glass fishbowl with water, gravel, a plant, and a goldfish.+- id: 67_viking_ship+ tier: hard+ category: batch_07+ tags:+ - detailed+ prompt: A Viking longship with an overlapping clinker-built plank hull, a carved dragon-head prow and curled stern,+ a single mast with a red-and-white striped square sail, a row of painted round shields along each gunwale, and+ oars.+- id: 68_drum_kit+ tier: hard+ category: batch_07+ tags:+ - detailed+ prompt: 'A five-piece rock drum kit: a bass drum with a logo head and pedal, a snare on a stand, two rack toms+ mounted on the bass drum, a floor tom on legs, a hi-hat, and two cymbals on boom stands, all with sparkle-red+ wraps and chrome hoops.'+- id: 69_mailbox+ tier: easy+ category: batch_07+ tags:+ - simple+ prompt: A mailbox.+- id: 70_skateboard+ tier: medium+ category: batch_07+ tags:+ - medium+ prompt: A skateboard with a concave deck, kicktails, trucks, and colorful wheels.+- id: 71_star+ tier: easy+ category: batch_08+ tags:+ - simple+ prompt: A gold star.+- id: 72_gem+ tier: easy+ category: batch_08+ tags:+ - simple+ prompt: A blue gemstone.+- id: 73_bucket+ tier: easy+ category: batch_08+ tags:+ - simple+ prompt: A metal bucket.+- id: 74_toaster+ tier: medium+ category: batch_08+ tags:+ - medium+ prompt: A retro chrome two-slice toaster with toast popping out.+- id: 75_headphones+ tier: medium+ category: batch_08+ tags:+ - medium+ prompt: Over-ear headphones with a padded headband and cushioned ear cups.+- id: 76_excavator+ tier: hard+ category: batch_08+ tags:+ - detailed+ prompt: A yellow hydraulic excavator on steel crawler tracks, with a rotating cab with windows, a counterweight,+ and an articulated boom, stick, and toothed bucket driven by hydraulic cylinders, digging pose.+- id: 77_carousel+ tier: hard+ category: batch_08+ tags:+ - detailed+ prompt: A fairground carousel with a striped conical canopy with scalloped valance, a central mirrored column,+ twisted brass poles, eight horses at alternating heights, a round wooden platform, and bulbs around the canopy+ edge.+- id: 78_ice_cream+ tier: easy+ category: batch_08+ tags:+ - simple+ prompt: An ice cream cone.+- id: 79_picnic_table+ tier: medium+ category: batch_08+ tags:+ - medium+ prompt: A wooden picnic table with attached benches and a checkered tablecloth.+- id: 80_trophy+ tier: medium+ category: batch_08+ tags:+ - medium+ prompt: A gold trophy cup with two handles on a marble base.+- id: 81_cloud+ tier: easy+ category: batch_09+ tags:+ - simple+ prompt: A cloud.+- id: 82_mushroom_house+ tier: medium+ category: batch_09+ tags:+ - medium+ prompt: A fairy-tale mushroom house with a round door, windows, and a chimney.+- id: 83_biplane+ tier: hard+ category: batch_09+ tags:+ - detailed+ prompt: A 1930s biplane with fabric-covered upper and lower wings joined by struts and bracing wires, an open+ cockpit with a windscreen, a radial engine with a two-blade wooden propeller, a tail with rudder and elevators,+ and fixed landing gear with spoked wheels.+- id: 84_vase+ tier: easy+ category: batch_09+ tags:+ - simple+ prompt: A vase.+- id: 85_watering_can+ tier: medium+ category: batch_09+ tags:+ - medium+ prompt: A green metal watering can with a long spout and a sprinkler rose.+- id: 86_stone_bridge+ tier: hard+ category: batch_09+ tags:+ - detailed+ prompt: A medieval stone arch bridge with three semicircular arches of individual voussoir stones, cutwater piers,+ a paved deck, and low parapet walls, spanning a stream.+- id: 87_toy_blocks+ tier: easy+ category: batch_09+ tags:+ - simple+ prompt: Toy blocks.+- id: 88_sewing_machine+ tier: hard+ category: batch_09+ tags:+ - detailed+ prompt: An antique black cast-iron sewing machine with gold decals, a curved arm and head, a hand wheel, a needle+ bar and presser foot above a chrome bed plate, a thread spool on a pin, mounted on a wooden base.+- id: 89_bell+ tier: easy+ category: batch_09+ tags:+ - simple+ prompt: A bell.+- id: 90_kite+ tier: easy+ category: batch_09+ tags:+ - simple+ prompt: A kite.+- id: 91_snail+ tier: medium+ category: batch_10+ tags:+ - medium+ prompt: A garden snail with a spiral shell and eye stalks.+- id: 92_robot_vacuum+ tier: easy+ category: batch_10+ tags:+ - simple+ prompt: A robot vacuum.+- id: 93_water_tower+ tier: medium+ category: batch_10+ tags:+ - medium+ prompt: A rooftop wooden water tower on a steel stand with a conical roof and a ladder.+- id: 94_typewriter+ tier: hard+ category: batch_10+ tags:+ - detailed+ prompt: 'A 1920s manual typewriter in black enamel: four staggered rows of round glass-topped keys with chrome+ rims, a semicircular basket of typebars, a platen roller with knobs and a sheet of paper, a carriage return+ lever, and gold lettering.'+- id: 95_igloo+ tier: easy+ category: batch_10+ tags:+ - simple+ prompt: An igloo.+- id: 96_tent+ tier: easy+ category: batch_10+ tags:+ - simple+ prompt: A camping tent.+- id: 97_submarine+ tier: medium+ category: batch_10+ tags:+ - medium+ prompt: A yellow cartoon submarine with portholes, a periscope, and a propeller.+- id: 98_treehouse+ tier: hard+ category: batch_10+ tags:+ - detailed+ prompt: 'A treehouse built into a large oak: a thick trunk with branching limbs and leafy canopy clusters, a plank+ platform with railings resting on the branches, a small cabin with a shingled roof and a window, a rope ladder+ hanging to the ground, and a tire swing on another branch.'+- id: 99_pumpkin+ tier: easy+ category: batch_10+ tags:+ - simple+ prompt: A pumpkin.+- id: 100_gramophone+ tier: hard+ category: batch_10+ tags:+ - detailed+ prompt: 'An antique gramophone: a square mahogany cabinet with a side winding crank, a turntable with a black+ shellac record showing grooves and a red label, a curved tonearm with a soundbox and needle, and a large flared+ brass horn.'diff --git a/eval/tests/test_import_arm.py b/eval/tests/test_import_arm.pynew file mode 100644index 00000000..db187aa4--- /dev/null+++ b/eval/tests/test_import_arm.py@@ -0,0 +1,76 @@+"""The import arm scores files another tool produced (no model asked); every arm records its tokens (offline)."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++REPO = Path(__file__).resolve().parents[1]+if str(REPO) not in sys.path:+ sys.path.insert(0, str(REPO))++from bench.compare_backends import CompareDeps, CompareOptions, parse_arm, run_cell # noqa: E402+from bench.run_bench import Battery # noqa: E402+from codeverse3d.contracts.common import Usage # noqa: E402+from tests.conftest import BATTERY, GOOD, FakeBackend, FakeEvaluator # noqa: E402+++def test_import_arm_needs_an_existing_directory(tmp_path):+ assert parse_arm(f"import:{tmp_path}").kind == "import"+ with pytest.raises(ValueError):+ parse_arm(f"import:{tmp_path / 'missing'}")+++def test_import_arm_scores_the_file_named_after_the_prompt(tmp_path):+ battery = Battery.load(BATTERY)+ item = battery.prompts[0]+ src = tmp_path / "design"+ src.mkdir()+ (src / f"{item.id}.py").write_text(GOOD.format(score=0.66))+ ev = FakeEvaluator()+ r = run_cell(battery, item, parse_arm(f"import:{src}"), tmp_path / "out",+ CompareOptions(judge="gemini:x", loop_judge="gemini:x"), CompareDeps(ev))+ assert (r.status, r.score) == ("scored", 0.66) and len(ev.evaluated) == 1 and ev.evaluated[0].endswith(item.id)+ assert r.gen_cost_usd == 0 and r.gen_input_tokens == 0+++def test_import_arm_without_the_file_is_no_code(tmp_path):+ battery = Battery.load(BATTERY)+ src = tmp_path / "design"+ src.mkdir()+ r = run_cell(battery, battery.prompts[0], parse_arm(f"import:{src}"), tmp_path / "out",+ CompareOptions(judge="gemini:x", loop_judge="gemini:x"), CompareDeps(FakeEvaluator()))+ assert (r.status, r.score) == ("no_code", 0.0) and "no " in r.error+++def test_a_one_shot_cell_records_its_tokens(tmp_path):+ battery = Battery.load(BATTERY)++ class Counted(FakeBackend):+ def generate(self, prompt, *, out_dir, timeout_s=0, label=""):+ res = super().generate(prompt, out_dir=out_dir, timeout_s=timeout_s, label=label)+ res.usage = Usage(input_tokens=1200, cached_tokens=200, output_tokens=800, thoughts_tokens=300, cost_usd=0.01)+ return res++ backend = Counted(["```python\n" + GOOD.format(score=0.5) + "```"])+ r = run_cell(battery, battery.prompts[0], parse_arm("oneshot:gemini:gemini-3.8-flash"), tmp_path,+ CompareOptions(judge="gemini:x", loop_judge="gemini:x"),+ CompareDeps(FakeEvaluator(), oneshot_backend=lambda t: backend))+ assert (r.gen_input_tokens, r.gen_output_tokens, r.gen_cached_tokens) == (1200, 1100, 200)+++def test_the_report_names_an_import_arms_assets_without_its_path(tmp_path):++ battery = Battery.load(BATTERY)+ item = battery.prompts[0]+ src = tmp_path / "some" / "design"+ src.mkdir(parents=True)+ (src / f"{item.id}.py").write_text(GOOD.format(score=0.7))+ from bench.compare_backends import run_matrix+ run_matrix(BATTERY, tmp_path / "out", [parse_arm(f"import:{src}")],+ CompareOptions(judge="gemini:x", loop_judge="gemini:x", ids=[item.id], pairwise=False),+ CompareDeps(FakeEvaluator()))+ assets = list((tmp_path / "out" / "report_assets").glob("*"))+ assert assets and all("/" not in a.name and a.is_file() for a in assets)diff --git a/eval/tests/test_oneshot.py b/eval/tests/test_oneshot.pyindex 26de3746..0a527cfe 100644--- a/eval/tests/test_oneshot.py+++ b/eval/tests/test_oneshot.py@@ -120,8 +120,10 @@ def test_single_file_answers_are_extracted_under_their_own_entry():glsl = "void mainImage(out vec4 o, in vec2 p) { o = vec4(p / iResolution.xy, 0.5, 1.0); }"assert extract_files(f"```glsl\n{glsl}\n```", Language.GLSL_SHADER) == {SHADER_FILE: glsl + "\n"}assert files_for(Language.SCENE_THREEJS) == [SCENE_FILE] and files_for(Language.GLSL_SHADER) == [SHADER_FILE]+ obj = "export function build(THREE) { return new THREE.Group(); }"+ assert extract_files(f"```js\n{obj}\n```", Language.THREEJS) == {"src/object.js": obj + "\n"}with pytest.raises(ValueError, match="no one-shot contract"):- files_for(Language.THREEJS)+ files_for(Language.CADQUERY)def test_a_hallucinated_write_tool_of_a_scene_module_is_still_read():diff --git a/harness/codeverse3d/models/pricing.py b/harness/codeverse3d/models/pricing.pyindex 32c3e600..2eccf484 100644--- a/harness/codeverse3d/models/pricing.py+++ b/harness/codeverse3d/models/pricing.py@@ -111,6 +111,9 @@ PRICES: dict[tuple[str, str], Price] = {("anthropic", "claude-fable-5"): Price(10.00, 50.00, 1.00, cache_write=12.50,note="1h cache write 20.00 (not modelled)"),("anthropic", "claude-mythos-5"): Price(10.00, 50.00, 1.00, cache_write=12.50, note="limited availability"),+ # Claude Opus 5.5: cheaper than Opus 5 (claude-api skill model table, cached 2026-06-24); cache write = the+ # standard 1.25x input for the 5-minute TTL, as every other Anthropic row.+ ("anthropic", "claude-opus-5-5"): Price(4.00, 20.00, 0.20, cache_write=5.00, checked="2026-09-25"),("anthropic", "claude-opus-5"): Price(5.00, 25.00, 0.50, cache_write=6.25,note="fast mode is 10.00/50.00 (not modelled)"),# the 1M-context variant of opus-5, and the id the DEFAULT claude-code arm serves.@@ -146,6 +149,12 @@ PRICES: dict[tuple[str, str], Price] = {("anthropic", "claude-haiku-3-5"): Price(0.80, 4.00, 0.08, cache_write=1.00,note="retired except on Bedrock / Google Cloud"),# ---------------------------------------------------------------- openai+ ("openai", "gpt-6-luna"): Price(0.10, 0.50, 0.01,+ long_context=LongContext(272_000, 0.20, 0.75, 0.02),+ checked="2026-09-25",+ note="Standard rates: developers.openai.com/api/docs/models/gpt-6-luna; "+ ">272k input uses 2x input/cache and 1.5x output. "+ "CLI aggregate usage is a notional estimate, not per-request billing."),("openai", "gpt-6-astra"): Price(10.00, 50.00, 1.00, cache_write=12.50,long_context=LongContext(272_000, 20.00, 75.00, 2.00),checked="2026-09-23",