Official Codebase of 3DCodeVerse
Python
bench: v2 hard batteries (62 prompts, 4 tracks + compare_v2) — named difficulty axes, render-verifiable must_have with dims, pose-sweep-verifiable articulation, scene composition contracts with luminance bands; adversarially reviewed
gaoypeng233c9ef944cb
diff --git a/harness/bench/prompts/articulated_v2.yaml b/harness/bench/prompts/articulated_v2.yamlnew file mode 100644index 00000000..87ac89e3--- /dev/null+++ b/harness/bench/prompts/articulated_v2.yaml@@ -0,0 +1,227 @@+# articulated_v2 — multi-DoF battery (v1: easy 75% pass; hard discriminated at 50%).+#+# Every prompt has >= 3 joints and at least one COUPLED constraint: a clearance or+# rest condition that only holds when several joints are posed together (folded stack+# thickness, sashes passing in separate tracks, a lid resting on a prop stick).+# must_have items name the MOTION with a quantity ("drawer slides 0.25 m out toward+# the front without hitting the frame") so the judge can verify each DoF in the+# per-joint limit poses. The harness renders rest + ONE joint at a time (lower/+# upper limit) and IGNORES <mimic>, so coupled conditions are stated against the+# rest pose or against the collision sweep, which samples random combined poses+# and reports interpenetration as gate errors.+# Difficulty axes: joint count (3-8 DoF), coupled clearances, mixed joint types+# (revolute + prismatic + continuous in one model).+# Expected discrimination: medium 30-60% pass (3-4 loosely coupled joints), hard+# 0-30% pass (4+ joints with clearance coupling — v1 showed baseline collapse to+# 0.25 on such prompts, with large refine deltas).+name: articulated_v2+track: articulated_object+language: urdf_blender+description: >+ 14 articulated prompts (URDF + bpy link meshes), all >= 3 DoF with coupled+ constraints. 4 medium, 10 hard. must_have names each joint's motion with a+ quantity and at least one cross-joint clearance verifiable in the rest pose+ or by the collision sweep over random combined poses.+prompts:+ # ------------------------------------------------------------------ medium+ - id: art_med_architect_lamp+ tier: medium+ category: lighting+ prompt: >+ an adjustable architect lamp with four revolute joints: base swivel, shoulder,+ elbow and head tilt, with parallel link bars beside the lower arm+ must_have:+ - "weighted round base"+ - "arm assembly swivels at least 180 degrees about the vertical base axis"+ - "lower arm about 0.35 m long, shoulder revolute raises it from 20 to 80 degrees"+ - "upper arm about 0.35 m long, elbow revolute with at least 120 degrees range"+ - "head tilt revolute at the wrist, shade opens downward"+ - "twin parallel link bars alongside the lower arm"+ - "shoulder at its lower limit folds the lower arm down toward the base without touching it"+ - id: art_med_tool_chest+ tier: medium+ category: tools+ prompt: >+ a machinist tool chest with a rear-hinged lid over a top tray and three+ drawers of decreasing height on slides+ must_have:+ - "chest carcass about 0.60 m wide and 0.40 m tall"+ - "lid hinged along the rear edge, opens 90 degrees to expose a top tray"+ - "3 drawers of decreasing height from bottom to top"+ - "each drawer slides 0.25 m out toward the front without hitting its neighbours"+ - "drawer pulls on every drawer front"+ - "with the lid closed, each fully opened drawer clears the lid overhang"+ - id: art_med_camera_tripod+ tier: medium+ category: props+ prompt: >+ a camera tripod: three legs that splay at the crown, a sliding lower section+ per leg, and a pan-tilt head with a mounting plate+ must_have:+ - "3 legs, each splaying by revolute from vertical to 25 degrees at the crown"+ - "each leg's lower section slides out 0.40 m along the leg axis"+ - "pan joint at the head rotates continuously about the vertical axis"+ - "tilt joint pitches the mounting plate from -90 to +30 degrees"+ - "rest pose: legs at equal splay and full extension, mounting plate level about 1.4 m up"+ - "rubber feet at all 3 leg tips, resting on the ground plane in the rest pose"+ - id: art_med_dutch_door+ tier: medium+ category: architecture+ prompt: >+ a dutch stable door in its frame: independent upper and lower leaves on the+ same jamb, a slide bolt that joins them, and a latch handle on the lower leaf+ must_have:+ - "door frame with both leaves fitted, total opening about 2.0 m tall"+ - "upper leaf swings open 100 degrees on hinges at the jamb"+ - "lower leaf swings open 100 degrees on hinges at the same jamb"+ - "slide bolt on the upper leaf travels 0.08 m down into the lower leaf edge"+ - "bolt at its extended limit spans the seam into a notch in the lower leaf edge (leaves closed at rest)"+ - "shelf ledge on top of the lower leaf clears the swinging upper leaf in every sweep pose"+ - "latch handle on the lower leaf"+ # -------------------------------------------------------------------- hard+ - id: art_hard_rolltop_desk+ tier: hard+ category: furniture+ prompt: >+ a roll-top desk: a tambour of hinged slats that retracts along curved side+ tracks to expose pigeonholes and the desktop, plus two pedestal drawers+ per side+ must_have:+ - "desk carcass with 2 pedestals and a knee-hole"+ - "tambour approximated as at least 8 slats riding in curved side tracks"+ - "tambour travels from fully closed at rest to fully open at its joint limit, moving toward the rear"+ - "in the rest pose the closed tambour's bottom rail meets the desktop with under 0.01 m gap"+ - "at the tambour's open limit at least 4 pigeonhole compartments are exposed"+ - "2 drawers per pedestal, each slides 0.30 m out toward the front"+ - "open drawers do not intersect the knee-hole space or each other"+ - "drawer pulls and a writing surface about 0.75 m above the floor"+ - id: art_hard_folding_workbench+ tier: hard+ category: tools+ prompt: >+ a folding workbench: a top on two leg frames that fold inward, each held+ open by a diagonal locking brace with an elbow pivot+ must_have:+ - "top about 1.2 m by 0.6 m, work height 0.85 m when deployed"+ - "2 leg frames, each folding inward by revolute at the top rail"+ - "2 locking braces, each with an elbow revolute that snaps straight when deployed"+ - "each brace elbow folds through at least 90 degrees"+ - "each leg frame at its folded limit lies within 0.15 m of the top's underside, with no collision in any sweep pose"+ - "rest pose: deployed, all 4 feet on the ground and braces straight"+ - id: art_hard_parallel_clamp+ tier: hard+ category: tools+ prompt: >+ a parallel-jaw bench clamp: a fixed jaw and a moving jaw on two guide rods,+ driven by a lead screw with a crank handle and a swivel foot pad+ must_have:+ - "fixed jaw body and moving jaw about 0.12 m wide"+ - "2 guide rods passing through the moving jaw"+ - "lead screw between the rods, crank handle rotates continuously"+ - "moving jaw slides 0.12 m along the guide rods toward the fixed jaw"+ - "jaw faces remain parallel over the whole travel"+ - "swivel pad on the screw tip tilts up to 5 degrees against the fixed jaw"+ - "at the moving jaw's closed limit the jaw faces meet with no interpenetration"+ - id: art_hard_sash_window+ tier: hard+ category: architecture+ prompt: >+ a double-hung sash window: two counterweighted sashes sliding in separate+ tracks of one frame, with a rotating lock lever on the meeting rail+ must_have:+ - "outer frame about 0.9 m by 1.5 m with sill and casing"+ - "2 sashes, each divided into at least 4 panes by muntins"+ - "lower sash slides up 0.40 m in the inner track"+ - "upper sash slides down 0.35 m in the outer track"+ - "sashes pass each other with visible clearance, never touching"+ - "lock lever on the meeting rail rotates 90 degrees"+ - "lower sash at its upper limit overlaps the upper sash, both visible in front view"+ - id: art_hard_grand_piano+ tier: hard+ category: musical_instruments+ prompt: >+ a baby grand piano with a hinged main lid held by a prop stick, a rotating+ fallboard over the keyboard, three pedals on a lyre, and a bench with a+ hinged storage lid+ must_have:+ - "curved-rim piano body on 3 legs"+ - "keyboard with white and black key rows exposed when the fallboard is open"+ - "fallboard rotates 90 degrees from covering the keys to lying flat behind them"+ - "main lid hinged along the straight side, held open about 30 degrees at its lower limit, hinging up to about 45 degrees"+ - "prop stick pivots up from the rim; at its raised limit the stick tip meets the lid's underside within 0.01 m"+ - "prop stick at rest lies flat inside the rim"+ - "3 pedals on a lyre post under the body"+ - "bench with a storage lid hinged at the rear, opening 60 degrees"+ - id: art_hard_umbrella+ tier: hard+ category: props+ prompt: >+ an umbrella with a simplified rib mechanism: a runner sliding on the shaft+ drives at least six ribs that pivot at the crown, with canopy panels+ stretched between the ribs+ must_have:+ - "central shaft about 0.9 m with a curved handle"+ - "runner slides 0.30 m up the shaft"+ - "at least 6 ribs, each revolute at the crown"+ - "ribs swing from folded along the shaft to 80 degrees open"+ - "stretcher links connect the runner to each rib mid-point in the open rest pose"+ - "rest pose: open, with canopy panels stretched between adjacent ribs"+ - "each rib at its folded limit lies within 0.05 m of the shaft without crossing it"+ - id: art_hard_casement_window+ tier: hard+ category: architecture+ prompt: >+ a casement window driven by a sill crank: the sash swings outward on side+ hinges while a folding operator arm with a sliding shoe pushes its bottom rail+ must_have:+ - "frame about 0.6 m by 1.2 m with a gridded sash"+ - "sash swings outward 90 degrees on hinges at one side edge"+ - "crank handle at the sill rotates continuously"+ - "operator arm with an elbow revolute links the sill to the sash bottom rail"+ - "operator arm shoe slides 0.30 m along a track in the sash bottom rail"+ - "rest pose: window closed, operator arm folded flat over the sill without touching the sash"+ - "latch lever on the free edge rotates to lock the sash"+ - id: art_hard_step_ladder+ tier: hard+ category: tools+ prompt: >+ an A-frame step ladder: a five-step front frame and a rear frame hinged at+ the top cap, held open by two spreader arms with center elbows+ must_have:+ - "front frame with 5 flat steps"+ - "rear frame hinged to the front frame at the top cap"+ - "frames open from parallel to about 30 degrees apart"+ - "2 spreader arms, each with a center elbow revolute"+ - "rest pose: open, spreaders locked straight between the frames"+ - "rear frame at its closed limit lies parallel to the front frame within 0.10 m; spreaders fold without hitting the steps in any sweep pose"+ - "top cap platform and 4 feet touching the ground in the open rest pose"+ - id: art_hard_drawleaf_table+ tier: hard+ category: furniture+ prompt: >+ a draw-leaf dining table: two end leaves stowed under the fixed center top+ slide out horizontally on tapered runners, plus an apron drawer on one long+ side+ must_have:+ - "fixed center top about 1.2 m by 0.9 m on 4 legs, surface about 0.75 m high"+ - "2 end leaves about 0.45 m deep stowed under the center top at rest"+ - "each leaf slides 0.45 m outward along the table's long axis"+ - "each leaf rides on 2 tapered runners that emerge from under the top as it extends"+ - "extended leaf clears the center top edge with a gap under 0.02 m"+ - "apron drawer on one long side slides 0.25 m out"+ - "leaves and drawer extend without touching the legs or each other in any sweep pose"+ - "rest pose: both leaves fully stowed inside the center top's footprint"+ - id: art_hard_scissor_mirror+ tier: hard+ category: household+ prompt: >+ a wall-mounted scissor extension mirror: a pantograph of three X-links+ between a wall plate and a mirror head that also swivels and tilts+ must_have:+ - "wall plate anchoring the mechanism"+ - "pantograph of at least 3 X-links with revolute pivots at centers and ends"+ - "rest pose: mirror head retracted 0.15 m from the wall plate; each pantograph joint at its limit moves the head farther from the wall"+ - "all pantograph bars lie in a horizontal plane, scissoring parallel to the floor"+ - "mirror head swivels at least 90 degrees about its vertical post"+ - "mirror tilts at least 30 degrees about its horizontal axis"+ - "X-links nest without bar intersections in the retracted rest pose"diff --git a/harness/bench/prompts/compare_v2.yaml b/harness/bench/prompts/compare_v2.yamlnew file mode 100644index 00000000..0b0b612e--- /dev/null+++ b/harness/bench/prompts/compare_v2.yaml@@ -0,0 +1,176 @@+# compare_v2 — harness-vs-oneshot arms on the v2 static battery.+#+# 8 prompts drawn verbatim from static_objects_v2 (same prompt text, must_have and+# dimensions): 2 medium controls + 6 hard, one per difficulty axis+# (decomposition, thin features x2, organic, architectural detail, multi-material).+# must_have is the FIXED judge's acceptance checklist for every arm (no plan-derived+# items) so harness and one-shot are judged alike; compare_v1 showed one-shot arms+# separating mostly on build success — the v2 hard set should separate surviving+# builds on assembly_fit and geometry_detail instead.+# Expected discrimination: controls pass for strong one-shot models; hard prompts+# 0-25% one-shot pass, with the harness loop's refine delta as the headline metric.+name: compare_v2+track: static_object+language: blender+description: >+ 8 static-object prompts (blender / bpy) shared with static_objects_v2, used to+ compare the harness loop against raw one-shot generation under ONE fixed judge.+ 2 medium controls + 6 hard across the v2 difficulty axes.+prompts:+ - id: cmp_med_dining_chair+ tier: medium+ category: furniture+ tags: [control, "axis:decomposition"]+ prompt: a mid-century wooden dining chair with a slatted backrest and tapered legs+ must_have:+ - "four tapered legs, each thinner at the floor than at the seat"+ - "seat slab about 0.42 m wide"+ - "seat surface about 0.45 m above the floor"+ - "backrest with at least 3 vertical slats"+ - "back legs continue up into the backrest posts"+ - "backrest top rail between 0.80 m and 0.95 m above the floor"+ - "all four feet touch the ground plane"+ - "no leg or slat intersects the seat slab"+ dimensions_m: {width: 0.45, depth: 0.50, height: 0.90}+ - id: cmp_med_toaster+ tier: medium+ category: appliances+ tags: [control, "axis:curved_organic"]+ prompt: a two-slot retro toaster with rounded chrome body, a lever on the side, a browning dial and a crumb tray+ must_have:+ - "2 bread slots on the top face"+ - "each slot about 0.13 m long and under 0.04 m wide"+ - "lever on one end face"+ - "round dial knob on the front face"+ - "rounded body with no sharp vertical edges"+ - "crumb tray seam line near the base"+ - "4 small feet"+ - "body about 0.28 m wide and 0.19 m tall"+ dimensions_m: {width: 0.28, depth: 0.17, height: 0.19}+ - id: cmp_hard_coffee_grinder+ tier: hard+ category: mechanisms+ tags: ["axis:decomposition"]+ prompt: >+ a hand-crank coffee grinder: square wooden box body with a small front drawer,+ cast-iron dome top with an open hopper funnel, a conical burr visible inside+ the funnel throat, and a top crank arm with a wooden knob+ must_have:+ - "square wooden box body about 0.14 m wide and 0.12 m tall"+ - "front drawer about 0.05 m tall with a round knob, in the lower half of the box"+ - "drawer front sits flush in its opening with a visible seam"+ - "metal dome top mounted on the box"+ - "open hopper funnel about 0.08 m across at the rim"+ - "conical burr visible inside the funnel throat"+ - "central spindle about 0.01 m thick rising from the burr through the dome"+ - "crank arm about 0.11 m long attached to the spindle top"+ - "wooden knob on a short vertical pin at the crank's free end"+ - "adjustment nut on the spindle above the crank hub"+ dimensions_m: {width: 0.14, depth: 0.14, height: 0.30}+ - id: cmp_hard_garden_arch+ tier: hard+ category: garden+ tags: ["axis:thin_features"]+ prompt: >+ a wrought-iron garden arch: two lattice side frames, a semicircular arch head+ filled with S-scrolls, spear-point finials and a tie bar at the spring line+ must_have:+ - "2 side frames, each of 2 uprights about 0.01 m thick"+ - "diamond lattice bars between the uprights of each side frame"+ - "semicircular arch head spanning about 1.2 m between the side frames"+ - "at least 6 S-scrolls filling the arch head"+ - "at least 4 scrolls in each side frame"+ - "3 spear-point finials on top of the arch"+ - "horizontal tie bar connecting both sides at the spring line"+ - "small ground plates under all 4 upright feet"+ - "overall height about 2.3 m"+ - "all bars slender: no member thicker than 0.02 m"+ dimensions_m: {width: 1.4, depth: 0.5, height: 2.3}+ - id: cmp_hard_violin+ tier: hard+ category: musical_instruments+ tags: ["axis:thin_features", "axis:curved_organic"]+ prompt: >+ a violin: hourglass body with C-bout waists, arched top with two f-holes, a+ bridge, four strings from tailpiece over the bridge along the fingerboard to+ the pegbox, a scroll head, four tuning pegs and a chin rest+ must_have:+ - "hourglass body with 2 concave C-bout waist cutaways"+ - "body length about 0.36 m"+ - "top plate visibly arched, not flat"+ - "2 f-holes through the top, one each side of the bridge"+ - "bridge about 0.03 m tall standing upright between the f-holes"+ - "4 strings running in parallel from tailpiece to pegbox"+ - "strings pass over the bridge with a visible break angle"+ - "black fingerboard extending over the body without touching it"+ - "spiral scroll at the head"+ - "4 tuning pegs, 2 per side of the pegbox"+ - "tailpiece anchored toward the end button"+ - "chin rest beside the tailpiece"+ - "total length about 0.60 m"+ dimensions_m: {length: 0.60, width: 0.21, height: 0.09}+ - id: cmp_hard_bonsai_rock+ tier: hard+ category: plants+ tags: ["axis:curved_organic"]+ prompt: >+ a root-over-rock bonsai: a shallow oval pot with soil, a rounded rock gripped+ by exposed roots, a tapering trunk with two pronounced curves, and five+ distinct flattened foliage pads+ must_have:+ - "shallow oval pot about 0.30 m long with a rim and small feet"+ - "soil surface inside the pot"+ - "rounded rock about 0.12 m tall sitting on the soil"+ - "at least 4 roots gripping over the rock faces and entering the soil"+ - "trunk rising from the rock top with at least 2 pronounced curves"+ - "trunk tapers from about 0.04 m at the base to under 0.015 m at the apex"+ - "5 distinct flattened foliage pads with gaps between them"+ - "lowest branch extends beyond the pot rim"+ - "no foliage pad touches the soil or the rock"+ - "overall height about 0.50 m"+ dimensions_m: {height: 0.50, length: 0.45}+ - id: cmp_hard_bay_window+ tier: hard+ category: buildings+ tags: ["axis:arch_detail", "axis:decomposition"]+ prompt: >+ a Victorian bay window facade section: a three-sided bay projecting from a+ clapboard wall, a sash window with divided panes in each face, panel aprons,+ corner pilasters, scroll brackets and a dentil cornice under a faceted roof cap+ must_have:+ - "three-sided bay projecting from a flat wall section"+ - "center face wider than the 2 angled side faces"+ - "side faces angled about 45 degrees from the wall"+ - "sash window in each of the 3 faces"+ - "upper sash of each window divided into at least 4 panes by muntins"+ - "continuous sill band under all 3 windows"+ - "recessed panel apron under each window"+ - "pilasters at the 4 vertical bay corners"+ - "cornice with at least 10 dentil blocks across the bay"+ - "at least 4 scroll brackets under the bay roof"+ - "three-faceted roof cap over the bay"+ - "horizontal siding boards on the flat wall section"+ dimensions_m: {width: 3.4, height: 3.2, depth: 1.0}+ - id: cmp_hard_drafting_table+ tier: hard+ category: furniture+ tags: ["axis:multi_material", "axis:decomposition"]+ prompt: >+ a drafting table setup: tilted drawing board on a trestle frame with quadrant+ tilt brackets, a parallel ruler riding on side rails, a pen tray, a sheet of+ paper held by two corner clips, a clamp-on two-arm lamp aimed at the board,+ and a stool with a footring+ must_have:+ - "drawing board about 1.0 m by 0.7 m tilted about 20 degrees"+ - "parallel ruler bar spanning the board width"+ - "ruler rides on a rail or wire at each side edge"+ - "pen tray along the bottom edge of the board"+ - "trestle legs with a cross stretcher"+ - "quadrant tilt bracket with a knob at each of the board's 2 sides"+ - "sheet of paper on the board held by clips at 2 corners"+ - "lamp clamped to a top corner of the board"+ - "lamp has 2 arm segments and a shade aimed down at the board"+ - "stool with a round seat about 0.65 m high"+ - "stool footring between its legs"+ - "board top edge about 1.1 m above the floor"+ dimensions_m: {width: 1.1, depth: 0.9, height: 1.2}diff --git a/harness/bench/prompts/graphics_v1.yaml b/harness/bench/prompts/graphics_v1.yamlnew file mode 100644index 00000000..b2ad1acc--- /dev/null+++ b/harness/bench/prompts/graphics_v1.yaml@@ -0,0 +1,30 @@+# Graphics track battery v1 — GLSL fragment shaders + raw OpenGL programs.+name: graphics_v1+track: graphics+language: glsl_shader+prompts:+ - id: glsl_easy_lava_lamp+ tier: easy+ prompt: "A relaxing lava-lamp shader: slowly rising and merging warm blobs over a dark gradient background, soft glow"+ must_have: ["blobs that move and merge over time", "warm orange/pink palette on a dark background"]+ - id: glsl_easy_ocean_sunset+ tier: easy+ prompt: "A stylised ocean sunset: layered animated waves, sun disk with shimmering reflection path, gradient sky with a few clouds"+ must_have: ["animated waves", "sun reflection path on the water"]+ - id: glsl_med_neon_rain+ tier: medium+ prompt: "Cyberpunk rain on a window at night: streaking raindrops refracting neon city bokeh lights, occasional lightning flash"+ must_have: ["raindrop streaks that run downward", "colourful bokeh lights", "brightness varies over time"]+ - id: glsl_med_nebula+ tier: medium+ prompt: "A deep-space nebula flythrough: layered volumetric clouds with domain warping, twinkling star field, slow forward drift"+ must_have: ["parallax star field", "warped nebula clouds", "continuous forward motion"]+ - id: glsl_hard_raymarch_temple+ tier: hard+ prompt: "A raymarched ancient temple interior: columns, god rays through an opening, drifting dust, soft shadows, slow camera orbit"+ must_have: ["recognisable columns", "visible light shafts", "camera motion"]+ - id: ogl_med_particle_galaxy+ tier: medium+ language: opengl_python+ prompt: "An instanced particle galaxy: 20k points orbiting in spiral arms with additive glow, slowly rotating camera"+ must_have: ["spiral arm structure", "rotation over time"]diff --git a/harness/bench/prompts/graphics_v2.yaml b/harness/bench/prompts/graphics_v2.yamlnew file mode 100644index 00000000..4d8bcceb--- /dev/null+++ b/harness/bench/prompts/graphics_v2.yaml@@ -0,0 +1,170 @@+# graphics_v2 — shader / OpenGL battery.+#+# Difficulty axes:+# raymarching depth full raymarched scenes with material response (SSS-ish glow,+# fractal surface detail, correct-looking gear meshing) — v1's+# only raymarch prompt scored 0.30, the strongest discriminator+# feedback state multi-pass programs whose look emerges from a feedback buffer+# (reaction-diffusion, ink advection) — cannot be faked in one pass+# stylisation control hitting a named art style with countable features (ukiyo-e wave,+# CRT boot) instead of generic noise beauty+# instancing scale opengl_python prompts with explicit instance counts and a+# structured motion the VLM can verify across frames+# Per-prompt `language: opengl_python` marks the raw-OpenGL prompts (battery default+# is glsl_shader); the bench loader honours the per-prompt override (BenchPrompt.language).+# Expected discrimination: medium 40-70% pass; hard 0-30% pass — feedback-buffer and+# instancing prompts also stress the build/runtime, where v1's opengl prompt errored.+name: graphics_v2+track: graphics+language: glsl_shader+description: >+ 10 graphics prompts: 3 medium stylised shaders, 7 hard (raymarched scenes,+ feedback-buffer simulations, large-scale instancing). must_have items are+ frame-verifiable features with counts and motion statements.+prompts:+ # ------------------------------------------------------------- raymarched (hard)+ - id: glsl_hard_mushroom_forest+ tier: hard+ category: raymarch+ prompt: >+ a raymarched night mushroom forest: giant glowing mushrooms whose caps+ transmit light like wax, thin stems, drifting spore motes, soft shadows and+ a slow camera glide between the stems+ must_have:+ - "at least 5 mushrooms with domed caps on visible stems"+ - "caps glow from inside with a soft falloff toward the rim, brighter where thin"+ - "ground plane receiving soft shadows from the stems"+ - "drifting bright spore motes between the mushrooms"+ - "dark blue or violet night ambience with the caps as main light sources"+ - "camera glides forward between the stems over the loop"+ - id: glsl_hard_infinite_corridor+ tier: hard+ category: raymarch+ prompt: >+ a raymarched infinite corridor: repeating arched segments receding to a+ vanishing point, walls carved with self-similar fractal relief that gains+ detail as the camera approaches, cool fog in the distance, endless forward+ motion+ must_have:+ - "corridor of repeating arched segments receding to a vanishing point"+ - "at least 6 segment repetitions visible at once"+ - "wall surfaces show self-similar relief with at least 3 visible scales of detail"+ - "distance fog fades the far segments"+ - "forward motion continues across all sampled frames without the camera clipping the walls"+ - "light sources along the corridor alternate or repeat with the segments"+ - id: glsl_hard_clockwork_gears+ tier: hard+ category: raymarch+ prompt: >+ a raymarched brass clockwork: at least three meshing gears with visible+ teeth turning at consistent ratios, a slowly swinging pendulum disc behind+ them, metallic highlights and a dark workshop background+ must_have:+ - "at least 3 gears with clearly separated teeth"+ - "adjacent gears rotate in opposite directions"+ - "a small gear visibly turns faster than a large one it meshes with"+ - "teeth of meshing gears interleave without overlapping"+ - "pendulum disc swings periodically behind the gears"+ - "metallic specular highlights on the gear faces"+ # ---------------------------------------------------- feedback simulations (hard)+ - id: ogl_hard_reaction_diffusion+ tier: hard+ category: simulation+ language: opengl_python+ prompt: >+ a reaction-diffusion coral garden grown in a feedback buffer: patterns seeded+ at several points spread outward into labyrinthine coral growth, shaded with+ height-like lighting and an ocean palette, growth visible over the run+ must_have:+ - "labyrinthine or spotted pattern characteristic of reaction-diffusion"+ - "pattern grows outward from at least 3 seed points over time"+ - "later frames show strictly more covered area than early frames"+ - "pattern edges shaded with a relief or rim-light effect"+ - "ocean palette: teal to orange or pink gradient mapped to the pattern"+ - "the pattern boundary advances in every successive sampled frame"+ - id: ogl_hard_ink_swirls+ tier: hard+ category: simulation+ language: opengl_python+ prompt: >+ fluid ink in water via a feedback advection buffer: dark ink blobs injected+ periodically curl into filament swirls that stretch and fold without+ dissipating instantly, on a paper-white background+ must_have:+ - "ink blobs appear at intervals during the run"+ - "blobs stretch into curling filaments over subsequent frames"+ - "filaments fold back on themselves forming spiral structures"+ - "earlier ink stays visible as faded wisps while new ink is injected"+ - "paper-white background with dark ink, no full-frame darkening over time"+ # ------------------------------------------------------------- stylised (medium)+ - id: glsl_med_ukiyoe_wave+ tier: medium+ category: stylised+ prompt: >+ a looping ukiyo-e style great wave: a large curling crest with claw-like foam+ fingers, flat indigo color bands with paper-cream foam, smaller waves in+ front, and a seamless loop where the crest rises, curls and collapses+ must_have:+ - "one dominant wave with a forward-curling crest"+ - "foam edge broken into claw-like fingers"+ - "flat color bands in indigo and cream, no photoreal gradients"+ - "at least 2 smaller waves in the foreground"+ - "across the sampled frames the crest rises, curls forward and collapses"+ - id: glsl_med_crt_boot+ tier: medium+ category: stylised+ prompt: >+ a retro CRT boot sequence: black screen flickers on with a horizontal flash,+ green blocky glyph rows type in line by line, a blinking cursor, scanlines,+ barrel distortion at the screen corners and phosphor glow+ must_have:+ - "earliest sampled frame shows the turn-on state: mostly dark or a bright horizontal flash band"+ - "rows of blocky glyphs appear line by line down the screen"+ - "blinking rectangular cursor at the end of the last row"+ - "visible scanlines across the whole frame"+ - "image bulges with barrel distortion, corners darkened"+ - "green phosphor glow bleeding around bright glyphs"+ - id: glsl_med_kaleido_mandala+ tier: medium+ category: stylised+ prompt: >+ a kaleidoscopic mandala: 12-fold mirror symmetry, at least three concentric+ pattern rings rotating at different speeds in alternating directions, petal+ shapes morphing smoothly, jewel palette on a dark background+ must_have:+ - "12-fold mirror symmetry about the center"+ - "at least 3 concentric pattern rings"+ - "adjacent rings rotate in opposite directions at different speeds"+ - "petal or lobe shapes morph smoothly over time"+ - "saturated jewel palette on a dark background, center stays centered"+ # ------------------------------------------------------ instancing scale (hard)+ - id: ogl_hard_boids_murmuration+ tier: hard+ category: instancing+ language: opengl_python+ prompt: >+ a starling murmuration of 30000 instanced boids over a dusk landscape+ silhouette: the flock stretches, folds and splits into lobes that merge+ again, dense cores darker than the edges, slow camera pan+ must_have:+ - "flock of thousands of small instances, far more than any countable number"+ - "flock forms one connected shape that stretches and folds over time"+ - "flock splits into at least 2 lobes and merges again during the run"+ - "denser flock regions read darker or thicker than sparse edges"+ - "dusk gradient sky with a dark landscape silhouette at the bottom"+ - "camera pans slowly while the flock stays in frame"+ - id: ogl_hard_voxel_city+ tier: hard+ category: instancing+ language: opengl_python+ prompt: >+ a fly-through of an instanced voxel city with a full day-night cycle:+ cube-built towers of varying heights on a street grid, sun light and long+ shadows by day, lit windows by night, the camera weaving between towers+ must_have:+ - "city of cube-built towers on a regular street grid"+ - "at least 50 towers of clearly varying heights"+ - "continuous camera flight between the towers without entering any tower"+ - "day phase: directional sun with visible long shadows"+ - "night phase: dark sky with emissive lit windows on the towers"+ - "smooth day-to-night transition within the run"diff --git a/harness/bench/prompts/scenes_v2.yaml b/harness/bench/prompts/scenes_v2.yamlnew file mode 100644index 00000000..d055fe46--- /dev/null+++ b/harness/bench/prompts/scenes_v2.yaml@@ -0,0 +1,227 @@+# scenes_v2 — composition-contract battery.+#+# Every scene states an explicit COMPOSITION CONTRACT in must_have:+# * 4-6 named zones (spatial layout the judge can point at in wide shots)+# * 8-14 distinct asset types WITH COUNTS ("at least 12 market stalls of 3 designs")+# * a lighting mood the VLM can verify in renders, annotated with an approximate+# luminance band (normalised 0-1); the frame-metrics gate enforces the floor —+# it errors on authored frames with mean luminance < 0.12 or > 20% blown pixels,+# and those gate errors reach the judge (the VLM itself cannot measure a band)+# * one water / weather / particle effect+# * one animation+# * a camera brief (path + height + no-clip requirement; camera-in-geometry and+# exposure were the top audited defect classes in the reference scene corpus)+# Difficulty axes: dense scene composition (asset count x variety), lighting-mood+# control under a luminance band, effect + animation integration, camera authoring.+# Expected discrimination: scenes_v1 EASY already failed 0/4 with 3.7-flash (missing+# animation, exposure, world-edge defects), so flash-class pass rates here may be 0+# across tiers; the battery discriminates on score spread (zone coverage, lighting+# compliance, camera hygiene) — medium is expected to score 0.5-0.7, hard 0.3-0.6.+name: scenes_v2+track: scene+language: scene_threejs+description: >+ 10 scene prompts (multi-file Three.js + GLSL), 3 medium / 7 hard. Each carries a+ composition contract: named zones, asset types with counts, a measurable lighting+ band, one water/weather/particle effect, one animation and a camera brief.+prompts:+ - id: scn_hard_night_market+ tier: hard+ category: urban+ prompt: >+ a bustling night market street: an entrance gate, lantern-lit food lane,+ games row, a small ferris wheel plaza at the far end, steam rising from+ the food stalls and a slow walk-through camera+ must_have:+ - "4 zones along the street: entrance gate, food lane, games row, ferris wheel plaza"+ - "at least 12 market stalls using at least 3 distinct stall designs"+ - "at least 30 glowing lanterns strung above the lane"+ - "ferris wheel with at least 8 gondolas at the far plaza"+ - "at least 8 simplified pedestrian figures in the lane"+ - "string lights connecting stall roofs"+ - "food props on at least 4 stall counters"+ - "night mood: dark sky with lantern glow as the dominant light, no black or blown-out frames (mean luminance roughly 0.15-0.45)"+ - "steam particles rising from at least 2 food stalls"+ - "ferris wheel rotates continuously"+ - "camera dollies forward down the lane at 1.6 m height, visibly advancing between the sampled times"+ - "camera never intersects stalls, figures or lanterns"+ - id: scn_hard_canal_town+ tier: hard+ category: urban+ prompt: >+ a canal town at golden hour: gabled rowhouses along both quays, an arched+ stone bridge, moored and drifting boats, a cafe terrace, warm low sun with+ long reflections on the water+ must_have:+ - "5 zones: canal, left quay, right quay, arched bridge, cafe terrace"+ - "at least 10 gabled rowhouses using at least 3 facade variants"+ - "at least 5 boats, at least 3 moored at bollards"+ - "arched stone bridge spanning the canal"+ - "cafe terrace with at least 6 tables and umbrellas"+ - "at least 8 lamp posts along the quays"+ - "at least 6 bicycles leaning at the quay railings"+ - "water surface with ripples and sun reflections"+ - "golden hour mood: low warm sun with long shadows, mid-bright frames, none black or blown out (mean luminance roughly 0.30-0.60)"+ - "one boat drifts along the canal and passes under the bridge"+ - "camera cranes upward from quay level toward the bridge crest between the sampled times"+ - "camera never clips the bridge or house facades"+ - id: scn_hard_cliff_monastery+ tier: hard+ category: landscape+ prompt: >+ a monastery clinging to a cliff face in fog: switchback stair path, a+ gatehouse, main hall on a terrace, a bell tower, fog banks drifting below+ the buildings+ must_have:+ - "5 zones: cliff face, switchback stair path, gatehouse, main hall terrace, bell tower"+ - "at least 4 buildings with tiled roofs stepped down the cliff"+ - "at least 6 stair flights zigzagging up the cliff"+ - "railing or rope line along the stair path"+ - "at least 8 pine trees growing from cliff ledges"+ - "strings of flags or banners between buildings"+ - "hanging bell visible in the tower"+ - "layered fog banks below the terrace, partially hiding the cliff base"+ - "muted mood: soft diffuse fog light without hard shadows, mid-bright frames (mean luminance roughly 0.35-0.65)"+ - "fog banks drift slowly and the flags flutter"+ - "camera flies in through the fog toward the gatehouse, visibly closer to the terrace at the later sampled time"+ - "camera never enters the cliff or building geometry"+ - id: scn_med_conservatory+ tier: medium+ category: interior+ prompt: >+ a botanical conservatory interior: glass and iron rotunda, palm court,+ koi pond with a fountain, fern aisle and a succulent bench, bright light+ shafts through the glazing+ must_have:+ - "5 zones: entry rotunda, palm court, koi pond, fern aisle, succulent bench"+ - "glazed roof and walls with a visible iron frame grid"+ - "at least 6 palms in the court"+ - "at least 20 potted ferns along the aisle"+ - "at least 8 hanging baskets under the roof"+ - "koi pond with a central fountain"+ - "at least 4 benches along the paths"+ - "gravel or tile path linking all zones"+ - "bright mood: sunlit glasshouse with visible light shafts, bright but not blown-out frames (mean luminance roughly 0.45-0.75)"+ - "fountain water animates with ripples spreading in the pond"+ - "visible dust motes in the light shafts"+ - "camera walks the aisle from the rotunda toward the pond at 1.6 m height, advancing between the sampled times"+ - id: scn_hard_shipyard_dawn+ tier: hard+ category: industrial+ prompt: >+ a wooden shipyard at dawn: a partially planked hull on a slipway cradle,+ a gantry crane, scaffolding, timber stacks, a workshop shed with a smoking+ chimney, calm water at the slipway foot+ must_have:+ - "5 zones: slipway with hull, crane pad, timber yard, workshop shed, waterline"+ - "ship hull with exposed ribs and partial planking on a cradle"+ - "gantry or jib crane beside the hull"+ - "scaffolding with at least 6 bays along the hull"+ - "at least 5 timber stacks in the yard"+ - "at least 12 barrels grouped near the shed"+ - "workshop shed with a chimney"+ - "2 rowboats at the waterline"+ - "dawn mood: low warm sun raking the hull, dim-to-mid frames, none black or blown out (mean luminance roughly 0.25-0.55)"+ - "calm water with reflections at the slipway foot"+ - "smoke rises from the shed chimney while the crane jib slews slowly"+ - "camera orbits the hull at half its height without clipping the scaffolding"+ - id: scn_hard_alpine_station+ tier: hard+ category: landscape+ prompt: >+ an alpine railway station in snowfall: a lamp-lit platform, timber station+ house, track line with a signal mast, a snow shed, snow-loaded pines on the+ slope behind+ must_have:+ - "5 zones: platform, station house, track line, snow shed, pine slope"+ - "timber station house with snow-covered roof and lit windows"+ - "at least 6 platform lamps with warm glow"+ - "at least 4 benches on the platform"+ - "2 luggage carts near the door"+ - "track with visible sleepers running past the platform into a snow shed"+ - "signal mast beside the track"+ - "at least 15 snow-loaded pines on the slope"+ - "overcast mood: flat grey daylight contrasting with warm lamp glow, mid-bright frames (mean luminance roughly 0.35-0.65)"+ - "continuous falling snow particles"+ - "signal arm position differs between the sampled times and the platform lamps glow warmly"+ - "camera tracks along the platform edge at 1.6 m without clipping lamps or benches"+ - id: scn_med_bazaar_noon+ tier: medium+ category: urban+ prompt: >+ a bazaar courtyard at noon: arcaded perimeter, carpet display, spice stalls+ under awnings, a central fountain and a tea corner, hard sun with crisp+ shadows+ must_have:+ - "5 zones: arcade perimeter, carpet display, spice stalls, central fountain, tea corner"+ - "arcade with at least 8 arches around the courtyard"+ - "at least 10 hanging or draped carpets in at least 4 patterns"+ - "at least 12 spice sacks or bowls at the stalls"+ - "at least 6 awnings shading the stalls"+ - "central fountain with a basin"+ - "at least 8 pottery stacks around the arcade"+ - "tea corner with a low table and cushions"+ - "noon mood: hard sun with short crisp shadows, bright but not blown-out frames (mean luminance roughly 0.45-0.75)"+ - "fountain water animates"+ - "awnings ripple in a light breeze"+ - "camera circles the fountain at 2 m height without clipping the awnings"+ - id: scn_hard_neon_arcade_street+ tier: hard+ category: urban+ prompt: >+ a neon arcade street in rain at night: glowing signs stacked up the+ facades, an arcade entrance with cabinets visible inside, a side alley,+ wet reflective asphalt and falling rain+ must_have:+ - "4 zones: main street, arcade entrance, side alley, elevated walkway"+ - "at least 10 emissive neon signs in at least 4 distinct shapes"+ - "arcade entrance with at least 4 cabinets visible through the front"+ - "3 vending machines along the street"+ - "at least 6 puddles on the asphalt"+ - "at least 5 simplified figures with umbrellas"+ - "power lines or cables crossing overhead"+ - "wet ground shader reflecting the neon signs"+ - "night mood: neon glow dominant over a dark wet street, no fully black frames (mean luminance roughly 0.15-0.40)"+ - "falling rain streaks"+ - "at least 2 neon signs differ in on/off state between the sampled times"+ - "camera pushes slowly down the street at 1.6 m, never clipping figures or machines"+ - id: scn_med_rice_terraces+ tier: medium+ category: landscape+ prompt: >+ terraced rice fields at sunset: stacked water-filled terraces mirroring the+ sky, a farmhouse, a footbridge over an irrigation channel, palm cluster and+ a path down the terraces+ must_have:+ - "5 zones: upper terraces, irrigation channel, farmhouse, palm cluster, path"+ - "at least 7 terrace levels with water surfaces"+ - "terrace water reflects the sunset sky"+ - "rows of rice tufts on at least 4 terraces"+ - "farmhouse with a pitched roof at mid-slope"+ - "footbridge over the channel"+ - "2 sluice gates in the channel"+ - "scarecrow on an upper terrace"+ - "at least 5 palms in the cluster"+ - "sunset mood: sun near the horizon mirrored in the terrace water, dim-to-mid frames, none black (mean luminance roughly 0.25-0.55)"+ - "water shimmer animates and 3 birds fly across the sky"+ - "camera descends along the terraces from above the farmhouse to the path"+ - id: scn_hard_lighthouse_storm+ tier: hard+ category: landscape+ prompt: >+ a lighthouse island in a storm: banded lighthouse with a gallery and lamp+ room, keeper's cottage, boathouse, wooden jetty on piles, heavy seas with+ whitecaps, rain and spray+ must_have:+ - "5 zones: rock island, lighthouse, keeper's cottage, boathouse, jetty"+ - "lighthouse with color bands, gallery railing and glazed lamp room"+ - "keeper's cottage with at least 3 lit windows"+ - "boathouse with a ramp to the water"+ - "jetty on at least 8 piles"+ - "at least 15 rocks around the island shore"+ - "sea surface with large animated waves and whitecaps"+ - "spray bursts where waves hit the rocks"+ - "storm mood: dark sky and sea with the rotating beam as the brightest element, no fully black frames (mean luminance roughly 0.15-0.40)"+ - "falling rain driven at an angle"+ - "lamp room beam rotates and sweeps the sea"+ - "camera holds offshore at wave-top height with a slow heave, never entering the water volume"diff --git a/harness/bench/prompts/static_objects_v2.yaml b/harness/bench/prompts/static_objects_v2.yamlnew file mode 100644index 00000000..bf9f339d--- /dev/null+++ b/harness/bench/prompts/static_objects_v2.yaml@@ -0,0 +1,425 @@+# static_objects_v2 — hard-tier battery (v1 easy saturates: 100% pass, 0.85-0.96 with 3.7-flash).+#+# Difficulty axes (named per prompt via tags):+# axis:decomposition 10+ distinct nameable parts that must fit together (v1 hard failure+# modes were floating parts / interpenetration / stray islands — these+# prompts maximise part-to-part contact surface count)+# axis:thin_features wires, spokes, scrolls, strings at 3-30 mm thickness+# axis:curved_organic compound-curved / organic surfaces (no box-stack shortcut)+# axis:arch_detail repeated countable architectural elements (dentils, rafters, panes)+# axis:multi_material compositions of 2+ objects in distinct materials that must relate+# Controls: 4 medium prompts whose PROMPT TEXT is copied verbatim from v1 medium+# (furniture / vehicles / appliances / tools — 2/4 passed in static_v1_flash, the+# discriminating band of that run) so+# generation difficulty is comparable across versions; their must_have lists are extended+# to the v2 checklist standard (8+ items, half with dims), so judge scores are v2-internal.+# Expected discrimination: controls ~30-60% pass; hard tier 0-30% pass with score spread+# 0.4-0.7 driven by assembly_fit and geometry_detail, not intent_fidelity.+name: static_objects_v2+track: static_object+language: blender+description: >+ 20 static-object prompts: 4 medium controls (prompt text shared with v1 medium for+ cross-version comparison) and 16 hard prompts across five named difficulty axes+ (decomposition depth, thin features, curved/organic surfaces, architectural detail,+ multi-material composition). Every prompt carries 8-16 nameable parts in must_have,+ at least half with measurable dimensions.+prompts:+ # ------------------------------------------------------------- controls (v1 medium text)+ - id: ctrl_med_dining_chair+ tier: medium+ category: furniture+ tags: [control, "axis:decomposition"]+ prompt: a mid-century wooden dining chair with a slatted backrest and tapered legs+ must_have:+ - "four tapered legs, each thinner at the floor than at the seat"+ - "seat slab about 0.42 m wide"+ - "seat surface about 0.45 m above the floor"+ - "backrest with at least 3 vertical slats"+ - "back legs continue up into the backrest posts"+ - "backrest top rail between 0.80 m and 0.95 m above the floor"+ - "all four feet touch the ground plane"+ - "no leg or slat intersects the seat slab"+ dimensions_m: {width: 0.45, depth: 0.50, height: 0.90}+ - id: ctrl_med_pickup+ tier: medium+ category: vehicles+ tags: [control, "axis:decomposition"]+ prompt: a pickup truck with a two-door cab, open cargo bed, wheel arches and side mirrors+ must_have:+ - "cab with windshield and 2 side windows"+ - "open rear bed with 3 walls and a tailgate"+ - "4 wheels with hubs, all touching the ground"+ - "wheel diameter about 0.75 m"+ - "front grille and 2 headlights"+ - "2 side mirrors on stalks"+ - "bed floor about 0.80 m above the ground"+ - "overall length about 5.2 m"+ dimensions_m: {length: 5.2, width: 1.9, height: 1.8}+ - id: ctrl_med_toaster+ tier: medium+ category: appliances+ tags: [control, "axis:curved_organic"]+ prompt: a two-slot retro toaster with rounded chrome body, a lever on the side, a browning dial and a crumb tray+ must_have:+ - "2 bread slots on the top face"+ - "each slot about 0.13 m long and under 0.04 m wide"+ - "lever on one end face"+ - "round dial knob on the front face"+ - "rounded body with no sharp vertical edges"+ - "crumb tray seam line near the base"+ - "4 small feet"+ - "body about 0.28 m wide and 0.19 m tall"+ dimensions_m: {width: 0.28, depth: 0.17, height: 0.19}+ - id: ctrl_med_hand_drill+ tier: medium+ category: tools+ tags: [control, "axis:decomposition"]+ prompt: a cordless power drill with a pistol grip, trigger, battery pack in the base of the grip and a keyless chuck holding a drill bit+ must_have:+ - "pistol-grip body with the grip angled back from vertical"+ - "trigger recessed at the top of the grip"+ - "battery block under the grip, wider than the grip"+ - "chuck at the front, about 0.04 m diameter"+ - "drill bit protruding about 0.08 m from the chuck"+ - "side vents on the motor housing"+ - "overall length about 0.22 m from chuck tip to body rear"+ - "overall height about 0.24 m from battery base to body top"+ dimensions_m: {length: 0.22, height: 0.24}+ # ------------------------------------------------------- mechanical assemblies (hard)+ - id: mech_hard_coffee_grinder+ tier: hard+ category: mechanisms+ tags: ["axis:decomposition"]+ prompt: >+ a hand-crank coffee grinder: square wooden box body with a small front drawer,+ cast-iron dome top with an open hopper funnel, a conical burr visible inside+ the funnel throat, and a top crank arm with a wooden knob+ must_have:+ - "square wooden box body about 0.14 m wide and 0.12 m tall"+ - "front drawer about 0.05 m tall with a round knob, in the lower half of the box"+ - "drawer front sits flush in its opening with a visible seam"+ - "metal dome top mounted on the box"+ - "open hopper funnel about 0.08 m across at the rim"+ - "conical burr visible inside the funnel throat"+ - "central spindle about 0.01 m thick rising from the burr through the dome"+ - "crank arm about 0.11 m long attached to the spindle top"+ - "wooden knob on a short vertical pin at the crank's free end"+ - "adjustment nut on the spindle above the crank hub"+ dimensions_m: {width: 0.14, depth: 0.14, height: 0.30}+ - id: mech_hard_machinist_vise+ tier: hard+ category: tools+ tags: ["axis:decomposition", "axis:thin_features"]+ prompt: >+ a machinist bench vise with an acme lead screw showing its thread helix, twin+ cylindrical guide rails, a swivel base with two lock handles, serrated jaw+ plates and a sliding tommy-bar handle+ must_have:+ - "fixed jaw and sliding jaw with a 0.05 m open gap between them"+ - "flat jaw plates about 0.10 m wide with serration grooves"+ - "lead screw between the jaws with at least 6 visible thread turns"+ - "2 cylindrical guide rails flanking the screw"+ - "sliding jaw rides on the rails and the screw"+ - "tommy bar about 0.16 m long through the screw head, with ball ends"+ - "swivel base disc under the vise body"+ - "2 lock handles on the swivel base rim"+ - "mounting flange with 3 bolt holes"+ - "overall body length about 0.30 m"+ dimensions_m: {length: 0.30, width: 0.14, height: 0.18}+ - id: mech_hard_drill_press+ tier: hard+ category: tools+ tags: ["axis:decomposition"]+ prompt: >+ a benchtop drill press: cast base, vertical column, adjustable work table with+ a crank, head casting with a belt housing, chuck with a bit over the table,+ three-spoke quill feed lever and a depth-stop rod+ must_have:+ - "flat base about 0.30 m by 0.20 m with 2 mounting slots"+ - "vertical column about 0.05 m diameter rising from the base"+ - "work table clamped to the column between base and head"+ - "table raise crank handle below the table"+ - "head casting at the top of the column"+ - "belt housing on top of the head with a lid seam"+ - "cylindrical motor behind the column, connected to the head"+ - "chuck about 0.04 m diameter under the head with a drill bit pointing down"+ - "bit tip aligned over the table center"+ - "quill feed lever with 3 spokes ending in ball knobs"+ - "depth-stop rod with 2 nuts beside the quill"+ - "overall height about 0.85 m"+ dimensions_m: {height: 0.85, depth: 0.45}+ - id: mech_hard_pitcher_pump+ tier: hard+ category: garden+ tags: ["axis:decomposition", "axis:curved_organic"]+ prompt: >+ a cast-iron pitcher water pump on a wooden platform: fluted cylindrical body,+ curved downward spout, long lever handle raised mid-stroke, connecting link to+ the piston rod, domed cap and a bolted base flange+ must_have:+ - "cylindrical pump body about 0.09 m diameter"+ - "at least 6 vertical flutes on the body"+ - "base flange with 4 bolt heads"+ - "square wooden platform under the flange"+ - "curved spout leaving the body two-thirds up, opening facing down"+ - "domed cap on top of the body"+ - "lever handle about 0.40 m long, raised at an angle"+ - "handle pivots in a clevis bracket at the top rear of the body"+ - "connecting link from the handle to a piston rod entering the cap"+ - "overall height about 0.65 m"+ dimensions_m: {height: 0.65}+ # ------------------------------------------------------- thin / curved structures (hard)+ - id: thin_hard_garden_arch+ tier: hard+ category: garden+ tags: ["axis:thin_features"]+ prompt: >+ a wrought-iron garden arch: two lattice side frames, a semicircular arch head+ filled with S-scrolls, spear-point finials and a tie bar at the spring line+ must_have:+ - "2 side frames, each of 2 uprights about 0.01 m thick"+ - "diamond lattice bars between the uprights of each side frame"+ - "semicircular arch head spanning about 1.2 m between the side frames"+ - "at least 6 S-scrolls filling the arch head"+ - "at least 4 scrolls in each side frame"+ - "3 spear-point finials on top of the arch"+ - "horizontal tie bar connecting both sides at the spring line"+ - "small ground plates under all 4 upright feet"+ - "overall height about 2.3 m"+ - "all bars slender: no member thicker than 0.02 m"+ dimensions_m: {width: 1.4, depth: 0.5, height: 2.3}+ - id: thin_hard_violin+ tier: hard+ category: musical_instruments+ tags: ["axis:thin_features", "axis:curved_organic"]+ prompt: >+ a violin: hourglass body with C-bout waists, arched top with two f-holes, a+ bridge, four strings from tailpiece over the bridge along the fingerboard to+ the pegbox, a scroll head, four tuning pegs and a chin rest+ must_have:+ - "hourglass body with 2 concave C-bout waist cutaways"+ - "body length about 0.36 m"+ - "top plate visibly arched, not flat"+ - "2 f-holes through the top, one each side of the bridge"+ - "bridge about 0.03 m tall standing upright between the f-holes"+ - "4 strings running in parallel from tailpiece to pegbox"+ - "strings pass over the bridge with a visible break angle"+ - "black fingerboard extending over the body without touching it"+ - "spiral scroll at the head"+ - "4 tuning pegs, 2 per side of the pegbox"+ - "tailpiece anchored toward the end button"+ - "chin rest beside the tailpiece"+ - "total length about 0.60 m"+ dimensions_m: {length: 0.60, width: 0.21, height: 0.09}+ - id: thin_hard_penny_farthing+ tier: hard+ category: vehicles+ tags: ["axis:thin_features", "axis:curved_organic"]+ prompt: >+ a penny-farthing bicycle: huge spoked front wheel with pedal cranks on its hub,+ small spoked rear wheel, curved backbone tube, front fork, handlebar and a+ sprung leather saddle+ must_have:+ - "front wheel about 1.3 m diameter with at least 24 thin spokes"+ - "rear wheel about 0.40 m diameter with at least 12 spokes"+ - "tyres no thicker than 0.03 m"+ - "backbone tube arcing from the fork crown down to the rear wheel fork"+ - "front fork straddling the big wheel to its hub"+ - "handlebar with 2 grips above the front hub"+ - "2 pedal cranks on the front hub with pedals, opposed at 180 degrees"+ - "saddle on the backbone behind the handlebar, on a visible spring"+ - "small mounting step peg on the backbone above the rear wheel"+ - "both wheels touch the ground plane"+ dimensions_m: {length: 1.7, height: 1.5}+ - id: thin_hard_birdcage+ tier: hard+ category: household+ tags: ["axis:thin_features"]+ prompt: >+ a round wire birdcage with an onion-dome top, a hanging ring, a wire door with+ a latch, a trapeze perch hanging inside, a straight perch, two feed cups and a+ base tray with a rolled rim+ must_have:+ - "round cage about 0.35 m diameter"+ - "at least 20 vertical wires around the circumference"+ - "wires about 0.003 m thick"+ - "3 horizontal rings tying the wires"+ - "onion-dome top converging to a finial"+ - "hanging ring at the very top"+ - "rectangular door frame set into the wires with a small latch"+ - "trapeze perch hanging inside from the dome on 2 wires"+ - "straight perch spanning the cage between opposite walls"+ - "2 feed cups mounted at a ring"+ - "solid base tray with a rolled rim"+ - "overall height about 0.55 m"+ dimensions_m: {height: 0.55}+ # ------------------------------------------------------------------ organic (hard)+ - id: org_hard_bonsai_rock+ tier: hard+ category: plants+ tags: ["axis:curved_organic"]+ prompt: >+ a root-over-rock bonsai: a shallow oval pot with soil, a rounded rock gripped+ by exposed roots, a tapering trunk with two pronounced curves, and five+ distinct flattened foliage pads+ must_have:+ - "shallow oval pot about 0.30 m long with a rim and small feet"+ - "soil surface inside the pot"+ - "rounded rock about 0.12 m tall sitting on the soil"+ - "at least 4 roots gripping over the rock faces and entering the soil"+ - "trunk rising from the rock top with at least 2 pronounced curves"+ - "trunk tapers from about 0.04 m at the base to under 0.015 m at the apex"+ - "5 distinct flattened foliage pads with gaps between them"+ - "lowest branch extends beyond the pot rim"+ - "no foliage pad touches the soil or the rock"+ - "overall height about 0.50 m"+ dimensions_m: {height: 0.50, length: 0.45}+ - id: org_hard_seated_cat+ tier: hard+ category: animals+ tags: ["axis:curved_organic"]+ prompt: >+ a domestic cat sitting upright: haunches on the ground, straight vertical+ front legs, tail curled around beside the front paws, short muzzle and+ triangular ears+ must_have:+ - "upright seated pose with haunches on the ground and chest raised"+ - "2 straight front legs, nearly vertical"+ - "front paws on the ground between the haunches"+ - "hind legs folded with thighs bulging wider than the chest"+ - "tail about 0.30 m long curling around to lie beside the front paws"+ - "head with a short muzzle, 2 eyes and a nose"+ - "2 triangular ears about 0.04 m tall"+ - "back forms a smooth continuous curve from neck to tail base"+ - "silhouette narrows from haunches up to the shoulders"+ - "overall height about 0.35 m"+ dimensions_m: {height: 0.35, depth: 0.25}+ - id: org_hard_octopus_rock+ tier: hard+ category: animals+ tags: ["axis:curved_organic", "axis:thin_features"]+ prompt: >+ an octopus draped over a low rock: bulbous mantle, raised eyes, eight+ tapering arms curling in different directions, with sucker discs visible+ along the underside of at least three arms+ must_have:+ - "bulbous mantle about 0.20 m long"+ - "2 raised eye bumps on the head"+ - "8 arms radiating from the head"+ - "each arm tapers from about 0.05 m at the base to a point"+ - "arm tips curl, at least 4 curling in different directions"+ - "sucker discs along the underside of at least 3 arms"+ - "low rock under the body, arms draping over its sides"+ - "at least 2 arms touch the ground plane"+ - "overall arm spread about 0.8 m"+ dimensions_m: {length: 0.8, height: 0.3}+ # ------------------------------------------------------- architectural detail (hard)+ - id: arch_hard_bay_window+ tier: hard+ category: buildings+ tags: ["axis:arch_detail", "axis:decomposition"]+ prompt: >+ a Victorian bay window facade section: a three-sided bay projecting from a+ clapboard wall, a sash window with divided panes in each face, panel aprons,+ corner pilasters, scroll brackets and a dentil cornice under a faceted roof cap+ must_have:+ - "three-sided bay projecting from a flat wall section"+ - "center face wider than the 2 angled side faces"+ - "side faces angled about 45 degrees from the wall"+ - "sash window in each of the 3 faces"+ - "upper sash of each window divided into at least 4 panes by muntins"+ - "continuous sill band under all 3 windows"+ - "recessed panel apron under each window"+ - "pilasters at the 4 vertical bay corners"+ - "cornice with at least 10 dentil blocks across the bay"+ - "at least 4 scroll brackets under the bay roof"+ - "three-faceted roof cap over the bay"+ - "horizontal siding boards on the flat wall section"+ dimensions_m: {width: 3.4, height: 3.2, depth: 1.0}+ - id: arch_hard_spiral_stair+ tier: hard+ category: buildings+ tags: ["axis:arch_detail", "axis:thin_features"]+ prompt: >+ a cast-iron spiral staircase: central column, wedge treads winding a full+ turn, a baluster at each tread's outer edge, a continuous helical handrail+ and a railed top landing+ must_have:+ - "central column about 0.10 m diameter"+ - "at least 12 wedge-shaped treads attached to the column"+ - "treads climb evenly through at least 360 degrees total rotation"+ - "open risers: visible gaps between consecutive treads"+ - "1 baluster at the outer edge of each tread"+ - "continuous helical handrail on top of the balusters"+ - "top landing platform with its own railing"+ - "base plate at the bottom of the column"+ - "each tread about 0.70 m from column to outer edge"+ - "overall height about 2.7 m"+ dimensions_m: {height: 2.7, width: 1.6}+ - id: arch_hard_teahouse+ tier: hard+ category: buildings+ tags: ["axis:arch_detail"]+ prompt: >+ a Japanese teahouse: raised wooden floor on short posts, a veranda strip+ along the front, sliding shoji panels with visible grids, corner posts, and+ a hipped roof with deep eaves showing exposed rafter tails+ must_have:+ - "floor platform raised about 0.5 m on short posts"+ - "wooden veranda strip along the front edge"+ - "at least 3 sliding shoji panels in the front wall"+ - "each shoji panel gridded into at least 12 rectangles"+ - "square corner posts at the 4 corners"+ - "hipped roof with eaves overhanging about 0.6 m"+ - "at least 8 exposed rafter tails visible under each eave side"+ - "ridge cap along the roof top"+ - "solid back and side walls contrasting with the shoji front"+ - "step stone below the veranda"+ dimensions_m: {width: 3.5, depth: 3.0, height: 3.0}+ # ------------------------------------------------------- multi-material compositions (hard)+ - id: comp_hard_drafting_table+ tier: hard+ category: furniture+ tags: ["axis:multi_material", "axis:decomposition"]+ prompt: >+ a drafting table setup: tilted drawing board on a trestle frame with quadrant+ tilt brackets, a parallel ruler riding on side rails, a pen tray, a sheet of+ paper held by two corner clips, a clamp-on two-arm lamp aimed at the board,+ and a stool with a footring+ must_have:+ - "drawing board about 1.0 m by 0.7 m tilted about 20 degrees"+ - "parallel ruler bar spanning the board width"+ - "ruler rides on a rail or wire at each side edge"+ - "pen tray along the bottom edge of the board"+ - "trestle legs with a cross stretcher"+ - "quadrant tilt bracket with a knob at each of the board's 2 sides"+ - "sheet of paper on the board held by clips at 2 corners"+ - "lamp clamped to a top corner of the board"+ - "lamp has 2 arm segments and a shade aimed down at the board"+ - "stool with a round seat about 0.65 m high"+ - "stool footring between its legs"+ - "board top edge about 1.1 m above the floor"+ dimensions_m: {width: 1.1, depth: 0.9, height: 1.2}+ - id: comp_hard_telescope_tripod+ tier: hard+ category: props+ tags: ["axis:multi_material", "axis:decomposition"]+ prompt: >+ a brass refractor telescope on a wooden tripod: main tube with an objective+ ring, a thinner draw tube with an eyepiece cup, a clamping collar and cradle+ on a pan-tilt head, three splayed legs joined by a spreader tray, metal feet+ must_have:+ - "main tube about 0.7 m long and 0.08 m diameter"+ - "objective ring at the front of the tube"+ - "draw tube about 0.05 m diameter extending from the rear, thinner than the main tube"+ - "eyepiece cup at the draw tube end"+ - "clamping collar around the main tube's balance point"+ - "cradle connecting the collar to a pan-tilt head"+ - "tension knob on the head"+ - "3 wooden legs splayed outward, each about 1.2 m long"+ - "triangular spreader tray connecting the legs at about 0.7 m height"+ - "metal feet at the leg tips"+ - "tube tilted upward between 20 and 40 degrees"+ - "eyepiece height about 1.5 m above the ground"+ dimensions_m: {height: 1.7}diff --git a/harness/tests/bench_prompts/__init__.py b/harness/tests/bench_prompts/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/harness/tests/bench_prompts/test_v2_batteries.py b/harness/tests/bench_prompts/test_v2_batteries.pynew file mode 100644index 00000000..e70a02e9--- /dev/null+++ b/harness/tests/bench_prompts/test_v2_batteries.py@@ -0,0 +1,167 @@+"""Structural validation of the v2 prompt batteries (offline, no runs).++The v2 batteries drop the saturated easy tier and encode their difficulty+contracts directly in ``must_have``. These tests pin those contracts:++* every file parses through ``bench.run_bench.Battery.load`` (the real loader);+* every prompt has a tier, a category, a non-empty ``must_have`` and a+ snake_case id containing its tier token;+* static_objects_v2 / compare_v2: 8-16 must_have parts per prompt, at least+ half carrying a measurable number, and most prompts with ``dimensions_m``;+* articulated_v2: every prompt names >= 3 quantified motions for the+ pose-sweep judge;+* scenes_v2: every scene states a luminance band and a camera brief;+* compare_v2 prompts are drawn verbatim from static_objects_v2.+"""++from __future__ import annotations++import re+import sys+from pathlib import Path++import pytest++REPO = Path(__file__).resolve().parents[2]+if str(REPO) not in sys.path:+ sys.path.insert(0, str(REPO))++from bench.run_bench import Battery # noqa: E402++PROMPTS = REPO / "bench" / "prompts"++V2_FILES = {+ "static_objects_v2": PROMPTS / "static_objects_v2.yaml",+ "articulated_v2": PROMPTS / "articulated_v2.yaml",+ "scenes_v2": PROMPTS / "scenes_v2.yaml",+ "graphics_v2": PROMPTS / "graphics_v2.yaml",+ "compare_v2": PROMPTS / "compare_v2.yaml",+}++EXPECTED_COUNTS = {+ "static_objects_v2": 20,+ "articulated_v2": 14,+ "scenes_v2": 10,+ "graphics_v2": 10,+ "compare_v2": 8,+}++ID_RE = re.compile(r"^[a-z0-9]+(_[a-z0-9]+)+$")+NUMBER_RE = re.compile(r"\d")+MOTION_RE = re.compile(+ r"\b(slid(?:e|es|ing)|swing(?:s|ing)?|rotat(?:e|es|ing)|swivel(?:s)?|"+ r"pivot(?:s)?|open(?:s|ed|ing)?|fold(?:s|ed|ing)?|travel(?:s)?|"+ r"advanc(?:e|es)|rais(?:e|es)|lower(?:s)?|tilt(?:s)?|extend(?:s)?|"+ r"retract(?:s)?|splay(?:s|ing)?|pan|lift(?:s)?|drop(?:s)?|hinge(?:d)?|"+ r"revolute|prismatic|continuous(?:ly)?)\b"+)++[email protected](scope="module", params=sorted(V2_FILES), ids=sorted(V2_FILES))+def battery(request) -> Battery:+ b = Battery.load(V2_FILES[request.param])+ assert b.name == request.param+ return b+++def test_all_v2_files_exist():+ missing = [str(p) for p in V2_FILES.values() if not p.is_file()]+ assert not missing+++def test_battery_schema(battery: Battery):+ assert len(battery.prompts) == EXPECTED_COUNTS[battery.name]+ ids = [p.id for p in battery.prompts]+ assert len(ids) == len(set(ids)), "duplicate prompt ids"+ for p in battery.prompts:+ assert p.tier in {"medium", "hard"}, f"{p.id}: v2 has no easy tier"+ assert p.category, f"{p.id}: category required"+ assert p.must_have, f"{p.id}: must_have required"+ assert ID_RE.match(p.id), f"{p.id}: not snake_case"+ tier_token = {"medium": "med"}.get(p.tier, p.tier)+ assert f"_{tier_token}_" in p.id, f"{p.id}: id must contain its tier token"+ for item in p.must_have:+ assert item.strip(), f"{p.id}: empty must_have item"+ for banned in ("ornate", "detailed"):+ assert banned not in p.prompt.lower(), f"{p.id}: vague word '{banned}'"+++def _load(name: str) -> Battery:+ return Battery.load(V2_FILES[name])++[email protected]("name", ["static_objects_v2", "compare_v2"])+def test_static_prompts_have_dense_dimensioned_checklists(name: str):+ b = _load(name)+ with_dims = 0+ for p in b.prompts:+ n = len(p.must_have)+ assert 8 <= n <= 16, f"{p.id}: {n} must_have items, want 8-16 nameable parts"+ numbered = sum(1 for item in p.must_have if NUMBER_RE.search(item))+ assert numbered * 2 >= n, f"{p.id}: under half of must_have items carry a number"+ if p.dimensions_m:+ with_dims += 1+ assert with_dims * 2 >= len(b.prompts), f"{name}: under half the prompts set dimensions_m"+++def test_static_v2_has_controls_and_hard_axes():+ b = _load("static_objects_v2")+ controls = [p for p in b.prompts if "control" in p.tags]+ assert len(controls) == 4 and all(p.tier == "medium" for p in controls)+ hard = [p for p in b.prompts if p.tier == "hard"]+ assert len(hard) == 16+ axes = {t for p in b.prompts for t in p.tags if t.startswith("axis:")}+ assert {"axis:decomposition", "axis:thin_features", "axis:curved_organic",+ "axis:arch_detail", "axis:multi_material"} <= axes+++def test_articulated_prompts_name_quantified_motions():+ b = _load("articulated_v2")+ for p in b.prompts:+ motions = [i for i in p.must_have if MOTION_RE.search(i.lower())]+ assert len(motions) >= 3, f"{p.id}: fewer than 3 motion items for the pose sweep"+ quantified = [i for i in motions if NUMBER_RE.search(i)]+ assert len(quantified) >= 2, f"{p.id}: motions lack quantities (range / travel)"+++def test_scene_prompts_state_composition_contracts():+ b = _load("scenes_v2")+ for p in b.prompts:+ assert len(p.must_have) >= 10, f"{p.id}: composition contract too thin"+ joined = " ".join(p.must_have).lower()+ assert "zones" in joined, f"{p.id}: no zone list"+ assert "luminance" in joined, f"{p.id}: no measurable lighting band"+ assert "camera" in joined, f"{p.id}: no camera brief"+ counted = [i for i in p.must_have if re.search(r"at least \d+|\b\d+ zones\b", i.lower())]+ assert len(counted) >= 4, f"{p.id}: too few counted asset types"+++def test_graphics_v2_opengl_prompts_carry_language_override():+ """The ogl_* rows must run under opengl_python; the loader forwards the+ per-prompt override into the Spec (BenchPrompt.language or battery.language)."""+ from codeverse.contracts.common import Backends, Language++ from bench.run_bench import build_spec++ b = _load("graphics_v2")+ assert b.language == Language.GLSL_SHADER+ backends = Backends()+ for prompt in b.prompts:+ if prompt.id.startswith("ogl_"):+ assert prompt.language == Language.OPENGL_PYTHON, f"{prompt.id}: missing language override"+ else:+ assert prompt.language is None, f"{prompt.id}: unexpected language override"+ spec = build_spec(b, prompt, backends=backends, rounds=1, max_usd=1, max_minutes=1, tag0="t")+ want = Language.OPENGL_PYTHON if prompt.id.startswith("ogl_") else Language.GLSL_SHADER+ assert spec.language == want, f"{prompt.id}: spec language {spec.language} != {want}"+++def test_compare_v2_is_a_subset_of_static_v2():+ static = {p.id.split("_", 1)[1]: p for p in _load("static_objects_v2").prompts}+ for p in _load("compare_v2").prompts:+ key = p.id.split("_", 1)[1] # cmp_hard_violin -> hard_violin == prefix-stripped static key+ src = static.get(key)+ assert src is not None, f"{p.id}: no matching static_objects_v2 prompt"+ assert p.prompt == src.prompt, f"{p.id}: prompt text drifted from static_objects_v2"+ assert p.must_have == src.must_have, f"{p.id}: must_have drifted"+ assert p.dimensions_m == src.dimensions_m, f"{p.id}: dimensions drifted"diff --git a/harness/tests/flywheel_cli/test_bench.py b/harness/tests/flywheel_cli/test_bench.pyindex 2f37e6ce..51804067 100644--- a/harness/tests/flywheel_cli/test_bench.py+++ b/harness/tests/flywheel_cli/test_bench.py@@ -27,10 +27,11 @@ def test_batteries_are_valid(path: Path):ids = [p.id for p in b.prompts]assert len(ids) == len(set(ids))tiers = {p.tier for p in b.prompts}- assert tiers <= {"easy", "medium", "hard"} and len(tiers) == 3+ assert tiers <= {"easy", "medium", "hard"}assert all(p.must_have for p in b.prompts)expected = {"static_objects_v1": 24, "articulated_v1": 12, "scenes_v1": 12}- if b.name in expected: # other batteries (e.g. compare_v1) are owned elsewhere; only the schema is checked+ if b.name in expected: # other batteries (compare_*, *_v2) are owned elsewhere; only the schema is checked+ assert len(tiers) == 3 # v1 batteries span all three tiers; v2 drops easy (it saturated)assert len(b.prompts) == expected[b.name]