Skip to content

Official Codebase of 3DCodeVerse

Python

bench: compare_backends learns the articulated track

ziyao660784f4fd60
diff --git a/harness/bench/_compare_report.py b/harness/bench/_compare_report.py
index b6539fac..206c449e 100644
--- a/harness/bench/_compare_report.py
+++ b/harness/bench/_compare_report.py
@@ -22,6 +22,10 @@ from bench._jsonl import read_jsonl
T = TypeVar("T", bound=BaseModel)
+#: The fixed judge's rubric per battery track (bench/_fixed_eval.py builds the evaluator from it).
+RUBRIC_BY_TRACK: dict[str, str] = {"static_object": "static_object_v1", "articulated_object": "articulated_v1"}
+
+
class CellResult(BaseModel):
prompt_id: str
tier: str = ""
@@ -218,7 +222,7 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], me
prompts = _prompt_order(rows, meta)
cells = {(r.prompt_id, r.arm): r for r in rows}
md = [f"# harness vs one-shot — {meta.get('battery', {}).get('name', out.name)}", "",
- f"fixed judge: **{opts.get('judge', '?')}** (rubric static_object_v1, n_samples={opts.get('n_samples', 2)}, "
+ f"fixed judge: **{opts.get('judge', '?')}** (rubric {RUBRIC_BY_TRACK.get(str(meta.get('battery', {}).get('track', 'static_object')), 'static_object_v1')}, n_samples={opts.get('n_samples', 2)}, "
f"acceptance = must_have list) · harness loop judge: {opts.get('loop_judge') or 'settings default'} · "
f"harness rounds ≤ {opts.get('rounds', '?')}, ≤ ${opts.get('max_usd', '?')} · cells: {len(rows)}", "",
"Every arm's final `src/model.py` is re-built, re-rendered and judged by the same evaluator; a failed "
diff --git a/harness/bench/_fixed_eval.py b/harness/bench/_fixed_eval.py
index b37cdc8e..843d9ebc 100644
--- a/harness/bench/_fixed_eval.py
+++ b/harness/bench/_fixed_eval.py
@@ -14,9 +14,10 @@ from typing import Any
from pydantic import BaseModel, Field
+from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402
from codeverse.config import Settings, get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import Language, Track
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.spec import Spec
@@ -50,11 +51,16 @@ class EvalOutcome(BaseModel):
class FixedEvaluator:
"""build → measure → connectivity → render → VlmJudge, the same for every arm."""
- def __init__(self, judge_model: str, *, n_samples: int = 2, settings: Settings | None = None, rubric: str = RUBRIC):
+ def __init__(self, judge_model: str, *, n_samples: int = 2, settings: Settings | None = None, rubric: str | None = None,
+ track: Track = Track.STATIC_OBJECT, language: Language = Language.BLENDER):
self.judge_model = judge_model
self.n_samples = n_samples
self.settings = settings or get_settings()
- self.rubric = rubric
+ self.track = track
+ self.language = language
+ # articulated batteries are judged on articulated_v1 (rest views + the pose sheet the
+ # joint sweep renders); the rubric follows the battery's track unless a caller pins one
+ self.rubric = rubric or RUBRIC_BY_TRACK.get(track.value, RUBRIC)
self._runtime: Any = None
self._judge: Any = None
@@ -63,7 +69,7 @@ class FixedEvaluator:
if self._runtime is None:
from codeverse.languages import get_runtime
- self._runtime = get_runtime(Language.BLENDER)
+ self._runtime = get_runtime(self.language)
return self._runtime
@property
@@ -95,6 +101,16 @@ class FixedEvaluator:
out.gates.append(check_connectivity(glb))
r = self.settings.render
out.renders = render_glb(glb, ws.renders_dir(0), views=list(OBJECT_VIEWS), width=r.width, height=r.height, sheet=True)
+ if self.track is Track.ARTICULATED_OBJECT:
+ # the same deterministic articulation evidence the track gives its judge: the
+ # joint sweep (collisions over every joint's range) and the pose sheet / pose
+ # tiles, read straight from the built URDF — no plan, so every arm is treated alike
+ from codeverse.tracks.articulated_object import default_joint_sweep
+
+ sweep, pose_views = default_joint_sweep(ws, None, ws.renders_dir(0) / "poses")
+ out.gates.append(sweep)
+ if pose_views:
+ out.renders.views = list(out.renders.views) + pose_views
inp = JudgeInput(spec=spec, renders=out.renders, measurement=out.measurement, gates=out.gates,
acceptance=acceptance_from_spec(spec), round_index=0)
out.judgment = self.judge.judge(inp)
diff --git a/harness/bench/_oneshot.py b/harness/bench/_oneshot.py
index ad6faede..50c3d1b9 100644
--- a/harness/bench/_oneshot.py
+++ b/harness/bench/_oneshot.py
@@ -41,7 +41,7 @@ from codeverse.agents.watchdog import run_with_watchdog
from codeverse.config import get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport
from codeverse.contracts.chat import ChatMessage, ChatRequest
-from codeverse.contracts.common import Usage
+from codeverse.contracts.common import Language, Usage
from codeverse.contracts.spec import Spec
from codeverse.conventions import LANGUAGE_FRAME, frame_doc
from codeverse.tracks.generation import MultiFileParseError, parse_multifile
@@ -49,6 +49,7 @@ from codeverse.tracks.repair import format_error_report
from codeverse.workspace import Workspace
MODEL_FILE = "src/model.py"
+URDF_FILE = "src/robot.urdf"
DEFAULT_TIMEOUT_S = 900.0
IDLE_GRACE_S = 600.0
@@ -77,32 +78,98 @@ _OUTPUT_RULE = (f"You have NO tools in this session: you cannot write files, run
f"the code must appear in your reply. Reply with the COMPLETE contents of `{MODEL_FILE}` as ONE "
f"```python fenced code block and nothing else (no prose before or after, no partial snippets).")
-
-def minimal_contract() -> str:
+#: The URDF (track articulated_object, language urdf_blender) minimal contract: the SAME
+#: bpy rules for the link meshes plus the frame recipe of codeverse/prompts/urdf/contract.md
+#: (D18) condensed to its rules — no worked example, no cookbook, no skeleton: the one-shot
+#: arm gets what a careful reader of the contract would know, nothing the harness builds.
+_URDF_CONTRACT_BODY = """You write TWO files.
+`{model_file}` in raw Blender Python (bpy, Blender 5.x API): ONE mesh object per LINK, `obj.name` == the
+link name, built in WORLD coordinates at the REST pose (URDF q = 0: doors closed, drawers in, lids down,
+arms at home). The harness runs it headless in an emptied scene and exports `meshes/<link>.glb` for every
+link with the world coordinates baked in — you never export.
+`{urdf_file}` hand-written URDF (links + joints) for the SAME object.
+
+Frame, units and placement: {frame} Build at real-world size, on z = 0.
+
+URDF recipe (the harness verifies every line numerically and FAILS the build otherwise):
+- Every joint `rpy="0 0 0"`: all link frames stay axis-aligned with the world.
+- Root link frame = world origin (0,0,0). Every other link frame = its joint's PIVOT: a world point on the
+ joint axis (hinge line / slide axis / axle); for a fixed joint, the child's attachment point.
+- `<joint><origin xyz>` = pivot_child − frame_parent (plain subtraction; for a child of the root that is the
+ pivot itself).
+- `<visual>` AND `<collision>` `<origin xyz>` = −pivot_link (root: 0 0 0), because the mesh holds world
+ coordinates and the link frame sits at the pivot. One `<visual>` and one identical `<collision>` per link,
+ geometry `<mesh filename="meshes/<link>.glb"/>`, no scale.
+- `<axis xyz>` is a unit vector in WORLD coordinates; positive q must move the child the way a user expects
+ (door swings open, drawer pulls out towards −Y, lid lifts up) — negate the axis if not, never swap limits.
+- `revolute` / `prismatic` joints need `<limit lower upper effort velocity/>` with lower ≤ 0 ≤ upper;
+ `continuous` only `<limit effort velocity/>`; `fixed` none.
+- One tree: exactly one root link, every other link the child of exactly one joint, no cycles, ≤ 60 links.
+ Hardware on a moving part (handle on a door) attaches to the moving link with a `fixed` joint.
+- Clearance 1–3 mm between a moving part and its housing over the whole range; fixed children touch their
+ parent (gap ≤ 2 mm).
+- Names: link name == Blender object name == `meshes/<link>.glb` stem, case-sensitive, identical in both files
+ (snake_case, e.g. `door`, `handle_left`; never `Door.001`); joint names unique; `world` is reserved.
+- Forbidden in the URDF: `<gazebo>`, `<transmission>`, `<sensor>`, xacro, `package://`, mesh scale, inline
+ primitives, `mimic`. Inertial blocks optional.
+
+`{model_file}` rules: one mesh object per link linked to `bpy.context.scene.collection`, every mesh gets a
+material (Principled BSDF); do NOT create cameras or lights, touch render/world settings, call
+`bpy.ops.render.*`, `bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the
+network, or import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Under 500k
+triangles, under 120 s. The script must build every link when executed top-to-bottom (call your `main()`
+at module level) and run without errors in headless Blender — there is no second chance."""
+
+_OUTPUT_RULE_URDF = (f"You have NO tools in this session: you cannot write files, run Blender or inspect anything — "
+ f"both files must appear in your reply. Reply with the COMPLETE contents of the two files in "
+ f"this exact envelope and nothing else (no prose before, between or after):\n"
+ f"=== FILE: {MODEL_FILE} ===\n<python>\n=== END FILE ===\n"
+ f"=== FILE: {URDF_FILE} ===\n<xml>\n=== END FILE ===")
+
+
+def files_for(language: Language) -> list[str]:
+ """The files a one-shot answer must contain for ``language``."""
+ return [MODEL_FILE, URDF_FILE] if language is Language.URDF_BLENDER else [MODEL_FILE]
+
+
+def minimal_contract(language: Language = Language.BLENDER) -> str:
"""Frame/units come from ``codeverse.conventions`` (the single source of truth)."""
+ if language is Language.URDF_BLENDER:
+ return _URDF_CONTRACT_BODY.format(model_file=MODEL_FILE, urdf_file=URDF_FILE,
+ frame=frame_doc(LANGUAGE_FRAME["urdf_blender"]))
return _CONTRACT_BODY.format(model_file=MODEL_FILE, frame=frame_doc(LANGUAGE_FRAME["blender"]))
+def output_rule(language: Language = Language.BLENDER) -> str:
+ return _OUTPUT_RULE_URDF if language is Language.URDF_BLENDER else _OUTPUT_RULE
+
+
def oneshot_prompt(spec: Spec) -> str:
"""The entire input of a one-shot arm: brief + constraints + minimal contract + output rule."""
c = spec.constraints
- lines = [f"Model this object in raw bpy: {spec.prompt.strip()}"]
+ urdf = spec.language is Language.URDF_BLENDER
+ lines = [(f"Model this ARTICULATED object as raw bpy link meshes plus a hand-written URDF: {spec.prompt.strip()}"
+ if urdf else f"Model this object in raw bpy: {spec.prompt.strip()}")]
if c.dimensions_m:
lines.append("Dimensions (m): " + ", ".join(f"{k}={v:g}" for k, v in c.dimensions_m.items()))
if c.style:
lines.append(f"Style: {c.style}")
lines.extend(f"MUST HAVE: {m}" for m in c.must_have)
lines.extend(f"MUST NOT: {m}" for m in c.must_not)
- return "\n".join(lines) + "\n\n" + minimal_contract() + "\n\n" + _OUTPUT_RULE
+ return "\n".join(lines) + "\n\n" + minimal_contract(spec.language) + "\n\n" + output_rule(spec.language)
-def repair_prompt(spec: Spec, previous_code: str, build: BuildResult, lint: GateReport, attempt: int) -> str:
- """Error-feedback retry (``oneshot+repair`` arm only): previous file + the build error report."""
+def repair_prompt(spec: Spec, previous_code: str | dict[str, str], build: BuildResult, lint: GateReport,
+ attempt: int) -> str:
+ """Error-feedback retry (``oneshot+repair`` arm only): previous file(s) + the build error report."""
report = format_error_report(build, lint)
+ files = {MODEL_FILE: previous_code} if isinstance(previous_code, str) else previous_code
+ prev = "\n\n".join(f"PREVIOUS `{path}`:\n```{'xml' if path.endswith('.urdf') else 'python'}\n{body[:40_000]}\n```"
+ for path, body in files.items())
+ plural = "files" if len(files) > 1 else "file"
return (oneshot_prompt(spec)
+ f"\n\nYour previous attempt ({attempt}) did NOT build. Fix the error below (smallest correct change, "
- f"keep every part) and return the COMPLETE corrected file.\n\nERROR REPORT:\n{report}\n\n"
- f"PREVIOUS `{MODEL_FILE}`:\n```python\n{previous_code[:40_000]}\n```")
+ f"keep every part) and return the COMPLETE corrected {plural}.\n\nERROR REPORT:\n{report}\n\n{prev}")
# ----------------------------------------------------------------------------- results
@@ -159,6 +226,35 @@ def write_model_file(ws: Workspace, text: str) -> Path:
return dest
+def extract_files(text: str, language: Language = Language.BLENDER) -> dict[str, str]:
+ """Every file a one-shot answer must contain for ``language`` → ``{path: body}``.
+
+ Single-file languages keep the tolerant static path (:func:`extract_model_file`). A
+ URDF answer must carry BOTH files in the ``=== FILE: … ===`` envelope (or fenced blocks
+ each preceded by its path); an answer with only one of them is a format failure —
+ ``MultiFileParseError`` → the cell is ``no_code``, an earned zero.
+ """
+ expected = files_for(language)
+ if len(expected) == 1:
+ return {MODEL_FILE: extract_model_file(text)}
+ files = parse_multifile(text, expected_files=expected)
+ missing = [f for f in expected if not (files.get(f) or "").strip()]
+ if missing:
+ raise MultiFileParseError(f"answer lacks {missing} (got {sorted(files)})")
+ return {f: (files[f] if files[f].endswith("\n") else files[f] + "\n") for f in expected}
+
+
+def write_answer_files(ws: Workspace, text: str, language: Language = Language.BLENDER) -> list[Path]:
+ """Parse the answer and write every expected file into the workspace (raises MultiFileParseError)."""
+ written = []
+ for rel, body in extract_files(text, language).items():
+ dest = ws.root / rel
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ dest.write_text(body)
+ written.append(dest)
+ return written
+
+
# ----------------------------------------------------------------------------- common
def _scratch_cwd(label: str) -> Path:
"""A fresh directory outside any repo: no CLAUDE.md/AGENTS.md/.git context can leak in."""
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index b398c4e3..f4486fd6 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -64,10 +64,11 @@ from bench._oneshot import ( # noqa: E402
MODEL_FILE,
OneShotBackend,
OneShotResult,
+ files_for,
get_oneshot_backend,
oneshot_prompt,
repair_prompt,
- write_model_file,
+ write_answer_files,
)
from bench.run_bench import ( # noqa: E402
Battery,
@@ -197,7 +198,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts
res.error_is_infra = gen.infra_failed
return
try:
- write_model_file(eval_ws, gen.text)
+ write_answer_files(eval_ws, gen.text, spec.language)
except MultiFileParseError as e:
res.error = f"unparseable answer: {e}"
return
@@ -207,7 +208,8 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, opts
build, lint = deps.evaluator.build(eval_ws) # repair arm: error feedback only
if build.ok and not lint.errors:
return
- prompt = repair_prompt(spec, (eval_ws.root / MODEL_FILE).read_text(), build, lint, attempt + 1)
+ previous = {rel: (eval_ws.root / rel).read_text() for rel in files_for(spec.language) if (eval_ws.root / rel).is_file()}
+ prompt = repair_prompt(spec, previous, build, lint, attempt + 1)
def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, deps: CompareDeps, res: CellResult) -> None:
@@ -495,7 +497,8 @@ def main(argv: Sequence[str] | None = None) -> int:
arms = parse_arms(ns.arms)
if not ns.no_preflight and not _preflight(opts.judge, arms, opts, wait_minutes=ns.wait_for_provider):
return 2
- deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples))
+ battery = Battery.load(ns.prompts) # the fixed evaluator follows the battery's track / language
+ deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples, track=battery.track, language=battery.language))
def _log(r: CellResult) -> None:
print(f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:44s} score={r.score} build_ok={r.build_ok} "
diff --git a/harness/tests/compare_bench/test_articulated_compare.py b/harness/tests/compare_bench/test_articulated_compare.py
new file mode 100644
index 00000000..21ded876
--- /dev/null
+++ b/harness/tests/compare_bench/test_articulated_compare.py
@@ -0,0 +1,122 @@
+"""compare_backends on an articulated battery: two-file one-shot answers, articulated_v1 fixed judge."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[2]
+if str(REPO) not in sys.path:
+ sys.path.insert(0, str(REPO))
+
+from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402
+from bench._fixed_eval import FixedEvaluator # noqa: E402
+from bench._oneshot import ( # noqa: E402
+ MODEL_FILE,
+ URDF_FILE,
+ extract_files,
+ files_for,
+ minimal_contract,
+ oneshot_prompt,
+ repair_prompt,
+ write_answer_files,
+)
+from bench.run_bench import Battery # noqa: E402
+from codeverse.contracts.artifacts import BuildResult, GateReport # noqa: E402
+from codeverse.contracts.common import Language, Track # noqa: E402
+from codeverse.contracts.spec import Constraints, Spec # noqa: E402
+from codeverse.tracks.generation import MultiFileParseError # noqa: E402
+from codeverse.workspace import Workspace # noqa: E402
+
+PROMPTS = REPO / "bench" / "prompts"
+PY = "import bpy\nprint(1)\n"
+XML = '<?xml version="1.0"?>\n<robot name="x"><link name="base"/></robot>\n'
+ENVELOPE = f"=== FILE: {MODEL_FILE} ===\n{PY}=== END FILE ===\n=== FILE: {URDF_FILE} ===\n{XML}=== END FILE ===\n"
+
+
+def _urdf_spec() -> Spec:
+ return Spec(id="t/lamp", track=Track.ARTICULATED_OBJECT, language=Language.URDF_BLENDER,
+ prompt="an architect lamp with two hinged arms",
+ constraints=Constraints(must_have=["two revolute joints"], dimensions_m={"height": 0.6}))
+
+
+def test_urdf_oneshot_prompt_carries_the_frame_recipe_and_the_two_file_envelope():
+ p = oneshot_prompt(_urdf_spec())
+ assert "hand-written URDF" in p and "architect lamp" in p and "MUST HAVE: two revolute joints" in p
+ for rule in ("rpy=\"0 0 0\"", "pivot_child − frame_parent", "−pivot_link", "meshes/<link>.glb", "lower ≤ 0 ≤ upper"):
+ assert rule in p, rule
+ assert f"=== FILE: {MODEL_FILE} ===" in p and f"=== FILE: {URDF_FILE} ===" in p
+ # no harness help: no worked example, no cookbook, no skeleton
+ assert "pedal" not in p.lower() and "cookbook" not in p.lower() and "PIVOT[" not in p
+ assert minimal_contract(Language.URDF_BLENDER) in p
+ # the static contract is untouched by the language switch
+ assert minimal_contract() == minimal_contract(Language.BLENDER) and URDF_FILE not in minimal_contract()
+
+
+def test_files_for_and_extract_files_two_file_answer():
+ assert files_for(Language.BLENDER) == [MODEL_FILE] and files_for(Language.URDF_BLENDER) == [MODEL_FILE, URDF_FILE]
+ got = extract_files(ENVELOPE, Language.URDF_BLENDER)
+ assert got == {MODEL_FILE: PY, URDF_FILE: XML}
+ # fenced blocks each preceded by their path also parse
+ fenced = f"{MODEL_FILE}\n```python\n{PY}```\n{URDF_FILE}\n```xml\n{XML}```\n"
+ assert extract_files(fenced, Language.URDF_BLENDER) == {MODEL_FILE: PY, URDF_FILE: XML}
+ # the static path is unchanged
+ assert extract_files("```python\nimport bpy\n```", Language.BLENDER) == {MODEL_FILE: "import bpy\n"}
+
+
+ f"```python\n{PY}```", # one block, two expected
+ f"=== FILE: {MODEL_FILE} ===\n{PY}=== END FILE ===\n", # envelope with the URDF missing
+ f"=== FILE: {MODEL_FILE} ===\n{PY}=== END FILE ===\n=== FILE: {URDF_FILE} ===\n\n=== END FILE ===\n", # empty urdf
+])
+def test_a_urdf_answer_without_both_files_is_a_format_failure(text: str):
+ with pytest.raises(MultiFileParseError):
+ extract_files(text, Language.URDF_BLENDER)
+
+
+def test_write_answer_files_writes_both(tmp_path):
+ ws = Workspace(tmp_path / "ws")
+ ws.create()
+ paths = write_answer_files(ws, ENVELOPE, Language.URDF_BLENDER)
+ assert [p.relative_to(ws.root).as_posix() for p in paths] == [MODEL_FILE, URDF_FILE]
+ assert (ws.root / URDF_FILE).read_text() == XML and (ws.root / MODEL_FILE).read_text() == PY
+
+
+def test_urdf_repair_prompt_carries_both_previous_files():
+ build = BuildResult(ok=False, language="urdf_blender", error_type="FKMismatch",
+ error_message="link 'arm' visual origin should be -0.1 0 0.3", error_file=URDF_FILE, error_line=9)
+ p = repair_prompt(_urdf_spec(), {MODEL_FILE: PY, URDF_FILE: XML}, build, GateReport(gate="lint:urdf", passed=True), attempt=1)
+ assert "FKMismatch" in p and "corrected files" in p
+ assert f"PREVIOUS `{MODEL_FILE}`" in p and f"PREVIOUS `{URDF_FILE}`" in p and "```xml" in p
+
+
+def test_fixed_evaluator_follows_the_battery_track():
+ ev = FixedEvaluator("gemini:x", n_samples=3, track=Track.ARTICULATED_OBJECT, language=Language.URDF_BLENDER)
+ assert ev.rubric == "articulated_v1" == RUBRIC_BY_TRACK["articulated_object"]
+ assert ev.language is Language.URDF_BLENDER and ev.runtime.language is Language.URDF_BLENDER
+ static = FixedEvaluator("gemini:x")
+ assert static.rubric == "static_object_v1" and static.language is Language.BLENDER
+ pinned = FixedEvaluator("gemini:x", rubric="asset_v1", track=Track.ARTICULATED_OBJECT)
+ assert pinned.rubric == "asset_v1"
+
+
+def test_main_builds_the_evaluator_from_the_battery(monkeypatch, tmp_path):
+ from bench import compare_backends as cb
+
+ seen = {}
+
+ def fake_run_matrix(battery_path, out_dir, arms, opts, deps, on_result=None):
+ seen["track"], seen["language"], seen["rubric"] = deps.evaluator.track, deps.evaluator.language, deps.evaluator.rubric
+ (Path(out_dir)).mkdir(parents=True, exist_ok=True)
+ (Path(out_dir) / "results.jsonl").write_text("")
+
+ monkeypatch.setattr(cb, "run_matrix", fake_run_matrix)
+ monkeypatch.setattr(cb, "_preflight", lambda *a, **k: True)
+ rc = cb.main(["--prompts", str(PROMPTS / "articulated_v2.yaml"), "--arms", "oneshot:gemini:x",
+ "--out", str(tmp_path / "out"), "--no-preflight", "--judge-samples", "3"])
+ assert rc == 0
+ assert seen == {"track": Track.ARTICULATED_OBJECT, "language": Language.URDF_BLENDER, "rubric": "articulated_v1"}
+ b = Battery.load(PROMPTS / "articulated_v2.yaml")
+ assert b.track is Track.ARTICULATED_OBJECT and all(p.must_have for p in b.prompts)