Official Codebase of 3DCodeVerse
Python
bench: compare_backends learns the articulated track
ziyao660784f4fd60
diff --git a/harness/bench/_compare_report.py b/harness/bench/_compare_report.pyindex b6539fac..206c449e 100644--- a/harness/bench/_compare_report.py+++ b/harness/bench/_compare_report.py@@ -22,6 +22,10 @@ from bench._jsonl import read_jsonlT = TypeVar("T", bound=BaseModel)+#: The fixed judge's rubric per battery track (bench/_fixed_eval.py builds the evaluator from it).+RUBRIC_BY_TRACK: dict[str, str] = {"static_object": "static_object_v1", "articulated_object": "articulated_v1"}++class CellResult(BaseModel):prompt_id: strtier: str = ""@@ -218,7 +222,7 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], meprompts = _prompt_order(rows, meta)cells = {(r.prompt_id, r.arm): r for r in rows}md = [f"# harness vs one-shot — {meta.get('battery', {}).get('name', out.name)}", "",- f"fixed judge: **{opts.get('judge', '?')}** (rubric static_object_v1, n_samples={opts.get('n_samples', 2)}, "+ f"fixed judge: **{opts.get('judge', '?')}** (rubric {RUBRIC_BY_TRACK.get(str(meta.get('battery', {}).get('track', 'static_object')), 'static_object_v1')}, n_samples={opts.get('n_samples', 2)}, "f"acceptance = must_have list) · harness loop judge: {opts.get('loop_judge') or 'settings default'} · "f"harness rounds ≤ {opts.get('rounds', '?')}, ≤ ${opts.get('max_usd', '?')} · cells: {len(rows)}", "","Every arm's final `src/model.py` is re-built, re-rendered and judged by the same evaluator; a failed "diff --git a/harness/bench/_fixed_eval.py b/harness/bench/_fixed_eval.pyindex b37cdc8e..843d9ebc 100644--- a/harness/bench/_fixed_eval.py+++ b/harness/bench/_fixed_eval.py@@ -14,9 +14,10 @@ from typing import Anyfrom pydantic import BaseModel, Field+from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402from codeverse.config import Settings, get_settingsfrom codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet-from codeverse.contracts.common import Language+from codeverse.contracts.common import Language, Trackfrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.spec import Spec@@ -50,11 +51,16 @@ class EvalOutcome(BaseModel):class FixedEvaluator:"""build → measure → connectivity → render → VlmJudge, the same for every arm."""- def __init__(self, judge_model: str, *, n_samples: int = 2, settings: Settings | None = None, rubric: str = RUBRIC):+ def __init__(self, judge_model: str, *, n_samples: int = 2, settings: Settings | None = None, rubric: str | None = None,+ track: Track = Track.STATIC_OBJECT, language: Language = Language.BLENDER):self.judge_model = judge_modelself.n_samples = n_samplesself.settings = settings or get_settings()- self.rubric = rubric+ self.track = track+ self.language = language+ # articulated batteries are judged on articulated_v1 (rest views + the pose sheet the+ # joint sweep renders); the rubric follows the battery's track unless a caller pins one+ self.rubric = rubric or RUBRIC_BY_TRACK.get(track.value, RUBRIC)self._runtime: Any = Noneself._judge: Any = None@@ -63,7 +69,7 @@ class FixedEvaluator:if self._runtime is None:from codeverse.languages import get_runtime- self._runtime = get_runtime(Language.BLENDER)+ self._runtime = get_runtime(self.language)return self._runtime@property@@ -95,6 +101,16 @@ class FixedEvaluator:out.gates.append(check_connectivity(glb))r = self.settings.renderout.renders = render_glb(glb, ws.renders_dir(0), views=list(OBJECT_VIEWS), width=r.width, height=r.height, sheet=True)+ if self.track is Track.ARTICULATED_OBJECT:+ # the same deterministic articulation evidence the track gives its judge: the+ # joint sweep (collisions over every joint's range) and the pose sheet / pose+ # tiles, read straight from the built URDF — no plan, so every arm is treated alike+ from codeverse.tracks.articulated_object import default_joint_sweep++ sweep, pose_views = default_joint_sweep(ws, None, ws.renders_dir(0) / "poses")+ out.gates.append(sweep)+ if pose_views:+ out.renders.views = list(out.renders.views) + pose_viewsinp = JudgeInput(spec=spec, renders=out.renders, measurement=out.measurement, gates=out.gates,acceptance=acceptance_from_spec(spec), round_index=0)out.judgment = self.judge.judge(inp)diff --git a/harness/bench/_oneshot.py b/harness/bench/_oneshot.pyindex ad6faede..50c3d1b9 100644--- a/harness/bench/_oneshot.py+++ b/harness/bench/_oneshot.py@@ -41,7 +41,7 @@ from codeverse.agents.watchdog import run_with_watchdogfrom codeverse.config import get_settingsfrom codeverse.contracts.artifacts import BuildResult, GateReportfrom codeverse.contracts.chat import ChatMessage, ChatRequest-from codeverse.contracts.common import Usage+from codeverse.contracts.common import Language, Usagefrom codeverse.contracts.spec import Specfrom codeverse.conventions import LANGUAGE_FRAME, frame_docfrom codeverse.tracks.generation import MultiFileParseError, parse_multifile@@ -49,6 +49,7 @@ from codeverse.tracks.repair import format_error_reportfrom codeverse.workspace import WorkspaceMODEL_FILE = "src/model.py"+URDF_FILE = "src/robot.urdf"DEFAULT_TIMEOUT_S = 900.0IDLE_GRACE_S = 600.0@@ -77,32 +78,98 @@ _OUTPUT_RULE = (f"You have NO tools in this session: you cannot write files, runf"the code must appear in your reply. Reply with the COMPLETE contents of `{MODEL_FILE}` as ONE "f"```python fenced code block and nothing else (no prose before or after, no partial snippets).")--def minimal_contract() -> str:+#: The URDF (track articulated_object, language urdf_blender) minimal contract: the SAME+#: bpy rules for the link meshes plus the frame recipe of codeverse/prompts/urdf/contract.md+#: (D18) condensed to its rules — no worked example, no cookbook, no skeleton: the one-shot+#: arm gets what a careful reader of the contract would know, nothing the harness builds.+_URDF_CONTRACT_BODY = """You write TWO files.+`{model_file}` in raw Blender Python (bpy, Blender 5.x API): ONE mesh object per LINK, `obj.name` == the+link name, built in WORLD coordinates at the REST pose (URDF q = 0: doors closed, drawers in, lids down,+arms at home). The harness runs it headless in an emptied scene and exports `meshes/<link>.glb` for every+link with the world coordinates baked in — you never export.+`{urdf_file}` hand-written URDF (links + joints) for the SAME object.++Frame, units and placement: {frame} Build at real-world size, on z = 0.++URDF recipe (the harness verifies every line numerically and FAILS the build otherwise):+- Every joint `rpy="0 0 0"`: all link frames stay axis-aligned with the world.+- Root link frame = world origin (0,0,0). Every other link frame = its joint's PIVOT: a world point on the+ joint axis (hinge line / slide axis / axle); for a fixed joint, the child's attachment point.+- `<joint><origin xyz>` = pivot_child − frame_parent (plain subtraction; for a child of the root that is the+ pivot itself).+- `<visual>` AND `<collision>` `<origin xyz>` = −pivot_link (root: 0 0 0), because the mesh holds world+ coordinates and the link frame sits at the pivot. One `<visual>` and one identical `<collision>` per link,+ geometry `<mesh filename="meshes/<link>.glb"/>`, no scale.+- `<axis xyz>` is a unit vector in WORLD coordinates; positive q must move the child the way a user expects+ (door swings open, drawer pulls out towards −Y, lid lifts up) — negate the axis if not, never swap limits.+- `revolute` / `prismatic` joints need `<limit lower upper effort velocity/>` with lower ≤ 0 ≤ upper;+ `continuous` only `<limit effort velocity/>`; `fixed` none.+- One tree: exactly one root link, every other link the child of exactly one joint, no cycles, ≤ 60 links.+ Hardware on a moving part (handle on a door) attaches to the moving link with a `fixed` joint.+- Clearance 1–3 mm between a moving part and its housing over the whole range; fixed children touch their+ parent (gap ≤ 2 mm).+- Names: link name == Blender object name == `meshes/<link>.glb` stem, case-sensitive, identical in both files+ (snake_case, e.g. `door`, `handle_left`; never `Door.001`); joint names unique; `world` is reserved.+- Forbidden in the URDF: `<gazebo>`, `<transmission>`, `<sensor>`, xacro, `package://`, mesh scale, inline+ primitives, `mimic`. Inertial blocks optional.++`{model_file}` rules: one mesh object per link linked to `bpy.context.scene.collection`, every mesh gets a+material (Principled BSDF); do NOT create cameras or lights, touch render/world settings, call+`bpy.ops.render.*`, `bpy.ops.export_*`, `bpy.ops.import_*`, `bpy.ops.wm.*`, read or write files, use the+network, or import anything other than `bpy`, `bmesh`, `mathutils`, `math`, `random`. Under 500k+triangles, under 120 s. The script must build every link when executed top-to-bottom (call your `main()`+at module level) and run without errors in headless Blender — there is no second chance."""++_OUTPUT_RULE_URDF = (f"You have NO tools in this session: you cannot write files, run Blender or inspect anything — "+ f"both files must appear in your reply. Reply with the COMPLETE contents of the two files in "+ f"this exact envelope and nothing else (no prose before, between or after):\n"+ f"=== FILE: {MODEL_FILE} ===\n<python>\n=== END FILE ===\n"+ f"=== FILE: {URDF_FILE} ===\n<xml>\n=== END FILE ===")+++def files_for(language: Language) -> list[str]:+ """The files a one-shot answer must contain for ``language``."""+ return [MODEL_FILE, URDF_FILE] if language is Language.URDF_BLENDER else [MODEL_FILE]+++def minimal_contract(language: Language = Language.BLENDER) -> str:"""Frame/units come from ``codeverse.conventions`` (the single source of truth)."""+ if language is Language.URDF_BLENDER:+ return _URDF_CONTRACT_BODY.format(model_file=MODEL_FILE, urdf_file=URDF_FILE,+ frame=frame_doc(LANGUAGE_FRAME["urdf_blender"]))return _CONTRACT_BODY.format(model_file=MODEL_FILE, frame=frame_doc(LANGUAGE_FRAME["blender"]))+def output_rule(language: Language = Language.BLENDER) -> str:+ return _OUTPUT_RULE_URDF if language is Language.URDF_BLENDER else _OUTPUT_RULE++def oneshot_prompt(spec: Spec) -> str:"""The entire input of a one-shot arm: brief + constraints + minimal contract + output rule."""c = spec.constraints- lines = [f"Model this object in raw bpy: {spec.prompt.strip()}"]+ urdf = spec.language is Language.URDF_BLENDER+ lines = [(f"Model this ARTICULATED object as raw bpy link meshes plus a hand-written URDF: {spec.prompt.strip()}"+ if urdf else f"Model this object in raw bpy: {spec.prompt.strip()}")]if c.dimensions_m:lines.append("Dimensions (m): " + ", ".join(f"{k}={v:g}" for k, v in c.dimensions_m.items()))if c.style:lines.append(f"Style: {c.style}")lines.extend(f"MUST HAVE: {m}" for m in c.must_have)lines.extend(f"MUST NOT: {m}" for m in c.must_not)- return "\n".join(lines) + "\n\n" + minimal_contract() + "\n\n" + _OUTPUT_RULE+ return "\n".join(lines) + "\n\n" + minimal_contract(spec.language) + "\n\n" + output_rule(spec.language)-def repair_prompt(spec: Spec, previous_code: str, build: BuildResult, lint: GateReport, attempt: int) -> str:- """Error-feedback retry (``oneshot+repair`` arm only): previous file + the build error report."""+def repair_prompt(spec: Spec, previous_code: str | dict[str, str], build: BuildResult, lint: GateReport,+ attempt: int) -> str:+ """Error-feedback retry (``oneshot+repair`` arm only): previous file(s) + the build error report."""report = format_error_report(build, lint)+ files = {MODEL_FILE: previous_code} if isinstance(previous_code, str) else previous_code+ prev = "\n\n".join(f"PREVIOUS `{path}`:\n```{'xml' if path.endswith('.urdf') else 'python'}\n{body[:40_000]}\n```"+ for path, body in files.items())+ plural = "files" if len(files) > 1 else "file"return (oneshot_prompt(spec)+ f"\n\nYour previous attempt ({attempt}) did NOT build. Fix the error below (smallest correct change, "- f"keep every part) and return the COMPLETE corrected file.\n\nERROR REPORT:\n{report}\n\n"- f"PREVIOUS `{MODEL_FILE}`:\n```python\n{previous_code[:40_000]}\n```")+ f"keep every part) and return the COMPLETE corrected {plural}.\n\nERROR REPORT:\n{report}\n\n{prev}")# ----------------------------------------------------------------------------- results@@ -159,6 +226,35 @@ def write_model_file(ws: Workspace, text: str) -> Path:return dest+def extract_files(text: str, language: Language = Language.BLENDER) -> dict[str, str]:+ """Every file a one-shot answer must contain for ``language`` → ``{path: body}``.++ Single-file languages keep the tolerant static path (:func:`extract_model_file`). A+ URDF answer must carry BOTH files in the ``=== FILE: … ===`` envelope (or fenced blocks+ each preceded by its path); an answer with only one of them is a format failure —+ ``MultiFileParseError`` → the cell is ``no_code``, an earned zero.+ """+ expected = files_for(language)+ if len(expected) == 1:+ return {MODEL_FILE: extract_model_file(text)}+ files = parse_multifile(text, expected_files=expected)+ missing = [f for f in expected if not (files.get(f) or "").strip()]+ if missing:+ raise MultiFileParseError(f"answer lacks {missing} (got {sorted(files)})")+ return {f: (files[f] if files[f].endswith("\n") else files[f] + "\n") for f in expected}+++def write_answer_files(ws: Workspace, text: str, language: Language = Language.BLENDER) -> list[Path]:+ """Parse the answer and write every expected file into the workspace (raises MultiFileParseError)."""+ written = []+ for rel, body in extract_files(text, language).items():+ dest = ws.root / rel+ dest.parent.mkdir(parents=True, exist_ok=True)+ dest.write_text(body)+ written.append(dest)+ return written++# ----------------------------------------------------------------------------- commondef _scratch_cwd(label: str) -> Path:"""A fresh directory outside any repo: no CLAUDE.md/AGENTS.md/.git context can leak in."""diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.pyindex b398c4e3..f4486fd6 100644--- a/harness/bench/compare_backends.py+++ b/harness/bench/compare_backends.py@@ -64,10 +64,11 @@ from bench._oneshot import ( # noqa: E402MODEL_FILE,OneShotBackend,OneShotResult,+ files_for,get_oneshot_backend,oneshot_prompt,repair_prompt,- write_model_file,+ write_answer_files,)from bench.run_bench import ( # noqa: E402Battery,@@ -197,7 +198,7 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, optsres.error_is_infra = gen.infra_failedreturntry:- write_model_file(eval_ws, gen.text)+ write_answer_files(eval_ws, gen.text, spec.language)except MultiFileParseError as e:res.error = f"unparseable answer: {e}"return@@ -207,7 +208,8 @@ def _generate_oneshot(arm: Arm, spec: Spec, cell: Path, eval_ws: Workspace, optsbuild, lint = deps.evaluator.build(eval_ws) # repair arm: error feedback onlyif build.ok and not lint.errors:return- prompt = repair_prompt(spec, (eval_ws.root / MODEL_FILE).read_text(), build, lint, attempt + 1)+ previous = {rel: (eval_ws.root / rel).read_text() for rel in files_for(spec.language) if (eval_ws.root / rel).is_file()}+ prompt = repair_prompt(spec, previous, build, lint, attempt + 1)def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, deps: CompareDeps, res: CellResult) -> None:@@ -495,7 +497,8 @@ def main(argv: Sequence[str] | None = None) -> int:arms = parse_arms(ns.arms)if not ns.no_preflight and not _preflight(opts.judge, arms, opts, wait_minutes=ns.wait_for_provider):return 2- deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples))+ battery = Battery.load(ns.prompts) # the fixed evaluator follows the battery's track / language+ deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples, track=battery.track, language=battery.language))def _log(r: CellResult) -> None:print(f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:44s} score={r.score} build_ok={r.build_ok} "diff --git a/harness/tests/compare_bench/test_articulated_compare.py b/harness/tests/compare_bench/test_articulated_compare.pynew file mode 100644index 00000000..21ded876--- /dev/null+++ b/harness/tests/compare_bench/test_articulated_compare.py@@ -0,0 +1,122 @@+"""compare_backends on an articulated battery: two-file one-shot answers, articulated_v1 fixed judge."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++REPO = Path(__file__).resolve().parents[2]+if str(REPO) not in sys.path:+ sys.path.insert(0, str(REPO))++from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402+from bench._fixed_eval import FixedEvaluator # noqa: E402+from bench._oneshot import ( # noqa: E402+ MODEL_FILE,+ URDF_FILE,+ extract_files,+ files_for,+ minimal_contract,+ oneshot_prompt,+ repair_prompt,+ write_answer_files,+)+from bench.run_bench import Battery # noqa: E402+from codeverse.contracts.artifacts import BuildResult, GateReport # noqa: E402+from codeverse.contracts.common import Language, Track # noqa: E402+from codeverse.contracts.spec import Constraints, Spec # noqa: E402+from codeverse.tracks.generation import MultiFileParseError # noqa: E402+from codeverse.workspace import Workspace # noqa: E402++PROMPTS = REPO / "bench" / "prompts"+PY = "import bpy\nprint(1)\n"+XML = '<?xml version="1.0"?>\n<robot name="x"><link name="base"/></robot>\n'+ENVELOPE = f"=== FILE: {MODEL_FILE} ===\n{PY}=== END FILE ===\n=== FILE: {URDF_FILE} ===\n{XML}=== END FILE ===\n"+++def _urdf_spec() -> Spec:+ return Spec(id="t/lamp", track=Track.ARTICULATED_OBJECT, language=Language.URDF_BLENDER,+ prompt="an architect lamp with two hinged arms",+ constraints=Constraints(must_have=["two revolute joints"], dimensions_m={"height": 0.6}))+++def test_urdf_oneshot_prompt_carries_the_frame_recipe_and_the_two_file_envelope():+ p = oneshot_prompt(_urdf_spec())+ assert "hand-written URDF" in p and "architect lamp" in p and "MUST HAVE: two revolute joints" in p+ for rule in ("rpy=\"0 0 0\"", "pivot_child − frame_parent", "−pivot_link", "meshes/<link>.glb", "lower ≤ 0 ≤ upper"):+ assert rule in p, rule+ assert f"=== FILE: {MODEL_FILE} ===" in p and f"=== FILE: {URDF_FILE} ===" in p+ # no harness help: no worked example, no cookbook, no skeleton+ assert "pedal" not in p.lower() and "cookbook" not in p.lower() and "PIVOT[" not in p+ assert minimal_contract(Language.URDF_BLENDER) in p+ # the static contract is untouched by the language switch+ assert minimal_contract() == minimal_contract(Language.BLENDER) and URDF_FILE not in minimal_contract()+++def test_files_for_and_extract_files_two_file_answer():+ assert files_for(Language.BLENDER) == [MODEL_FILE] and files_for(Language.URDF_BLENDER) == [MODEL_FILE, URDF_FILE]+ got = extract_files(ENVELOPE, Language.URDF_BLENDER)+ assert got == {MODEL_FILE: PY, URDF_FILE: XML}+ # fenced blocks each preceded by their path also parse+ fenced = f"{MODEL_FILE}\n```python\n{PY}```\n{URDF_FILE}\n```xml\n{XML}```\n"+ assert extract_files(fenced, Language.URDF_BLENDER) == {MODEL_FILE: PY, URDF_FILE: XML}+ # the static path is unchanged+ assert extract_files("```python\nimport bpy\n```", Language.BLENDER) == {MODEL_FILE: "import bpy\n"}++[email protected]("text", [+ f"```python\n{PY}```", # one block, two expected+ f"=== FILE: {MODEL_FILE} ===\n{PY}=== END FILE ===\n", # envelope with the URDF missing+ f"=== FILE: {MODEL_FILE} ===\n{PY}=== END FILE ===\n=== FILE: {URDF_FILE} ===\n\n=== END FILE ===\n", # empty urdf+])+def test_a_urdf_answer_without_both_files_is_a_format_failure(text: str):+ with pytest.raises(MultiFileParseError):+ extract_files(text, Language.URDF_BLENDER)+++def test_write_answer_files_writes_both(tmp_path):+ ws = Workspace(tmp_path / "ws")+ ws.create()+ paths = write_answer_files(ws, ENVELOPE, Language.URDF_BLENDER)+ assert [p.relative_to(ws.root).as_posix() for p in paths] == [MODEL_FILE, URDF_FILE]+ assert (ws.root / URDF_FILE).read_text() == XML and (ws.root / MODEL_FILE).read_text() == PY+++def test_urdf_repair_prompt_carries_both_previous_files():+ build = BuildResult(ok=False, language="urdf_blender", error_type="FKMismatch",+ error_message="link 'arm' visual origin should be -0.1 0 0.3", error_file=URDF_FILE, error_line=9)+ p = repair_prompt(_urdf_spec(), {MODEL_FILE: PY, URDF_FILE: XML}, build, GateReport(gate="lint:urdf", passed=True), attempt=1)+ assert "FKMismatch" in p and "corrected files" in p+ assert f"PREVIOUS `{MODEL_FILE}`" in p and f"PREVIOUS `{URDF_FILE}`" in p and "```xml" in p+++def test_fixed_evaluator_follows_the_battery_track():+ ev = FixedEvaluator("gemini:x", n_samples=3, track=Track.ARTICULATED_OBJECT, language=Language.URDF_BLENDER)+ assert ev.rubric == "articulated_v1" == RUBRIC_BY_TRACK["articulated_object"]+ assert ev.language is Language.URDF_BLENDER and ev.runtime.language is Language.URDF_BLENDER+ static = FixedEvaluator("gemini:x")+ assert static.rubric == "static_object_v1" and static.language is Language.BLENDER+ pinned = FixedEvaluator("gemini:x", rubric="asset_v1", track=Track.ARTICULATED_OBJECT)+ assert pinned.rubric == "asset_v1"+++def test_main_builds_the_evaluator_from_the_battery(monkeypatch, tmp_path):+ from bench import compare_backends as cb++ seen = {}++ def fake_run_matrix(battery_path, out_dir, arms, opts, deps, on_result=None):+ seen["track"], seen["language"], seen["rubric"] = deps.evaluator.track, deps.evaluator.language, deps.evaluator.rubric+ (Path(out_dir)).mkdir(parents=True, exist_ok=True)+ (Path(out_dir) / "results.jsonl").write_text("")++ monkeypatch.setattr(cb, "run_matrix", fake_run_matrix)+ monkeypatch.setattr(cb, "_preflight", lambda *a, **k: True)+ rc = cb.main(["--prompts", str(PROMPTS / "articulated_v2.yaml"), "--arms", "oneshot:gemini:x",+ "--out", str(tmp_path / "out"), "--no-preflight", "--judge-samples", "3"])+ assert rc == 0+ assert seen == {"track": Track.ARTICULATED_OBJECT, "language": Language.URDF_BLENDER, "rubric": "articulated_v1"}+ b = Battery.load(PROMPTS / "articulated_v2.yaml")+ assert b.track is Track.ARTICULATED_OBJECT and all(p.must_have for p in b.prompts)