Official Codebase of 3DCodeVerse
Python
bench: the mechanism claim is recomputed from the recorded URDFs, and narrowed
ziyao6fda377f8af7
diff --git a/harness/bench/coupling_stats.py b/harness/bench/coupling_stats.pynew file mode 100644index 00000000..882da8c4--- /dev/null+++ b/harness/bench/coupling_stats.py@@ -0,0 +1,121 @@+"""How many joints a battery's mechanisms really have, against how many the sweep drives.++A coupled mechanism (umbrella ribs on one runner, a pantograph, a folding brace) has ONE+input and many moving links. Before ``<mimic>`` support the pose sampler drove every+movable joint independently and posed states the mechanism cannot reach; it now drives+``independent_joints()`` only. Both counts are properties of the recorded URDF, so this+reads the difference off the artefacts without re-running anything:++ python bench/coupling_stats.py bench/out/wave2_lean++``joints`` is every movable joint (what the sampler used to drive), ``independent`` is+what it drives now, and ``coupled prompts`` is how many of the battery's prompts declared+a coupling at all — the mechanism claim in docs/PAPER_WRITING.md §5.3.+"""++from __future__ import annotations++import argparse+import statistics+import sys+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) # `python bench/coupling_stats.py` from the repo root++from codeverse.spatial.joints_model import UrdfError, load_urdf # noqa: E402+++def _urdfs(root: Path) -> list[Path]:+ """Every built ``robot.urdf`` under ``root``, deduplicated by resolved path."""+ seen: dict[Path, Path] = {}+ for p in root.rglob("robot.urdf"):+ seen.setdefault(p.resolve(), p)+ return sorted(seen.values())+++def _prompt_of(path: Path) -> str:+ """The battery cell a URDF belongs to (``cells/<prompt>/<arm>/run/...``)."""+ parts = path.parts+ return parts[parts.index("cells") + 1] if "cells" in parts else path.parent.name+++def per_prompt(root: Path) -> str:+ """One row per prompt that declared a coupling: what it moves, what drives it."""+ rows: dict[str, list[tuple[int, int]]] = {}+ for path in _urdfs(root):+ try:+ robot = load_urdf(path, load_meshes=False)+ except (UrdfError, OSError):+ continue+ moves, free = len(robot.movable_joints()), len(robot.independent_joints())+ if moves != free:+ rows.setdefault(_prompt_of(path), []).append((moves, free))+ out = [f"{root.name}, per prompt (coupled URDFs only):",+ "| prompt | URDFs | movable joints (median) | driven per pose (median) | fewest driven |",+ "|---|--:|--:|--:|--:|"]+ for prompt in sorted(rows):+ cells = rows[prompt]+ out.append(f"| {prompt} | {len(cells)} | {statistics.median(m for m, _ in cells):.0f} | "+ f"{statistics.median(i for _, i in cells):.0f} | {min(i for _, i in cells)} |")+ return "\n".join(out)+++def survey(root: Path) -> dict:+ movable: list[int] = []+ independent: list[int] = []+ coupled_prompts: set[str] = set()+ prompts: set[str] = set()+ unreadable = 0+ for path in _urdfs(root):+ try:+ robot = load_urdf(path, load_meshes=False)+ except (UrdfError, OSError):+ unreadable += 1+ continue+ moves = [j for j in robot.joints.values() if j.movable]+ free = robot.independent_joints()+ prompts.add(_prompt_of(path))+ movable.append(len(moves))+ independent.append(len(free))+ if len(free) != len(moves):+ coupled_prompts.add(_prompt_of(path))+ coupled = [(m, i) for m, i in zip(movable, independent, strict=True) if m != i]+ return {"urdfs": len(movable), "unreadable": unreadable, "prompts": len(prompts),+ "coupled_prompts": len(coupled_prompts), "movable": movable, "independent": independent,+ "coupled": coupled}+++def report(roots: list[Path]) -> str:+ lines = ["| battery | URDFs | prompts | prompts with a coupling | coupled URDFs | "+ "joints, coupled URDFs (median) | driven per pose there (median) | one-input |",+ "|---|--:|--:|--:|--:|--:|--:|--:|"]+ for root in roots:+ s = survey(root)+ if not s["urdfs"]:+ lines.append(f"| {root.name} | 0 | — | — | — | — | — | — |")+ continue+ # the claim is about the mechanisms that HAVE a coupling: an uncoupled cabinet+ # dilutes both medians towards each other and says nothing either way+ moves = [m for m, _ in s["coupled"]] or [0]+ free = [i for _, i in s["coupled"]] or [0]+ lines.append(+ f"| {root.name} | {s['urdfs']} | {s['prompts']} | {s['coupled_prompts']} | "+ f"{len(s['coupled'])} | {statistics.median(moves):.0f} | {statistics.median(free):.0f} | "+ f"{sum(1 for n in free if n == 1)} |")+ return "\n".join(lines)+++def main(argv: list[str] | None = None) -> int:+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])+ ap.add_argument("roots", nargs="+", type=Path)+ ap.add_argument("--per-prompt", action="store_true", help="also break each battery down by prompt")+ ns = ap.parse_args(argv)+ print(report(ns.roots))+ for root in ns.roots if ns.per_prompt else []:+ print()+ print(per_prompt(root))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/harness/bench/data/plan_stage/trigger_narrow.jsonl b/harness/bench/data/plan_stage/trigger_narrow.jsonlindex 66f713b4..39f0777a 100644--- a/harness/bench/data/plan_stage/trigger_narrow.jsonl+++ b/harness/bench/data/plan_stage/trigger_narrow.jsonl@@ -101,3 +101,4 @@{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 0, "ok": true, "error": "", "seconds": 205.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0433, "n_parts": 9, "attempt": 0}{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 1, "ok": true, "error": "", "seconds": 123.1, "invalid_reasks": 1, "geometry_reasks": 0, "restarts": 1, "missing": ["LeftLegFrame", "RightLegFrame", "LeftBraceUpper", "LeftBraceLower", "RightBraceUpper", "RightBraceLower", "CarryHandle", "TransportLatch"], "cost_usd": 0.0691, "n_parts": 9, "attempt": 1}{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 2, "ok": true, "error": "", "seconds": 354.7, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0487, "n_parts": 9, "attempt": 0}+{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 3, "ok": true, "error": "", "seconds": 561.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0319, "n_parts": 9, "attempt": 0}diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.mdindex 7cced247..925b9974 100644--- a/harness/docs/DECISIONS.md+++ b/harness/docs/DECISIONS.md@@ -720,12 +720,14 @@ written) that were accepted because the code works that way and the tests pin itthe measurement, a toolkit writes whatever a trainer wants (owner's boundary,2026-09-03).* **`<mimic>` is used and it changes the sweep, which is what it was for.** In the same- battery the planner and agent declared couplings in 8 of 14 prompts (umbrella 6 joints,- workbench 4, step ladder 4, scissor mirror 3, rolltop desk 3), and the sampled poses- now drive 1-2 joints where the same prompts previously produced 5-6-joint random- combinations — the mechanism is posed through its own input. The score effect on the- three coupled prompts is inside the noise band at n=2 per side, as expected; the pose- count is the readout that resolves.+ battery the planner and agent declared couplings in 8 of 14 prompts (98 of 176 built+ URDFs), and on those the sampler drives a median of 3 joints per pose against the 6 it+ would have driven before — 1 on the umbrella and the step ladder, 2 on the folding+ workbench, the mechanisms the support was built for. The pre-mimic batteries contain+ zero couplings, so nothing else changed shape. `bench/coupling_stats.py --per-prompt`+ recomputes all of it from the recorded URDFs (both counts are properties of the file).+ The score effect on the three coupled prompts is inside the noise band at n=2 per side,+ as expected; the pose count is the readout that resolves.## Rejected / deferreddiff --git a/harness/docs/PAPER_WRITING.md b/harness/docs/PAPER_WRITING.mdindex 3783a3f9..a6bafac3 100644--- a/harness/docs/PAPER_WRITING.md+++ b/harness/docs/PAPER_WRITING.md@@ -261,11 +261,27 @@ driven joint is ignored); `pose_samples` drives `independent_joints()` only; thecheck probes a driven joint **through its driver**; `JointPlan.mimic` makes the coupling aplanner decision and the skeleton writes it; the lint checks it instead of dismissing it.-**Measured** (`wave2_lean`, both arms): the planner and agent declared couplings in **8 of-14 prompts** (umbrella 6 joints, folding workbench 4, step ladder 4, scissor mirror 3,-rolltop desk 3, one each on casement window / grand piano / architect lamp), and the sampled-poses now drive **1–2 joints** where the same prompts previously produced **5–6-joint**-random combinations. The score effect on the three coupled prompts is inside the noise band+**Measured** (`wave2_lean`, both arms; `python bench/coupling_stats.py bench/out/wave2_lean+--per-prompt` recomputes every number here from the recorded URDFs — what the sampler used+to drive is every movable joint, what it drives now is the independent ones, and both are+properties of the file):++| prompt | coupled URDFs | movable joints (median) | driven per pose (median) |+|---|--:|--:|--:|+| umbrella | 12 | 7 | **1** |+| step ladder | 12 | 5 | **1** |+| folding workbench | 12 | 6 | **2** |+| scissor mirror | 12 | 6 | **3** |+| grand piano | 12 | 4 | 3 |+| architect lamp | 20 | 5 | 4 |+| rolltop desk | 12 | 8 | 5 |+| casement window | 6 | 7 | 6 |++**8 of 14 prompts** declared a coupling at all (98 of 176 built URDFs); across those the+median goes **6 → 3** driven joints, and 24 URDFs are fully one-input. The mechanisms the+support was built for — umbrella, step ladder, folding workbench — are the ones that collapse+to 1–2. The pre-mimic batteries (`aa_articulated`, `compare_art_v4_pf0`) contain **zero**+couplings, so nothing else on the battery changed shape. The score effect on the three coupled prompts is inside the noise bandat n = 2 per side (umbrella 0.35/0.65 → 0.54/0.64, scissor 0.46/0.54 → 0.49/0.60, workbench0.91/0.60 → 0.60/0.60); the **pose count is the readout that resolves.**@@ -328,8 +344,8 @@ instructions nor a diff.plausible levers landed inside that band; none of them is thereby shown to do nothing.3. **The changes that mattered were found by counting loss events, not by scoring artefacts.**Planner mortality 4.7 % → 0.7 % (p = 0.0067); retried tool calls 26.6 % → 1.3 %; uncached- tokens −64 %; dollars per round −38 %; sampled poses of a coupled mechanism from 5–6- joints to 1–2. Every one of these is invisible to the judge mean at this n.+ tokens −64 %; dollars per round −40 %; driven joints per sampled pose on a coupled+ mechanism 6 → 3 (1 on the umbrella and the step ladder). Every one of these is invisible to the judge mean at this n.4. **The biggest single defect was in the harness's own plumbing**, not in the model or theprompt: one line mapping a tool's verdict to a protocol error cost a quarter of all toolcalls to retries and two thirds of the uncached token bill. It was found by reading@@ -387,7 +403,7 @@ instructions nor a diff.| generator $ per round | median 1.572 → 0.950 | round records of both runs | same || base64 amplification | 261 k tokens vs 516 for one 275 kB sheet | measured on a recorded blob + vendor bundle | arithmetic + matching recorded prompt jumps || token growth in turns | input(t) = 12 364 + 1 822·t; total ∝ n^1.60; dollars ∝ n^1.19–1.34 | 1 063 in-process sessions, 30 401 turns | the in-process agent, since gemini-cli emits no per-turn usage |-| coupled mechanisms declared | 8 of 14 prompts; poses drive 1–2 joints (was 5–6) | `wave2_lean` URDFs and round records | — |+| coupled mechanisms declared | 8 of 14 prompts; driven joints per pose 6 → 3 (1–2 on the one-input mechanisms) | `bench/coupling_stats.py bench/out/wave2_lean --per-prompt` | recomputed from the recorded URDFs || refine corpus | 254 transitions, 112 SFT samples (2026-09-03) | `3dcv flywheel refine`, `toolkits/llamafactory/build_refine_sft.py` over `bench/out` | corpus grows with every battery |## 9.2 Open questionsdiff --git a/harness/tests/compare_bench/test_session_stats.py b/harness/tests/compare_bench/test_session_stats.pyindex 463e71fc..c5d5ed84 100644--- a/harness/tests/compare_bench/test_session_stats.py+++ b/harness/tests/compare_bench/test_session_stats.py@@ -68,3 +68,33 @@ def test_a_dead_session_is_skipped_not_fatal(tmp_path: Path) -> None:assert len(_sessions(tmp_path)) == 2calls, _ = tool_rates(_sessions(tmp_path))assert calls == {"build": 3}+++def test_the_coupling_survey_counts_degrees_of_freedom_not_joints(tmp_path: Path) -> None:+ """``bench/coupling_stats.py`` reads the mechanism claim off the recorded URDFs: what+ the sampler used to drive (every movable joint) against what it drives now (the+ independent ones). Both are properties of the file, so no re-run is needed."""+ from bench.coupling_stats import per_prompt, report, survey++ urdf = """<?xml version="1.0"?>+<robot name="rig">+ <link name="base"/><link name="a"/><link name="b"/>+ <joint name="drive" type="revolute"><parent link="base"/><child link="a"/>+ <axis xyz="0 0 1"/><limit lower="0" upper="1" effort="1" velocity="1"/></joint>+ <joint name="follow" type="revolute"><parent link="a"/><child link="b"/>+ <axis xyz="0 0 1"/><limit lower="0" upper="1" effort="1" velocity="1"/>+ <mimic joint="drive" multiplier="1" offset="0"/></joint>+</robot>+"""+ cell = tmp_path / "cells" / "art_hard_umbrella" / "arm" / "run" / "artifacts"+ cell.mkdir(parents=True)+ (cell / "robot.urdf").write_text(urdf)+ (tmp_path / "cells" / "plain" / "arm" / "run" / "artifacts").mkdir(parents=True)+ (tmp_path / "cells" / "plain" / "arm" / "run" / "artifacts" / "robot.urdf").write_text(+ urdf.replace('<mimic joint="drive" multiplier="1" offset="0"/>', ""))++ s = survey(tmp_path)+ assert s["urdfs"] == 2 and s["prompts"] == 2 and s["coupled_prompts"] == 1+ assert s["coupled"] == [(2, 1)] # 2 movable joints, 1 degree of freedom+ assert "| 2 | 2 | 1 | 1 | 2 | 1 | 1 |" in report([tmp_path])+ assert "| art_hard_umbrella | 1 | 2 | 1 | 1 |" in per_prompt(tmp_path)