Skip to content

Official Codebase of 3DCodeVerse

Python

bench: the mechanism claim is recomputed from the recorded URDFs, and narrowed

ziyao6fda377f8af7
diff --git a/harness/bench/coupling_stats.py b/harness/bench/coupling_stats.py
new file mode 100644
index 00000000..882da8c4
--- /dev/null
+++ b/harness/bench/coupling_stats.py
@@ -0,0 +1,121 @@
+"""How many joints a battery's mechanisms really have, against how many the sweep drives.
+
+A coupled mechanism (umbrella ribs on one runner, a pantograph, a folding brace) has ONE
+input and many moving links. Before ``<mimic>`` support the pose sampler drove every
+movable joint independently and posed states the mechanism cannot reach; it now drives
+``independent_joints()`` only. Both counts are properties of the recorded URDF, so this
+reads the difference off the artefacts without re-running anything:
+
+ python bench/coupling_stats.py bench/out/wave2_lean
+
+``joints`` is every movable joint (what the sampler used to drive), ``independent`` is
+what it drives now, and ``coupled prompts`` is how many of the battery's prompts declared
+a coupling at all — the mechanism claim in docs/PAPER_WRITING.md §5.3.
+"""
+
+from __future__ import annotations
+
+import argparse
+import statistics
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) # `python bench/coupling_stats.py` from the repo root
+
+from codeverse.spatial.joints_model import UrdfError, load_urdf # noqa: E402
+
+
+def _urdfs(root: Path) -> list[Path]:
+ """Every built ``robot.urdf`` under ``root``, deduplicated by resolved path."""
+ seen: dict[Path, Path] = {}
+ for p in root.rglob("robot.urdf"):
+ seen.setdefault(p.resolve(), p)
+ return sorted(seen.values())
+
+
+def _prompt_of(path: Path) -> str:
+ """The battery cell a URDF belongs to (``cells/<prompt>/<arm>/run/...``)."""
+ parts = path.parts
+ return parts[parts.index("cells") + 1] if "cells" in parts else path.parent.name
+
+
+def per_prompt(root: Path) -> str:
+ """One row per prompt that declared a coupling: what it moves, what drives it."""
+ rows: dict[str, list[tuple[int, int]]] = {}
+ for path in _urdfs(root):
+ try:
+ robot = load_urdf(path, load_meshes=False)
+ except (UrdfError, OSError):
+ continue
+ moves, free = len(robot.movable_joints()), len(robot.independent_joints())
+ if moves != free:
+ rows.setdefault(_prompt_of(path), []).append((moves, free))
+ out = [f"{root.name}, per prompt (coupled URDFs only):",
+ "| prompt | URDFs | movable joints (median) | driven per pose (median) | fewest driven |",
+ "|---|--:|--:|--:|--:|"]
+ for prompt in sorted(rows):
+ cells = rows[prompt]
+ out.append(f"| {prompt} | {len(cells)} | {statistics.median(m for m, _ in cells):.0f} | "
+ f"{statistics.median(i for _, i in cells):.0f} | {min(i for _, i in cells)} |")
+ return "\n".join(out)
+
+
+def survey(root: Path) -> dict:
+ movable: list[int] = []
+ independent: list[int] = []
+ coupled_prompts: set[str] = set()
+ prompts: set[str] = set()
+ unreadable = 0
+ for path in _urdfs(root):
+ try:
+ robot = load_urdf(path, load_meshes=False)
+ except (UrdfError, OSError):
+ unreadable += 1
+ continue
+ moves = [j for j in robot.joints.values() if j.movable]
+ free = robot.independent_joints()
+ prompts.add(_prompt_of(path))
+ movable.append(len(moves))
+ independent.append(len(free))
+ if len(free) != len(moves):
+ coupled_prompts.add(_prompt_of(path))
+ coupled = [(m, i) for m, i in zip(movable, independent, strict=True) if m != i]
+ return {"urdfs": len(movable), "unreadable": unreadable, "prompts": len(prompts),
+ "coupled_prompts": len(coupled_prompts), "movable": movable, "independent": independent,
+ "coupled": coupled}
+
+
+def report(roots: list[Path]) -> str:
+ lines = ["| battery | URDFs | prompts | prompts with a coupling | coupled URDFs | "
+ "joints, coupled URDFs (median) | driven per pose there (median) | one-input |",
+ "|---|--:|--:|--:|--:|--:|--:|--:|"]
+ for root in roots:
+ s = survey(root)
+ if not s["urdfs"]:
+ lines.append(f"| {root.name} | 0 | — | — | — | — | — | — |")
+ continue
+ # the claim is about the mechanisms that HAVE a coupling: an uncoupled cabinet
+ # dilutes both medians towards each other and says nothing either way
+ moves = [m for m, _ in s["coupled"]] or [0]
+ free = [i for _, i in s["coupled"]] or [0]
+ lines.append(
+ f"| {root.name} | {s['urdfs']} | {s['prompts']} | {s['coupled_prompts']} | "
+ f"{len(s['coupled'])} | {statistics.median(moves):.0f} | {statistics.median(free):.0f} | "
+ f"{sum(1 for n in free if n == 1)} |")
+ return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ ap.add_argument("roots", nargs="+", type=Path)
+ ap.add_argument("--per-prompt", action="store_true", help="also break each battery down by prompt")
+ ns = ap.parse_args(argv)
+ print(report(ns.roots))
+ for root in ns.roots if ns.per_prompt else []:
+ print()
+ print(per_prompt(root))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/harness/bench/data/plan_stage/trigger_narrow.jsonl b/harness/bench/data/plan_stage/trigger_narrow.jsonl
index 66f713b4..39f0777a 100644
--- a/harness/bench/data/plan_stage/trigger_narrow.jsonl
+++ b/harness/bench/data/plan_stage/trigger_narrow.jsonl
@@ -101,3 +101,4 @@
{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 0, "ok": true, "error": "", "seconds": 205.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0433, "n_parts": 9, "attempt": 0}
{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 1, "ok": true, "error": "", "seconds": 123.1, "invalid_reasks": 1, "geometry_reasks": 0, "restarts": 1, "missing": ["LeftLegFrame", "RightLegFrame", "LeftBraceUpper", "LeftBraceLower", "RightBraceUpper", "RightBraceLower", "CarryHandle", "TransportLatch"], "cost_usd": 0.0691, "n_parts": 9, "attempt": 1}
{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 2, "ok": true, "error": "", "seconds": 354.7, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0487, "n_parts": 9, "attempt": 0}
+{"tree": "trigger_narrow", "prompt": "art_hard_folding_workbench", "rep": 3, "ok": true, "error": "", "seconds": 561.0, "invalid_reasks": 0, "geometry_reasks": 0, "restarts": 0, "missing": [], "cost_usd": 0.0319, "n_parts": 9, "attempt": 0}
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index 7cced247..925b9974 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -720,12 +720,14 @@ written) that were accepted because the code works that way and the tests pin it
the measurement, a toolkit writes whatever a trainer wants (owner's boundary,
2026-09-03).
* **`<mimic>` is used and it changes the sweep, which is what it was for.** In the same
- battery the planner and agent declared couplings in 8 of 14 prompts (umbrella 6 joints,
- workbench 4, step ladder 4, scissor mirror 3, rolltop desk 3), and the sampled poses
- now drive 1-2 joints where the same prompts previously produced 5-6-joint random
- combinations — the mechanism is posed through its own input. The score effect on the
- three coupled prompts is inside the noise band at n=2 per side, as expected; the pose
- count is the readout that resolves.
+ battery the planner and agent declared couplings in 8 of 14 prompts (98 of 176 built
+ URDFs), and on those the sampler drives a median of 3 joints per pose against the 6 it
+ would have driven before — 1 on the umbrella and the step ladder, 2 on the folding
+ workbench, the mechanisms the support was built for. The pre-mimic batteries contain
+ zero couplings, so nothing else changed shape. `bench/coupling_stats.py --per-prompt`
+ recomputes all of it from the recorded URDFs (both counts are properties of the file).
+ The score effect on the three coupled prompts is inside the noise band at n=2 per side,
+ as expected; the pose count is the readout that resolves.
## Rejected / deferred
diff --git a/harness/docs/PAPER_WRITING.md b/harness/docs/PAPER_WRITING.md
index 3783a3f9..a6bafac3 100644
--- a/harness/docs/PAPER_WRITING.md
+++ b/harness/docs/PAPER_WRITING.md
@@ -261,11 +261,27 @@ driven joint is ignored); `pose_samples` drives `independent_joints()` only; the
check probes a driven joint **through its driver**; `JointPlan.mimic` makes the coupling a
planner decision and the skeleton writes it; the lint checks it instead of dismissing it.
-**Measured** (`wave2_lean`, both arms): the planner and agent declared couplings in **8 of
-14 prompts** (umbrella 6 joints, folding workbench 4, step ladder 4, scissor mirror 3,
-rolltop desk 3, one each on casement window / grand piano / architect lamp), and the sampled
-poses now drive **1–2 joints** where the same prompts previously produced **5–6-joint**
-random combinations. The score effect on the three coupled prompts is inside the noise band
+**Measured** (`wave2_lean`, both arms; `python bench/coupling_stats.py bench/out/wave2_lean
+--per-prompt` recomputes every number here from the recorded URDFs — what the sampler used
+to drive is every movable joint, what it drives now is the independent ones, and both are
+properties of the file):
+
+| prompt | coupled URDFs | movable joints (median) | driven per pose (median) |
+|---|--:|--:|--:|
+| umbrella | 12 | 7 | **1** |
+| step ladder | 12 | 5 | **1** |
+| folding workbench | 12 | 6 | **2** |
+| scissor mirror | 12 | 6 | **3** |
+| grand piano | 12 | 4 | 3 |
+| architect lamp | 20 | 5 | 4 |
+| rolltop desk | 12 | 8 | 5 |
+| casement window | 6 | 7 | 6 |
+
+**8 of 14 prompts** declared a coupling at all (98 of 176 built URDFs); across those the
+median goes **6 → 3** driven joints, and 24 URDFs are fully one-input. The mechanisms the
+support was built for — umbrella, step ladder, folding workbench — are the ones that collapse
+to 1–2. The pre-mimic batteries (`aa_articulated`, `compare_art_v4_pf0`) contain **zero**
+couplings, so nothing else on the battery changed shape. The score effect on the three coupled prompts is inside the noise band
at n = 2 per side (umbrella 0.35/0.65 → 0.54/0.64, scissor 0.46/0.54 → 0.49/0.60, workbench
0.91/0.60 → 0.60/0.60); the **pose count is the readout that resolves.**
@@ -328,8 +344,8 @@ instructions nor a diff.
plausible levers landed inside that band; none of them is thereby shown to do nothing.
3. **The changes that mattered were found by counting loss events, not by scoring artefacts.**
Planner mortality 4.7 % → 0.7 % (p = 0.0067); retried tool calls 26.6 % → 1.3 %; uncached
- tokens −64 %; dollars per round −38 %; sampled poses of a coupled mechanism from 5–6
- joints to 1–2. Every one of these is invisible to the judge mean at this n.
+ tokens −64 %; dollars per round −40 %; driven joints per sampled pose on a coupled
+ mechanism 6 → 3 (1 on the umbrella and the step ladder). Every one of these is invisible to the judge mean at this n.
4. **The biggest single defect was in the harness's own plumbing**, not in the model or the
prompt: one line mapping a tool's verdict to a protocol error cost a quarter of all tool
calls to retries and two thirds of the uncached token bill. It was found by reading
@@ -387,7 +403,7 @@ instructions nor a diff.
| generator $ per round | median 1.572 → 0.950 | round records of both runs | same |
| base64 amplification | 261 k tokens vs 516 for one 275 kB sheet | measured on a recorded blob + vendor bundle | arithmetic + matching recorded prompt jumps |
| token growth in turns | input(t) = 12 364 + 1 822·t; total ∝ n^1.60; dollars ∝ n^1.19–1.34 | 1 063 in-process sessions, 30 401 turns | the in-process agent, since gemini-cli emits no per-turn usage |
-| coupled mechanisms declared | 8 of 14 prompts; poses drive 1–2 joints (was 5–6) | `wave2_lean` URDFs and round records | — |
+| coupled mechanisms declared | 8 of 14 prompts; driven joints per pose 6 → 3 (1–2 on the one-input mechanisms) | `bench/coupling_stats.py bench/out/wave2_lean --per-prompt` | recomputed from the recorded URDFs |
| refine corpus | 254 transitions, 112 SFT samples (2026-09-03) | `3dcv flywheel refine`, `toolkits/llamafactory/build_refine_sft.py` over `bench/out` | corpus grows with every battery |
## 9.2 Open questions
diff --git a/harness/tests/compare_bench/test_session_stats.py b/harness/tests/compare_bench/test_session_stats.py
index 463e71fc..c5d5ed84 100644
--- a/harness/tests/compare_bench/test_session_stats.py
+++ b/harness/tests/compare_bench/test_session_stats.py
@@ -68,3 +68,33 @@ def test_a_dead_session_is_skipped_not_fatal(tmp_path: Path) -> None:
assert len(_sessions(tmp_path)) == 2
calls, _ = tool_rates(_sessions(tmp_path))
assert calls == {"build": 3}
+
+
+def test_the_coupling_survey_counts_degrees_of_freedom_not_joints(tmp_path: Path) -> None:
+ """``bench/coupling_stats.py`` reads the mechanism claim off the recorded URDFs: what
+ the sampler used to drive (every movable joint) against what it drives now (the
+ independent ones). Both are properties of the file, so no re-run is needed."""
+ from bench.coupling_stats import per_prompt, report, survey
+
+ urdf = """<?xml version="1.0"?>
+<robot name="rig">
+ <link name="base"/><link name="a"/><link name="b"/>
+ <joint name="drive" type="revolute"><parent link="base"/><child link="a"/>
+ <axis xyz="0 0 1"/><limit lower="0" upper="1" effort="1" velocity="1"/></joint>
+ <joint name="follow" type="revolute"><parent link="a"/><child link="b"/>
+ <axis xyz="0 0 1"/><limit lower="0" upper="1" effort="1" velocity="1"/>
+ <mimic joint="drive" multiplier="1" offset="0"/></joint>
+</robot>
+"""
+ cell = tmp_path / "cells" / "art_hard_umbrella" / "arm" / "run" / "artifacts"
+ cell.mkdir(parents=True)
+ (cell / "robot.urdf").write_text(urdf)
+ (tmp_path / "cells" / "plain" / "arm" / "run" / "artifacts").mkdir(parents=True)
+ (tmp_path / "cells" / "plain" / "arm" / "run" / "artifacts" / "robot.urdf").write_text(
+ urdf.replace('<mimic joint="drive" multiplier="1" offset="0"/>', ""))
+
+ s = survey(tmp_path)
+ assert s["urdfs"] == 2 and s["prompts"] == 2 and s["coupled_prompts"] == 1
+ assert s["coupled"] == [(2, 1)] # 2 movable joints, 1 degree of freedom
+ assert "| 2 | 2 | 1 | 1 | 2 | 1 | 1 |" in report([tmp_path])
+ assert "| art_hard_umbrella | 1 | 2 | 1 | 1 |" in per_prompt(tmp_path)