Skip to content

Official Codebase of 3DCodeVerse

Python

review round 4: the sweep's thresholds, the two probes' depths, and two scripts that lied

ziyaoe6fbeb20db89
diff --git a/harness/bench/data/coupled/README.md b/harness/bench/data/coupled/README.md
index 9612d21b..a50e925f 100644
--- a/harness/bench/data/coupled/README.md
+++ b/harness/bench/data/coupled/README.md
@@ -50,16 +50,33 @@ poses lie on a one-dimensional path, and the other two add a real second freedom
pram's wheel spins, the mirror's tilt). Every overlap the gate reports there is therefore
an overlap the mechanism can actually reach. Over both arms' 20 runs the gate raised **49
ERROR findings across 47 DISTINCT link pairs** — not one pair repeating — at a median
-penetration of **5.8 mm** (max 16.7) against a 1 mm tolerance, and some are already
-present at rest: `base_frame|screw_slider` on the scissor lift overlaps in 3 sampled poses,
-worst 8.8 mm, one of them the rest pose.
+penetration of **5.8 mm** (max 16.7), and some are already present at rest:
+`base_frame|screw_slider` on the scissor lift overlaps in 3 sampled poses, worst 8.8 mm at
+`screw_slider_joint=0.15`, and one of the three is the rest pose. All five numbers are
+printed by `python bench/penetration_thresholds.py <both arms>`.
+
+The sweep records an overlap from **2 mm** (`sweep_collisions(tol_m=)`, the default every
+caller takes) and calls a REST overlap an ERROR only above **5 mm**
+(`sweep_findings(rest_max_m=)`). An earlier version of this paragraph said "1 mm
+tolerance"; that was wrong, and so was reading the 8.8 mm as a rest-pose depth — it is the
+worst over three poses.
So the coupling support did not reduce gate errors because those errors were not mostly
artefacts of unreachable poses: **the mechanisms genuinely self-intersect**. The defect
-these prompts expose is in generation, not in the sampler and not in the gate — which is
-consistent with the scores (median 0.241) and with the judge's leading complaint on the
-scissor lift, "the deck does not rise".
+these prompts expose is in generation, not in the sampler and not in the gate.
+
+Two caveats the construction argument does not remove. Reachable means reachable in the
+KINEMATIC MODEL: a plan with wrong joint limits lets the model reach poses the real object
+could not. And "every sampled pose is reachable" holds for the eight one-DOF models; for
+the pram and the mirror the sampler still combines the second freedom at random, which is
+the pre-mimic situation for those joints.
-One caveat the construction argument does not remove: reachable means reachable in the
-KINEMATIC MODEL. A plan with wrong joint limits lets the model reach poses the real
-object could not.
+**What this paragraph may NOT lean on.** The rest-pose overlaps say nothing about whether
+the sampler's unreachable poses were costing score — rest is sampled by both arms — only
+that self-intersection is real. And the judge's complaint on the scissor lift ("the deck
+does not rise") is **not** corroboration: until `joints_export.robot_scene` resolves
+`<mimic>` followers, every per-pose GLB behind the articulation sheet posed the followers
+at rest, so on a lift whose deck follows the screw slider the judge saw a deck that could
+not rise whatever the code did. That is evidence about the sheet. The sweep poses through
+fk, which does resolve followers, so the sweep-side evidence above stands on its own; the
+judge quote has been removed rather than repaired.
diff --git a/harness/bench/data/plan_stage/README.md b/harness/bench/data/plan_stage/README.md
index c3b9ef8e..32d0e628 100644
--- a/harness/bench/data/plan_stage/README.md
+++ b/harness/bench/data/plan_stage/README.md
@@ -105,5 +105,11 @@ Two readings:
declared the coupling correctly in all ten — an earlier reading of this table called the
pram "3 of 8 declared" and was wrong.
+ **This is a hand reading, and it is not reproducible from this directory.** The rows
+ carry `n_parts` / `n_joints` / `n_mimic` only, so the joint-by-joint check was done on
+ plan JSON that is not in the tree. `plan_stage_bench.py --keep-plans` now writes each
+ call's `plan.json` beside its row so the next such reading can be re-done; the ten plans
+ behind THIS sentence pre-date that flag.
+
The plan stage is therefore NOT what a full run of this battery would be measuring: it
loses nothing here.
diff --git a/harness/bench/penetration_thresholds.py b/harness/bench/penetration_thresholds.py
index 616bc25c..ec2e32ee 100644
--- a/harness/bench/penetration_thresholds.py
+++ b/harness/bench/penetration_thresholds.py
@@ -17,8 +17,8 @@ link pair. A threshold change is a behaviour change; this is the blast radius f
from __future__ import annotations
import argparse
+import inspect
import json
-import re
import statistics
import sys
from collections import Counter
@@ -26,24 +26,41 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from codeverse.languages.urdf import REST_PENETRATION_MAX_M # noqa: E402
from codeverse.spatial.connectivity import ( # noqa: E402
PENETRATION_ERROR_M,
PENETRATION_WARN_M,
)
+from codeverse.spatial.joints_sweep import sweep_collisions, sweep_findings # noqa: E402
#: candidate ERROR thresholds in metres, including the one that ships
CANDIDATES = (0.001, 0.002, 0.005, PENETRATION_ERROR_M)
-#: `joint_sweep` reports a rest-pose overlap at this depth (joints_collide's tolerance)
-SWEEP_TOLERANCE_M = 0.001
-_PAIR = re.compile(r"links '([^|]+)\|([^']+)'")
-_WORST = re.compile(r"worst ([\d.]+) mm")
+#: The sweep's own two numbers, IMPORTED rather than restated — this file used to carry a
+#: third copy of a constant the two probes already disagree on, and it had it wrong.
+#: `sweep_collisions(tol_m=)` is the depth at which a moved-pose overlap is recorded at
+#: all; `sweep_findings(rest_max_m=)` is the depth above which a REST overlap is an ERROR
+#: rather than a WARN. Every caller takes both defaults.
+SWEEP_TOLERANCE_M = float(inspect.signature(sweep_collisions).parameters["tol_m"].default)
+SWEEP_REST_ERROR_M = float(inspect.signature(sweep_findings).parameters["rest_max_m"].default)
+assert SWEEP_REST_ERROR_M == REST_PENETRATION_MAX_M, "the urdf runtime disagrees with the sweep default"
+
+
+#: sub-workspaces a run owns; their records are not runs of their own (`find_runs` skips
+#: both, and `scene_stats.runs` did while this file skipped only `_cand`)
+SUB_WORKSPACES = ("_cand", "_assets")
def records(root: Path) -> list[dict]:
- """Every articulated run record under ``root``, once per run."""
+ """Every articulated run record under ``root``, once per run.
+
+ `resolve()` before the dedupe so a symlinked cell (`run/telemetry/trajectories` is the
+ documented one) is counted once, not twice — the trap that inflated COST §30. The
+ review branch adds `flywheel.record.unique_files` for exactly this; both bench walkers
+ should move to it when that lands.
+ """
out, seen = [], set()
for rec in root.rglob("record.json"):
- if rec.resolve() in seen or "_cand" in rec.parts:
+ if rec.resolve() in seen or any(w in rec.parts for w in SUB_WORKSPACES):
continue
seen.add(rec.resolve())
try:
@@ -55,11 +72,42 @@ def records(root: Path) -> list[dict]:
return out
+def sweep_pairs(sweep: dict | None, *, rest_only: bool) -> dict[tuple[str, ...], float]:
+ """``{link pair: worst depth}`` from a ``joint_sweep`` gate's ERROR findings.
+
+ Read off the structured finding, never the message: ``target`` IS the pair key and
+ ``data["max_depth_m"]`` the worst depth (``depth_m`` when the pair was not aggregated).
+ A wording change in ``aggregate_findings`` used to zero this whole report.
+
+ ``rest_only`` keeps the findings whose WORST pose is the rest pose — ``data["pose"]``
+ falsy, which is what ``aggregate_findings`` writes for rest. Without it this compares
+ the sweep's worst-over-all-poses depth against connectivity's rest-pose depth, and
+ "2-3x deeper on the same pairs" is then not a statement about one pose.
+ """
+ out: dict[tuple[str, ...], float] = {}
+ for f in (sweep or {}).get("findings") or []:
+ if f.get("severity") != "error":
+ continue
+ d = f.get("data") or {}
+ if d.get("kind") != "penetration":
+ continue
+ if rest_only and d.get("pose"):
+ continue
+ target = str(f.get("target") or "")
+ if "|" not in target:
+ continue
+ key = tuple(sorted(target.split("|", 1)))
+ depth = float(d.get("max_depth_m") or d.get("depth_m") or 0.0)
+ out[key] = max(out.get(key, 0.0), depth)
+ return out
+
+
def survey(recs: list[dict]) -> dict:
"""Per-round worst connectivity depth, and how the two probes compare where both ran."""
worst_per_round: list[float] = []
fired: Counter[str] = Counter()
both: list[tuple[str, str, float, float | None]] = []
+ all_sweep_pairs: list[tuple[str, float]] = []
for data in recs:
for rnd in data.get("rounds") or []:
gates = {g["gate"]: g for g in (rnd.get("gates") or [])}
@@ -78,16 +126,18 @@ def survey(recs: list[dict]) -> dict:
fired[f.get("severity", "")] += 1
worst_per_round.append(worst)
sweep = gates.get("joint_sweep")
- for f in (sweep or {}).get("findings") or []:
- if f.get("severity") != "error":
- continue
- pair, mm = _PAIR.search(f.get("message", "")), _WORST.search(f.get("message", ""))
- if not (pair and mm):
- continue
- key = tuple(sorted([pair.group(1), pair.group(2)]))
- both.append((data.get("spec", {}).get("id", "?"), "|".join(key),
- float(mm.group(1)) / 1000, depths.get(key)))
- return {"rounds": len(worst_per_round), "worst": worst_per_round, "fired": fired, "both": both}
+ # every ERROR pair, for the corpus counts the docs cite
+ for key, depth in sweep_pairs(sweep, rest_only=False).items():
+ all_sweep_pairs.append(("|".join(key), depth))
+ # and only the pairs whose worst pose IS rest, for the depth comparison
+ for key, depth in sweep_pairs(sweep, rest_only=True).items():
+ both.append((data.get("spec", {}).get("id", "?"), "|".join(key), depth, depths.get(key)))
+ n_rest = sum(1 for _, _, _, c in both if c is not None)
+ return {"rounds": len(worst_per_round), "worst": worst_per_round, "fired": fired, "both": both,
+ "sweep_findings": len(all_sweep_pairs),
+ "sweep_distinct": len({p for p, _ in all_sweep_pairs}),
+ "sweep_depths": [d for _, d in all_sweep_pairs],
+ "rest_comparable": n_rest}
def corroborate(recs: list[dict], threshold_m: float) -> dict:
@@ -111,13 +161,7 @@ def corroborate(recs: list[dict], threshold_m: float) -> dict:
d = f.get("data") or {}
if d.get("kind") == "penetration" and float(d.get("depth_m") or 0.0) >= threshold_m:
deep.add(tuple(sorted([f.get("target", ""), str(d.get("other") or "")])))
- flagged = set()
- for f in sweep.get("findings") or []:
- if f.get("severity") != "error":
- continue
- m = _PAIR.search(f.get("message", ""))
- if m:
- flagged.add(tuple(sorted([m.group(1), m.group(2)])))
+ flagged = set(sweep_pairs(sweep, rest_only=False))
both += len(deep & flagged)
conn_only += len(deep - flagged)
sweep_only += len(flagged - deep)
@@ -127,7 +171,8 @@ def corroborate(recs: list[dict], threshold_m: float) -> dict:
def report(root: Path) -> str:
- s = survey(records(root))
+ recs = records(root) # read once: this used to walk every record.json twice
+ s = survey(recs)
rounds, worst = s["rounds"], s["worst"]
nonzero = [w for w in worst if w > 0]
lines = [f"# penetration thresholds over {rounds} articulated round(s) under {root}", "",
@@ -137,11 +182,21 @@ def report(root: Path) -> str:
if nonzero:
lines.append(f"depth recorded by connectivity: median {statistics.median(nonzero) * 1000:.1f} mm, "
f"max {max(nonzero) * 1000:.1f} mm")
+ sd = s["sweep_depths"]
+ lines += ["",
+ f"joint_sweep ERROR findings: {s['sweep_findings']} across {s['sweep_distinct']} distinct link "
+ f"pair(s) (records an overlap at {SWEEP_TOLERANCE_M * 1000:.0f} mm; a REST overlap is an ERROR "
+ f"only above {SWEEP_REST_ERROR_M * 1000:.0f} mm)"]
+ if sd:
+ lines.append(f"depth recorded by the sweep: median {statistics.median(sd) * 1000:.1f} mm, "
+ f"max {max(sd) * 1000:.1f} mm")
+ lines.append(f"pairs whose WORST sweep pose is rest (the only ones comparable with connectivity "
+ f"pose-for-pose): {len(s['both'])}, of which {s['rest_comparable']} were also recorded "
+ f"by connectivity in the same round")
lines += ["", "| ERROR threshold | rounds at or over it | share |", "|---|--:|--:|"]
for th in sorted(CANDIDATES):
n = sum(1 for w in worst if w >= th)
lines.append(f"| {th * 1000:.0f} mm | {n} | {n / max(1, rounds):.0%} |")
- recs = records(root)
for th in (0.002, 0.005):
c = corroborate(recs, th)
total = c["both"] + c["conn_only"]
@@ -152,8 +207,8 @@ def report(root: Path) -> str:
if c["examples"]:
lines.append(f" uncorroborated examples: {', '.join(c['examples'][:4])}")
if s["both"]:
- lines += ["", f"the two probes on the same link pair ({len(s['both'])} sweep ERROR findings; "
- f"sweep tolerance {SWEEP_TOLERANCE_M * 1000:.0f} mm):",
+ lines += ["", "the two probes on the same link pair AT REST — the sweep rows below are all "
+ "findings whose worst pose is the rest pose, so both columns describe one pose:",
"", "| run | pair | sweep | connectivity |", "|---|---|--:|--:|"]
for run, pair, sw, cn in s["both"][:20]:
lines.append(f"| {run} | `{pair}` | {sw * 1000:.1f} mm | "
diff --git a/harness/bench/plan_stage_bench.py b/harness/bench/plan_stage_bench.py
index d564f27c..30fdfce8 100644
--- a/harness/bench/plan_stage_bench.py
+++ b/harness/bench/plan_stage_bench.py
@@ -119,14 +119,18 @@ def tree_provenance(tree: Path, codeverse_file: str) -> dict[str, str]:
def run_one(battery, item, backends, label: str, rep: int, provenance: dict[str, str] | None = None,
- keep_failed: Path | None = None) -> dict:
+ keep_failed: Path | None = None, keep_plans: Path | None = None) -> dict:
"""One plan-stage call in a throwaway workspace; never raises.
``plan_once`` deliberately bypasses ``Track.run``: there is no build, no judge and no
run ledger, so the JSONL row IS the record (it carries the call's own ``cost_usd``
from the plan events, and the totals are printed at the end). ``keep_failed`` moves a
workspace that produced no plan out of the temp dir instead of deleting it, which is
- the only way to read what the model actually wrote.
+ the only way to read what the model actually wrote. ``keep_plans`` copies the plan
+ JSON of a call that SUCCEEDED beside its row: the rows carry `n_parts` / `n_joints` /
+ `n_mimic` only, so any joint-level reading of a committed arm ("eight of ten leave
+ exactly one free joint and it IS the input") is otherwise a hand reading nobody can
+ redo — review round 3 asked for this.
"""
from bench.pin_plan import plan_once
from bench.run_bench import build_spec
@@ -143,6 +147,11 @@ def run_one(battery, item, backends, label: str, rep: int, provenance: dict[str,
"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(started)),
"seconds": round(time.time() - started, 1), **(provenance or {}),
**_plan_shape(root / "ws"), **_stats(root / "ws")}
+ if ok and keep_plans is not None:
+ src = root / "ws" / "plan.json"
+ if src.is_file():
+ keep_plans.mkdir(parents=True, exist_ok=True)
+ shutil.copy(src, keep_plans / f"{item.id}_r{rep}.json")
if not ok and keep_failed is not None:
# name what it was, by the same rule the report classifies the row: a provider block
# says nothing about the code under test, everything else (a validation failure, a
@@ -171,6 +180,9 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument("--keep-failed", type=Path, default=None,
help="move the workspace of any call that produced no plan here (plan stage "
"only, so they are small) instead of deleting it")
+ ap.add_argument("--keep-plans", type=Path, default=None,
+ help="copy each SUCCESSFUL call's plan.json here, so a joint-level reading of "
+ "the arm can be redone from the tree (the rows carry shape counts only)")
ns = ap.parse_args(argv)
for kv in (x for x in ns.env.split(",") if x.strip()):
@@ -203,7 +215,7 @@ def main(argv: list[str] | None = None) -> int:
# imap-style ordering: pool.map yields in submission order, so one slow call holds
# back every finished row behind it. Rows are the journal, so they go out as they
# land instead (resume reads what is on disk, order does not matter).
- futures = [pool.submit(run_one, battery, item, backends, ns.label, rep, prov, ns.keep_failed)
+ futures = [pool.submit(run_one, battery, item, backends, ns.label, rep, prov, ns.keep_failed, ns.keep_plans)
for item, rep in jobs]
for fut in cf.as_completed(futures):
row = fut.result()
diff --git a/harness/codeverse/spatial/connectivity.py b/harness/codeverse/spatial/connectivity.py
index 88f63eb6..5e87a7ef 100644
--- a/harness/codeverse/spatial/connectivity.py
+++ b/harness/codeverse/spatial/connectivity.py
@@ -64,10 +64,10 @@ PlannedEdge = tuple[str, "str | Sequence[str]"]
MIN_PART_SIZE_M = 0.01
#: penetration depth that is reported as WARN / ERROR. These bound the REST pose only,
#: and with a 600-point surface draw per part: this check asks "do these two surfaces sit
-#: inside each other as built". ``joint_sweep`` asks a different question with a denser
-#: probe — "does moving a joint drive one link through another" — at a 1 mm tolerance, and
-#: the two agree on only 7 of 50 link pairs at 2 mm (bench/penetration_thresholds.py,
-#: D-entry 2026-09-04). They are not one check with two dials; do not align the numbers.
+#: inside each other as built". ``joint_sweep`` asks a different question on posed meshes
+#: — "does moving a joint drive one link through another" — at its own thresholds. They
+#: are not one check with two dials: do not align the numbers (DECISIONS.md D53, with the
+#: corpus measurement behind it in bench/penetration_thresholds.py).
PENETRATION_WARN_M = 0.002
PENETRATION_ERROR_M = 0.010
#: fraction of one part's samples inside the other before we call it penetration ...
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index 85cffdc5..3775aaa9 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -763,35 +763,158 @@ written) that were accepted because the code works that way and the tests pin it
from the URDFs and stand; the scores and gate rates need a re-run on the corrected
export.
-* **The harness measures interpenetration twice, and the laxer probe is the one that
- gates (found 2026-09-04, NOT changed).** `spatial/connectivity` samples 600 points per
- surface, requires a minimum share of them inside the other part, and calls 2 mm a WARN
- and **10 mm** an ERROR; `spatial/joints_collide` (the `joint_sweep` gate) probes the
- posed meshes densely and reports at **1 mm**. Same quantity, different sensitivity —
- and the sparse probe is behind the gate that fails a round. Measured over 374 recorded
- articulated rounds (`bench/penetration_thresholds.py`): the connectivity penetration
- ERROR has fired **0 times**, its WARN 50; the depths it records are median 2.0 mm, max
- 9.9 mm, so its ERROR threshold is unreachable in practice. On the same link pairs the
- sweep measures 2–3x deeper (8.8 vs 3.0 mm, 5.9 vs 3.8, 8.0 vs 3.0) and sometimes finds
- an overlap connectivity does not report at all (15.0 mm on `base_frame|slider_crosshead`).
- The visible consequence: on `coupled_v1`, 11 sweep ERRORs about the REST pose were
- raised in rounds where the connectivity gate PASSED — the harness told the agent two
- different things about one pose. Blast radius of a change, already computed: an ERROR
- at 5 mm moves 1 round of 374, at 2 mm it moves 36 (10 %).
+* **D53 Interpenetration is measured twice, and the two probes are asking different
+ questions (found 2026-09-04, corrected 2026-09-06, NOT changed).** `spatial/connectivity`
+ draws 600 points per surface, requires a minimum share of them inside the other part, and
+ calls **2 mm** a WARN and **10 mm** an ERROR at the REST pose. `joint_sweep` poses the
+ mechanism through fk and probes densely: it records an overlap from **2 mm**
+ (`sweep_collisions(tol_m=)`, the default every caller takes) and treats a REST overlap as
+ an ERROR only above **5 mm** (`sweep_findings(rest_max_m=)`, which
+ `urdf.REST_PENETRATION_MAX_M` matches). So at rest the two share a WARN line and differ
+ 2x on ERROR — not 10x, and `bench/penetration_thresholds.py` now imports both numbers
+ instead of restating them.
- **And then the threshold experiment answered itself, offline (2026-09-04): do not
- change it.** Using the dense probe as the reference — for every pair connectivity
- records at or over a candidate threshold, did the SAME round's sweep flag that pair? —
- an ERROR at 2 mm would fire on 50 pairs of which only **7 are corroborated**, while the
- 43 uncorroborated ones are static contacts the design intends (`shoulder_lock_knob|
- swivel_post`, `base_underframe|center_top`) that a 600-point surface draw reads as
- penetration; and the dense probe would still be alone on **120** pairs. At 5 mm: 2
- pairs, 0 corroborated. The two probes are not measuring the same thing at different
- sensitivities — one asks "do these surfaces sit inside each other at rest", the other
- "does moving this joint drive one link through another". A shared threshold would
- create uncorroborated failures and still miss what the sweep finds. What is wrong is
- that the two read as one check with two dials; the fix belongs in what each is called
- and documented to do, not in the numbers.
+ Measured over 374 recorded articulated rounds: the connectivity ERROR has fired **0
+ times**, its WARN 50, and the depths it records are median 2.0 mm, max 9.9 mm — its ERROR
+ threshold is unreachable in practice. The sweep raises **295 ERROR findings across 191
+ distinct link pairs**, median 5.0 mm, max 34.3.
+
+ **A correction to how that gap was first read.** This entry used to say the sweep
+ measures "2-3x deeper on the same link pairs (8.8 vs 3.0 mm, 5.9 vs 3.8, 8.0 vs 3.0)".
+ That comparison was not about one pose: the sweep number was its worst over ALL sampled
+ poses and the connectivity number was the rest pose. Restricted to the pairs whose worst
+ sweep pose IS the rest pose — `data["pose"]` empty, which is the filter that makes the
+ comparison mean what it says — there are 19 such pairs in the corpus, 4 of them also
+ recorded by connectivity in the same round, and on those four **the two probes agree
+ exactly**: 4.4/4.4, 4.4/4.4, 2.2/2.2, 2.5/2.5 mm. The probes do not disagree about depth.
+ What differs is the question: 288 of the sweep's pairs are overlaps that exist only in a
+ moved pose, which a rest-pose check cannot see by construction.
+
+ **And the threshold experiment answered itself, offline: do not change it.** Using the
+ dense probe as the reference — for every pair connectivity records at or over a candidate
+ threshold, did the SAME round's sweep flag that pair? — an ERROR at 2 mm would fire on 50
+ pairs of which only **7 are corroborated**, while the 43 uncorroborated ones are static
+ contacts the design intends (`shoulder_lock_knob|swivel_post`, `base_underframe|center_top`)
+ that a 600-point surface draw reads as penetration; the dense probe would still be alone on
+ **288** pairs. At 5 mm: 2 pairs, 0 corroborated. A shared threshold would create
+ uncorroborated failures and still miss what the sweep finds. What was wrong is that the
+ two read as one check with two dials; the fix belongs in what each is called and documented
+ to do, not in the numbers.
+
+* **D54 A gate never reports the machine as a defect, and a render is retried once when
+ the browser dies (2026-09-05).** The first recorded `scenes_v1` battery produced six
+ cells of which three built, passed every gate that does not need pixels, and kept ZERO
+ renders: `driver: Attempted to use detached Frame '<id>'` — Chrome reaping the render
+ tab on a box at load 93 with swap full and all eight GPUs at ~100 %. The judge was
+ skipped for want of images, so $6.75 of already-paid generation produced no verdict, and
+ `bench/scene_stats.py` attributed the whole thing to `render_console`. Three separate
+ rules came out of it, and they are the general form, not three patches:
+ 1. **A driver that lost its browser is retried once, on a browser of its own.** The
+ scene funnel (`spatial/render_scene.run_scene_script`) had no retry at all; the object
+ path had one since 2026-08-28 but its marker tuple knew only the "Target closed"
+ spelling. One vocabulary, `spatial/node.BROWSER_LOST_MARKERS`, now serves both, and
+ the retry runs with `CV3D_BROWSER_REUSE=off` because the shared browser advertised in
+ the cache is the suspect. Exactly one retry: a box out of memory stays out of memory.
+ 2. **Two runtime trees never share a browser.** The daemon endpoint, its spawn lock and
+ its failure file carry a digest of the `runtime_js` that spawned them, so a worktree
+ and the main checkout cannot advertise over each other inside one `CV3D_CACHE_DIR`.
+ RUNBOOK had asked operators to remember this since the coupled battery lost an arm to
+ it; nothing enforced it.
+ 3. **A finding that names a dead browser is reported apart from a defect.**
+ `scene_stats.py` puts it in a `lost to the box` column and does not count the gate as
+ failed. A battery whose report shows that column non-zero is not yet a statement
+ about the generator.
+ Consequence: a red browser test is checked against `uptime` and `free -g` before it is
+ believed, and `test_studio_render_is_reproducible_and_stamps_the_rig_version` is
+ documented as flaky-by-construction under CPU contention (SwiftShader is not
+ bit-reproducible when the box is busy) rather than weakened.
+
+* **D55 `scene_placement` measures matter, and matter writes depth (2026-09-05).** Two of
+ the first six recorded scene cells failed the gate on nothing but their own atmosphere:
+ "BlackPine_5 is sunken 3.46 m into AtmosphereHaze" and "WindowSnowView/Mesh_49 and
+ Environment/MoonlightShaft overlap (100 % of the smaller box)". Both scenes were
+ correct; both offenders are `MeshBasicMaterial` at opacity 0.035-0.04 with
+ `depthWrite: false`, and all nine recorded scenes use that idiom 34-42 times each. A
+ pass that writes no depth occludes nothing, so it cannot support an object, nothing can
+ sink into it, and passing through it is what it is for. `host_placement.nonSolid` keeps
+ such meshes out of the column index entirely; an asset made only of them is listed with
+ the exempt reason `volumetric` rather than dropped. **Opacity is deliberately not part
+ of the rule** — glass sits at 0.3-0.6 and keeps writing depth, and a greenhouse pane
+ really is a surface; a solid wall the model mistakenly marked `depthWrite: false` stops
+ being a support, which is the cheaper error and matches what the frame shows. Measured
+ on the recorded workspaces: cozy_cabin FAIL → PASS, japanese_garden's 3.46 m becomes a
+ 0.044 m embed and the gate now fails on a real defect it had been reporting alongside
+ the fog (`SubmergedRock_1 is sunken 0.48 m into ArchedBridge`).
+
+ The same rule belongs in `nearGeometry`, and there it does more than quiet a report.
+ `repairCameraSpec` is driven by `camera_in_geometry`, so a false "inside" RETREATS the
+ authored lens: of the battery's seven recorded camera repairs, three were triggered by
+ fog, and cozy_cabin's two were each moved back 4 m and up 2 m for standing in a
+ `MoonlightShaft` — one of them ending NEARER geometry than it started (2.358 → 0.916 m).
+ Re-rendering that workspace under the battery's own `--camera-repair` flag with the rule
+ in place: `ArmchairHearthEye` goes from mean luminance 0.126 / 48 % near black to
+ **0.263 / 0.5 %**, `WindowFrostSnow` from 0.149 / 56 % to **0.298 / 17 %**, and both
+ cross from `dark_frame` ERROR to passing, while the two cameras that were never moved
+ are unchanged (one of them still genuinely dark). So **two of that cell's three
+ dark-frame ERRORs were manufactured by the harness**: fog → a false "camera in
+ geometry" → a 4 m retreat → a dark frame → a second gate's ERROR → a judge complaint.
+ Since "the frame is too dark" is this track's most recorded defect (153 findings over 32
+ runs, per the scene prompt itself), some unknown share of that history is the
+ instrument rather than the model. Across all 113 recorded camera checks in the battery
+ the RAY term (`nearest < 0.3 m`) never fired once; every `camera_in_geometry` finding
+ came from the bbox term, and every one of those was a volumetric or a scatter field.
+
+* **D56 A loaded GLB root is named after its asset (2026-09-05).** rooftop_garden's plan
+ asked for one `blender_glb` asset; the Blender sub-run built it, the assembler wrote the
+ loader, the browser loaded it and the census confirmed it was in the scene
+ (`meshes_in_scene: 1, in_scene: true`) — and `scene_placement` still reported
+ "zone PergolaLounge is missing planned contents: LoungeSofa", because the object was
+ called `Scene`. A glTF root carries whatever the exporter wrote and Blender writes
+ "Scene". The three assembled entry points that load GLBs (`render_scene_js`, the
+ assembler's probe module, and the plan-written `scene.js`) had drifted into three copies
+ of the same loop with three different error messages; they are now one emitter,
+ `_glb_preload_js`, which stamps `to_pascal(key)` on the loaded root. Consequence for the
+ Blender-scene question: **the cross-language seam is not what fails.** The Blender asset
+ path works end to end and only lost a name; an argument for a Blender assembly layer has
+ to be made on other grounds.
+
+* **D57 A frame rate is a measurement of the renderer that produced it (2026-09-05).**
+ Inside one `scenes_v1` battery `fps` was measured on two different backends — 11.5 fps on
+ an RTX 6000 Ada for one cell, 2.0 / 5.1 / 7.1 on SwiftShader for the next three, because
+ `gpu_launch.cjs` caches a negative GPU verdict for 20 minutes and the box's GPUs were at
+ ~100 % from other work. `render_console` raised "low frame rate … merge static geometry"
+ for all four, and the judge was handed "PROBE: measured N fps" as a fact; two of the four
+ **critical** judge issues across the scored cells were frame-rate complaints. Decision:
+ `RenderSet.hardware_fps` is the only form a gate or a judge may read, and
+ `RenderSet.software_rendered` reads the renderer string with the same words
+ `gpu_launch.cjs` uses to decide whether a GPU attempt is trusted at all. The judge prompt
+ keeps a software number but LABELS it, rather than dropping it silently — a genuinely
+ heavy scene should still be visible to a reader — and an unrecognised renderer string is
+ never claimed as software. Consequence: an fps comparison across cells is only valid
+ within one backend, which is a property of any battery run on a shared machine, not of
+ this one.
+
+* **D58 The scene texture pack is wired into the loop, behind a switch that is off
+ (2026-09-05).** Of 24 judge issues over the five scored cells of the first `scenes_v1`
+ battery, four say the GROUND is a flat untextured colour, in near-identical words —
+ "single flat brown color", "single flat color with no cobblestone texture", "flat,
+ untextured blueish plane with no material blending", "a hard, unblended circular seam".
+ It is the most consistent defect in the battery. It is also a harness gap:
+ `texturing.plan.scene_texture_pack` writes a tileable pack and `texture_pack_prompt`
+ renders the list plus the loading idiom — its docstring says "Prompt snippet for
+ zone/env generation" — and NOTHING in `tracks/scene.py` called either. The pack was
+ reachable only through `3dcv texture scene-pack`, whose output `cli/texture_cmd.py`
+ prints for a human to paste, and `Spec.options.texture` does nothing on this track.
+ Decision: a `textures` stage runs before env and zones (they can only name files that
+ exist when their prompts are built) and `_ctx` — the one place both prompts get their
+ context — carries the manifest. **`CV3D_SCENE_TEXTURES` is off by default**: it costs
+ an image-model call per run, measured at $0.15 for two 512 px textures (seam score
+ 0.001) and an estimated $1-2 per run at ten 1024 px ones, and what that buys is a
+ measurement nobody has made. Consequence for the Blender question: the material class
+ cannot be counted as evidence for changing renderer until this arm has run. (The
+ pack planner, given medieval_market's plan and told nothing about its judgment, planned
+ `medieval_cobblestone` first — against a judge complaint reading "no cobblestone
+ texture or material blending".)
## Rejected / deferred
diff --git a/harness/docs/PAPER_WRITING.md b/harness/docs/PAPER_WRITING.md
index ab8374b5..3897a436 100644
--- a/harness/docs/PAPER_WRITING.md
+++ b/harness/docs/PAPER_WRITING.md
@@ -343,11 +343,22 @@ into a question it answers every time.
**Why the gate rate did not move.** In the coupled arm every sampled pose is reachable by
construction — eight of ten models have exactly one free joint — so each overlap the gate
reports is one the mechanism can reach. Across both arms' 20 runs it raised 49 ERROR
-findings over 47 distinct link pairs at a median 5.8 mm penetration (1 mm tolerance), some
-already present at rest. The errors were never mostly artefacts of unreachable poses:
-these mechanisms genuinely self-intersect, and the defect the battery exposes is in
-generation rather than in the sampler or the gate. (Reachable here means reachable in the
-kinematic model; wrong limits still let a model reach what the object could not.)
+findings over 47 distinct link pairs at a median 5.8 mm penetration, some already present
+at rest; all of those are printed by `bench/penetration_thresholds.py`. The errors were
+never mostly artefacts of unreachable poses: these mechanisms genuinely self-intersect, and
+the defect the battery exposes is in generation rather than in the sampler or the gate.
+
+Three caveats, two of them corrections (2026-09-06). Reachable means reachable in the
+KINEMATIC MODEL; wrong limits still let a model reach what the object could not. "Every
+sampled pose is reachable" holds for the eight one-DOF models — for the pram and the mirror
+the sampler still combines the second freedom at random. And the sweep's tolerance is
+**2 mm**, with a rest-pose ERROR line at **5 mm** (`sweep_collisions(tol_m=)` and
+`sweep_findings(rest_max_m=)`, both defaults every caller takes); an earlier version of this
+paragraph said 1 mm. The judge's "the deck does not rise" was cited here as corroboration
+and has been removed: until `joints_export.robot_scene` resolves `<mimic>` followers, the
+articulation sheet posed every follower at rest, so that complaint is evidence about the
+sheet, not about the generation. The sweep poses through fk, which does resolve followers,
+so the sweep-side evidence stands on its own.
**What did NOT move: the sweep gate's own failure rate.** On those eight prompts the
`joint_sweep` gate fails 30 % of the rounds with couplings declared (11 of 37,
@@ -518,6 +529,27 @@ data the repo does not carry; they are marked in the caveat column.
three-arm run leaves that class at 1 / 2 / 4 across the arms with nothing acting on it
(§5.1). It never restarts and it survives all three re-asks — the same shape the
collapsed-plan class had before it was handled, and the obvious next measurement.
+* **The scene track's recorded numbers are not usable, and the first battery says why.**
+ The paper's "does not yet work" rests on a loop value of −0.009 at n = 4 whose records
+ are gone from both clones — no script in the tree recomputes it. The first battery that
+ CAN be recomputed (`bench/out/scene_baseline`, 12 prompts, 2026-09-05) returned no
+ verdict for four of its first ten cells, and **every one of those four is the machine**:
+ three lost the render tab to Chrome under memory pressure, and the fourth, recorded as
+ `[?] scene did not boot` through three repair attempts, has its own round artifact
+ reading `ok: true, boot.ok: true, stage: ready` and boots today unchanged in 600 ms.
+ Worse for a quality claim, the instrument was manufacturing defects: 15 of 19 recorded
+ `scene_placement` ERRORs were volumetric passes counted as solid matter, the judge
+ quotes gate findings back as defects (3 of 24 issues, one at *critical*), and a false
+ "camera in geometry" retreats the authored lens — cozy_cabin's two cameras were moved
+ back 4 m for standing in a moonbeam, which is what took their frames from mean luminance
+ 0.263 / 0.298 down to 0.126 / 0.149 and into `dark_frame` ERROR. Since "the frame is
+ too dark" is this track's most-cited defect (153 findings over 32 runs), some share of
+ that history is the instrument; **the share is unknowable because those records are
+ gone**, so no number should be put on it. The fixes are on `ziyao/scene-placement`
+ (D53–D58); arm B (`local/scripts/run_scene_fixed.sh`) is the measurement, and until it
+ lands the paper should not cite a scene result. Handover note:
+ `local/docs/PAPER-scene-track-correction.md`.
+
* **A loss rate is a property of its window.** The same control arm measured 4.7 % on
2026-09-02/03 and 2.9 % on 2026-09-03; single-arm comparisons across windows are not
safe, which is why the re-measurement ran all three arms at once.
@@ -536,6 +568,7 @@ a test enforces by grepping the tree; a switch nothing reads once produced "keep
| `CV3D_LEAN_PROMPT` | drop duplicated contract/tool cards, select cookbook chapters, focus the refine prompt | **off** | +0.030 ±0.076, no cost saving (§4); **removed from the tree 2026-09-03**; last carried on `ziyao/articulated-wave-2` before commit `4cbb28c`. |
| `CV3D_FEWER_TURNS` (pre-existing) | fold gate checks into build, inline refine files | **off** | +0.015 ±0.126, dollars flat (§4) |
| `CV3D_AXIS_REPAIR` (upstream) | deterministic axis rewrite from the measured motion | on | upstream's, kept |
+| `CV3D_SCENE_TEXTURES` | wire the scene texture pack into the loop: a stage before env/zones, and `texture_pack_prompt` in both prompts | **off** | added 2026-09-05 (D58) and NOT yet measured. Four of five scored cells of the first scene battery complain the ground is a flat untextured colour, in near-identical words; the pack generator existed and had never been called from the loop. Costs an image-model call per run (measured $0.15 for two 512 px textures, seam 0.001) |
Not switched, because they are bug fixes rather than levers: the MCP verdict/failed split
(§5.2), `<mimic>` support (§5.3), the provider-failure markers and retries (§5.4), the
diff --git a/harness/docs/RUNBOOK.md b/harness/docs/RUNBOOK.md
index bd6566f5..07941c55 100644
--- a/harness/docs/RUNBOOK.md
+++ b/harness/docs/RUNBOOK.md
@@ -169,16 +169,47 @@ unless you type `--host` yourself; it never serves a path outside the declared r
python -m codeverse.judges.calibration runs/<slug> [runs/<slug2> …] --model gemini:gemini-3.1-pro-preview --n 3 --out out/
# re-judges recorded rounds; writes calibration_<model>.md/.json (never touches runs/)
```
-**Two arms at once need separate `CV3D_CACHE_DIR`, not just separate `--out`.** The
-browser daemon advertises its endpoint in `CACHE_DIR/browser_<backend>.json` and a newer
-daemon supersedes an older one (`runtime_js/browser_daemon.cjs`), so two worktrees sharing
-a cache fight over ONE browser, and `render_glb` returns "produced no result" on one side
-only, looking like random flakiness (2026-09-04). The mechanism is not confirmed: each
-render serves its GLB from its own process (`render_glb.mjs` `serveDirs`), so it is not a
-server-root mismatch; the likely cause is the superseded daemon exiting while the other
-arm's render is still in flight on its browser, which is a race in shared code that a
-separate cache dir avoids rather than fixes. Give each arm its own cache dir; the render
-cache separates with it, which an A/B wants anyway.
+**Do not run the offline suite while a battery is running.** The suite is 2 300+ tests on
+24 cores under xdist; a battery holds Blender, a browser pool and several agent sessions.
+Twice on 2026-09-04/05 that produced 11 and 21 spurious failures that vanished on a clean
+re-run (`tests/scene_runtime/lib/*` first, since those hold the renderer longest). A red
+suite during a battery is **not evidence until it reproduces on a quiet machine** — re-run
+it there before pushing, and if the same tests are red again, it is real. The mechanism is
+the one the next paragraph describes: `tests/scene_runtime/lib/*` reach the battery's
+browser daemon through a shared `CV3D_CACHE_DIR`, so giving the suite its own cache dir may
+let the two coexist.
+
+The same holds for a box loaded by OTHER work. On 2026-09-05, with 12 `proseg` processes
+holding 528 GB, swap full and all eight GPUs at ~100 %, the offline suite failed 12, then
+44, then 17 tests in three runs at `-n 12 / 6 / 4`, every one of them a browser test
+reporting `Session closed` or `Target closed`; `tests/scene_runtime` alone at `-n 4` with
+its own cache dir passed 787/787. Check `uptime` and `free -g` before believing a red
+browser test. One test is flaky by construction under CPU contention and is NOT to be
+"fixed" by weakening it: `test_studio_render_is_reproducible_and_stamps_the_rig_version`
+compares two renders byte for byte, and SwiftShader is not bit-reproducible when the box
+is busy — it failed 2 runs in 3 on an unmodified tree at load 129. A second one behaves
+the same way: `test_example_scene_passes_frame_gate_and_judge_subset` asserts that a
+frame's content / ground / sky coverage fractions sum to 1 +- 0.02, and at `-n 4` under
+load ~100 it fails on `content 0.644 + ground 0.000 + sky 0.549 = 1.193`. An interleaved
+A/B — six runs each, alternating so both arms see the same load — gave **5/6 failures on
+the unmodified tree and 4/6 with the branch's changes**, i.e. the box, not the code. What
+produces a ground fraction of exactly zero on a scene that has a ground is an open
+question; it needs an idle machine to look at, not a guess. A third is a plain wall-clock
+assertion: `test_cabinet_door_end_to_end` requires the URDF build to finish in under 30 s
+and measured 38 970 ms at load ~100 with `ok=True` — the build succeeded, the box was
+busy. All three are the same story, and none of them is to be "fixed" by loosening what
+it asserts.
+
+**Two arms at once still want separate `CV3D_CACHE_DIR`, though the browser no longer
+depends on it.** Until 2026-09-05 the daemon advertised its endpoint at
+`CACHE_DIR/browser_<backend>.json`, a newer daemon superseded an older one, and two
+worktrees sharing a cache ended up on ONE browser: the arm that did not launch it
+rendered through a server rooted in the other tree, its GLB was outside that root, and
+`render_glb` returned "produced no result" — on one side only, looking like random
+flakiness (2026-09-04). The endpoint, its lock and its failure file now carry a digest of
+the `runtime_js` that spawned the daemon, so two trees keep separate browsers inside one
+cache dir and neither can retire the other's. Give each arm its own cache dir anyway: the
+render cache separates with it, which an A/B wants.
Also halve each arm's `--parallel`: two arms at 3 workers is six concurrent renders, and a
round with no renders skips the judge, so the loop stops at `judge_unavailable` and the
diff --git a/harness/tests/compare_bench/test_session_stats.py b/harness/tests/compare_bench/test_session_stats.py
index 04dc86dd..88dd32aa 100644
--- a/harness/tests/compare_bench/test_session_stats.py
+++ b/harness/tests/compare_bench/test_session_stats.py
@@ -166,6 +166,17 @@ def test_the_coupling_survey_counts_runs_not_copies_of_the_same_urdf(tmp_path: P
assert _prompt_of(found[0]) == "cpl_umbrella"
+def _sweep(pair: str, depth_m: float, *, poses: list[dict], worst_pose: dict) -> dict:
+ """An aggregated `joint_sweep` penetration finding, in the shape `aggregate_findings`
+ really emits: the pair is `target`, the worst depth is `data["max_depth_m"]`, and
+ `data["pose"]` is the pose that worst depth was measured in ({} = rest). The message
+ is deliberately junk here — reading it was the bug."""
+ return {"gate": "articulation", "severity": "error", "target": pair,
+ "message": "(wording changes; nothing may parse this)",
+ "data": {"kind": "penetration", "depth_m": depth_m, "max_depth_m": depth_m,
+ "pose": worst_pose, "poses": poses, "n_poses": len(poses)}}
+
+
def test_the_two_penetration_probes_are_compared_not_merged(tmp_path: Path) -> None:
"""The corroboration view is what stopped a threshold change: for every pair
connectivity records at or over a candidate depth, did the same round's sweep flag it?
@@ -181,8 +192,8 @@ def test_the_two_penetration_probes_are_compared_not_merged(tmp_path: Path) -> N
"depth_m": 0.003}},
]},
{"gate": "joint_sweep", "passed": False, "findings": [
- {"severity": "error", "message": "links 'arm|deck' overlap in 2 of the sampled poses (worst 6.0 mm)"},
- {"severity": "error", "message": "links 'rod|frame' overlap in 1 of the sampled poses (worst 9.0 mm)"},
+ _sweep("arm|deck", 0.006, poses=[{}, {"j": 0.15}], worst_pose={"j": 0.15}),
+ _sweep("rod|frame", 0.009, poses=[{"j": 0.3}], worst_pose={"j": 0.3}),
]},
]}]}
@@ -192,3 +203,41 @@ def test_the_two_penetration_probes_are_compared_not_merged(tmp_path: Path) -> N
assert c["sweep_only"] == 1 # rod|frame: the sparse draw never saw it
assert c["examples"] == ["b/p: knob|post"]
assert corroborate([rec], 0.005)["both"] == 0 # nothing that deep on the sparse side
+
+
+def test_the_depth_comparison_is_restricted_to_one_pose() -> None:
+ """`survey`'s two-probe table used to compare the sweep's worst-over-ALL-poses depth
+ with connectivity's rest-pose depth, so "2-3x deeper on the same pairs" was not a
+ statement about one pose at all. Over the 374-round corpus, restricted to the pairs
+ whose worst sweep pose IS rest, the two probes agree exactly (4.4/4.4, 4.4/4.4,
+ 2.2/2.2, 2.5/2.5 mm)."""
+ from bench.penetration_thresholds import survey, sweep_pairs
+
+ moved = _sweep("a|b", 0.0088, poses=[{}, {"j": 0.15}], worst_pose={"j": 0.15})
+ at_rest = _sweep("c|d", 0.0044, poses=[{}], worst_pose={})
+ gate = {"gate": "joint_sweep", "findings": [moved, at_rest]}
+
+ assert set(sweep_pairs(gate, rest_only=False)) == {("a", "b"), ("c", "d")}
+ assert set(sweep_pairs(gate, rest_only=True)) == {("c", "d")}, "the 8.8 mm is a moved pose"
+
+ rec = {"spec": {"id": "b/p", "track": "articulated_object"}, "rounds": [{"gates": [
+ {"gate": "connectivity", "passed": True, "findings": [
+ {"severity": "warn", "target": "c", "data": {"kind": "penetration", "other": "d", "depth_m": 0.0044}},
+ {"severity": "warn", "target": "a", "data": {"kind": "penetration", "other": "b", "depth_m": 0.0030}},
+ ]},
+ gate,
+ ]}]}
+ s = survey([rec])
+ assert s["sweep_findings"] == 2 and s["sweep_distinct"] == 2
+ assert [(p, sw, cn) for _, p, sw, cn in s["both"]] == [("c|d", 0.0044, 0.0044)], \
+ "only the rest-worst pair is comparable, and there the two probes agree"
+
+
+def test_a_finding_whose_message_changes_still_counts() -> None:
+ """The whole point of reading `target` / `data`: a wording change in
+ `aggregate_findings` used to silently zero this report."""
+ from bench.penetration_thresholds import sweep_pairs
+
+ f = _sweep("x|y", 0.01, poses=[{}], worst_pose={})
+ f["message"] = "completely different wording"
+ assert sweep_pairs({"gate": "joint_sweep", "findings": [f]}, rest_only=False) == {("x", "y"): 0.01}