Skip to content

Official Codebase of 3DCodeVerse

Python

bench: the threshold experiment answered itself offline — do not align the probes

ziyao32d914394770
diff --git a/harness/bench/penetration_thresholds.py b/harness/bench/penetration_thresholds.py
index fbe0f4c2..616bc25c 100644
--- a/harness/bench/penetration_thresholds.py
+++ b/harness/bench/penetration_thresholds.py
@@ -90,6 +90,42 @@ def survey(recs: list[dict]) -> dict:
return {"rounds": len(worst_per_round), "worst": worst_per_round, "fired": fired, "both": both}
+def corroborate(recs: list[dict], threshold_m: float) -> dict:
+ """Would an ERROR at ``threshold_m`` fire on real defects?
+
+ The dense probe is the reference: for every link pair connectivity records at or over
+ the threshold, ask whether the SAME round's ``joint_sweep`` also reported that pair.
+ ``both`` = a new failure the dense probe corroborates, ``conn_only`` = one it does not,
+ ``sweep_only`` = an overlap the sparse probe never saw (today's silent misses).
+ """
+ both = conn_only = sweep_only = 0
+ examples: list[str] = []
+ for data in recs:
+ for rnd in data.get("rounds") or []:
+ gates = {g["gate"]: g for g in (rnd.get("gates") or [])}
+ conn, sweep = gates.get("connectivity"), gates.get("joint_sweep")
+ if conn is None or sweep is None:
+ continue
+ deep = set()
+ for f in conn.get("findings") or []:
+ d = f.get("data") or {}
+ if d.get("kind") == "penetration" and float(d.get("depth_m") or 0.0) >= threshold_m:
+ deep.add(tuple(sorted([f.get("target", ""), str(d.get("other") or "")])))
+ flagged = set()
+ for f in sweep.get("findings") or []:
+ if f.get("severity") != "error":
+ continue
+ m = _PAIR.search(f.get("message", ""))
+ if m:
+ flagged.add(tuple(sorted([m.group(1), m.group(2)])))
+ both += len(deep & flagged)
+ conn_only += len(deep - flagged)
+ sweep_only += len(flagged - deep)
+ for pair in sorted(deep - flagged)[:1]:
+ examples.append(f"{data.get('spec', {}).get('id', '?')}: {'|'.join(pair)}")
+ return {"both": both, "conn_only": conn_only, "sweep_only": sweep_only, "examples": examples}
+
+
def report(root: Path) -> str:
s = survey(records(root))
rounds, worst = s["rounds"], s["worst"]
@@ -105,6 +141,16 @@ def report(root: Path) -> str:
for th in sorted(CANDIDATES):
n = sum(1 for w in worst if w >= th)
lines.append(f"| {th * 1000:.0f} mm | {n} | {n / max(1, rounds):.0%} |")
+ recs = records(root)
+ for th in (0.002, 0.005):
+ c = corroborate(recs, th)
+ total = c["both"] + c["conn_only"]
+ lines += ["", f"an ERROR at {th * 1000:.0f} mm would fire on {total} link pair(s): "
+ f"**{c['both']} corroborated** by the dense probe in the same round, "
+ f"{c['conn_only']} not; the dense probe additionally flags {c['sweep_only']} "
+ f"pair(s) this threshold still misses."]
+ if c["examples"]:
+ lines.append(f" uncorroborated examples: {', '.join(c['examples'][:4])}")
if s["both"]:
lines += ["", f"the two probes on the same link pair ({len(s['both'])} sweep ERROR findings; "
f"sweep tolerance {SWEEP_TOLERANCE_M * 1000:.0f} mm):",
diff --git a/harness/codeverse/spatial/connectivity.py b/harness/codeverse/spatial/connectivity.py
index de117a21..88f63eb6 100644
--- a/harness/codeverse/spatial/connectivity.py
+++ b/harness/codeverse/spatial/connectivity.py
@@ -62,7 +62,12 @@ GATE = "connectivity"
PlannedEdge = tuple[str, "str | Sequence[str]"]
#: parts smaller than this (max extent) are ignored for floating checks (INFO only)
MIN_PART_SIZE_M = 0.01
-#: penetration depth that is reported as WARN / ERROR
+#: penetration depth that is reported as WARN / ERROR. These bound the REST pose only,
+#: and with a 600-point surface draw per part: this check asks "do these two surfaces sit
+#: inside each other as built". ``joint_sweep`` asks a different question with a denser
+#: probe — "does moving a joint drive one link through another" — at a 1 mm tolerance, and
+#: the two agree on only 7 of 50 link pairs at 2 mm (bench/penetration_thresholds.py,
+#: D-entry 2026-09-04). They are not one check with two dials; do not align the numbers.
PENETRATION_WARN_M = 0.002
PENETRATION_ERROR_M = 0.010
#: fraction of one part's samples inside the other before we call it penetration ...
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index 6d31464b..85cffdc5 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -777,9 +777,21 @@ written) that were accepted because the code works that way and the tests pin it
The visible consequence: on `coupled_v1`, 11 sweep ERRORs about the REST pose were
raised in rounds where the connectivity gate PASSED — the harness told the agent two
different things about one pose. Blast radius of a change, already computed: an ERROR
- at 5 mm moves 1 round of 374, at 2 mm it moves 36 (10 %). Left as it ships: a gate
- threshold is a behaviour change and this branch is under review, so the finding, the
- numbers and the script land first.
+ at 5 mm moves 1 round of 374, at 2 mm it moves 36 (10 %).
+
+ **And then the threshold experiment answered itself, offline (2026-09-04): do not
+ change it.** Using the dense probe as the reference — for every pair connectivity
+ records at or over a candidate threshold, did the SAME round's sweep flag that pair? —
+ an ERROR at 2 mm would fire on 50 pairs of which only **7 are corroborated**, while the
+ 43 uncorroborated ones are static contacts the design intends (`shoulder_lock_knob|
+ swivel_post`, `base_underframe|center_top`) that a 600-point surface draw reads as
+ penetration; and the dense probe would still be alone on **120** pairs. At 5 mm: 2
+ pairs, 0 corroborated. The two probes are not measuring the same thing at different
+ sensitivities — one asks "do these surfaces sit inside each other at rest", the other
+ "does moving this joint drive one link through another". A shared threshold would
+ create uncorroborated failures and still miss what the sweep finds. What is wrong is
+ that the two read as one check with two dials; the fix belongs in what each is called
+ and documented to do, not in the numbers.
## Rejected / deferred
diff --git a/harness/tests/compare_bench/test_session_stats.py b/harness/tests/compare_bench/test_session_stats.py
index 9ae2380f..04dc86dd 100644
--- a/harness/tests/compare_bench/test_session_stats.py
+++ b/harness/tests/compare_bench/test_session_stats.py
@@ -164,3 +164,31 @@ def test_the_coupling_survey_counts_runs_not_copies_of_the_same_urdf(tmp_path: P
found = _urdfs(tmp_path)
assert [p.parent.name for p in found] == ["artifacts"], "one URDF per run, the built one"
assert _prompt_of(found[0]) == "cpl_umbrella"
+
+
+def test_the_two_penetration_probes_are_compared_not_merged(tmp_path: Path) -> None:
+ """The corroboration view is what stopped a threshold change: for every pair
+ connectivity records at or over a candidate depth, did the same round's sweep flag it?
+ Aligning the numbers would have created failures the dense probe does not support."""
+ from bench.penetration_thresholds import corroborate
+
+ rec = {"spec": {"id": "b/p", "track": "articulated_object"}, "rounds": [{
+ "gates": [
+ {"gate": "connectivity", "passed": True, "findings": [
+ {"severity": "warn", "target": "knob", "data": {"kind": "penetration", "other": "post",
+ "depth_m": 0.004}},
+ {"severity": "warn", "target": "arm", "data": {"kind": "penetration", "other": "deck",
+ "depth_m": 0.003}},
+ ]},
+ {"gate": "joint_sweep", "passed": False, "findings": [
+ {"severity": "error", "message": "links 'arm|deck' overlap in 2 of the sampled poses (worst 6.0 mm)"},
+ {"severity": "error", "message": "links 'rod|frame' overlap in 1 of the sampled poses (worst 9.0 mm)"},
+ ]},
+ ]}]}
+
+ c = corroborate([rec], 0.002)
+ assert c["both"] == 1 # arm|deck: both probes
+ assert c["conn_only"] == 1 # knob|post: a static contact only the sparse draw calls penetration
+ assert c["sweep_only"] == 1 # rod|frame: the sparse draw never saw it
+ assert c["examples"] == ["b/p: knob|post"]
+ assert corroborate([rec], 0.005)["both"] == 0 # nothing that deep on the sparse side