Skip to content

Official Codebase of 3DCodeVerse

Python

plan-loop: the wave's writeup, and the rule that decides when two A/B arms may share one plan

gaoypeng42bf95c17e02
diff --git a/harness/codeverse/tracks/plan_features.py b/harness/codeverse/tracks/plan_features.py
index ee80a3cf..29c4e64b 100644
--- a/harness/codeverse/tracks/plan_features.py
+++ b/harness/codeverse/tracks/plan_features.py
@@ -41,6 +41,40 @@ GRAPH_EXAMPLE = "graph_example" # the planner's worked example written as a gra
CONSISTENCY = "consistency" # plan overall bbox checked against the brief's dimension rows
KNOWN_FEATURES: tuple[str, ...] = (FIT, CONTACTS, GRAPH, GRAPH_BUDGET, GRAPH_EXAMPLE, CONSISTENCY)
+#: Which stage a switch changes. This is not documentation — ``bench/ab_plan.py --pin-plan``
+#: reads it, and pinning a PLAN-side switch would silently disable the very thing under test
+#: (both arms would share one plan, so a change that only alters planning becomes a no-op the
+#: rig would then report as "no effect"). A switch missing from here is treated as plan-side:
+#: refusing to pin costs one noisy A/B, pinning wrongly costs a confident wrong answer.
+#: docs/EVAL.md §8.1 measured why pinning matters — the A/A's worst pair differed 1 part vs 10.
+PLAN_SIDE: frozenset[str] = frozenset({FIT, GRAPH, GRAPH_BUDGET, GRAPH_EXAMPLE, CONSISTENCY})
+GENERATION_SIDE: frozenset[str] = frozenset({CONTACTS})
+#: env switches outside CV3D_PLAN_FEATURES, same rule
+PLAN_SIDE_ENV: frozenset[str] = frozenset({"CV3D_PLAN_BRIEF", "CV3D_SCOPED_PARTS"})
+GENERATION_SIDE_ENV: frozenset[str] = frozenset({"CV3D_SKILLS", "CV3D_SKILLS_MAX", "CV3D_SKILLS_UNVERIFIED",
+ "CV3D_DETAIL_ROUNDS", "CV3D_REFERENCE_DIFF"})
+
+
+def pin_plan_blockers(variant_env: dict[str, str]) -> list[str]:
+ """Which of ``variant_env``'s switches forbid sharing one plan between the arms.
+
+ Empty list = every switch this A/B changes acts after planning, so both arms can be
+ seeded with the same ``plan.json`` and the paired difference stops carrying the
+ planner's spread (the dominant variance term, docs/EVAL.md §8.1).
+ """
+ blockers: list[str] = []
+ for key, value in sorted(variant_env.items()):
+ if key == PLAN_FEATURES_ENV:
+ names = set(KNOWN_FEATURES) if value.strip() == "all" else {
+ n.strip().lstrip("-") for n in value.split(",") if n.strip()}
+ for name in sorted(names & (set(KNOWN_FEATURES) - GENERATION_SIDE)):
+ blockers.append(f"{PLAN_FEATURES_ENV}={name} changes the plan itself")
+ elif key in GENERATION_SIDE_ENV:
+ continue
+ else:
+ blockers.append(f"{key} is not known to act after planning")
+ return blockers
+
#: Plan-loop switches the tree ACTUALLY reads, name → the module that reads it. Kept
#: honest by tests/orchestrator_tracks/test_plan_features.py, which greps the tree.
LIVE_SWITCHES: dict[str, str] = {
@@ -111,6 +145,7 @@ def plan_feature_on(name: str) -> bool:
return name in plan_features()
-__all__ = ["CONSISTENCY", "CONTACTS", "DEAD_SWITCHES", "FIT", "GRAPH", "GRAPH_BUDGET", "GRAPH_EXAMPLE",
+__all__ = ["CONSISTENCY", "CONTACTS", "DEAD_SWITCHES", "FIT", "GENERATION_SIDE", "GENERATION_SIDE_ENV",
+ "GRAPH", "GRAPH_BUDGET", "GRAPH_EXAMPLE", "PLAN_SIDE", "PLAN_SIDE_ENV", "pin_plan_blockers",
"KNOWN_FEATURES", "LIVE_SWITCHES", "PLAN_FEATURES_ENV", "dead_env_keys", "parse_features",
"plan_feature_on", "plan_features"]
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 49cf240f..14b1c7ab 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -143,7 +143,8 @@ codeverse/
real dimensions / sub-assemblies / signature features; never fatal, CV3D_PLAN_BRIEF),
plan_budget.py (plan size derived from the request, capped per language),
plan_features.py (CV3D_PLAN_FEATURES: one switch per plan-loop change, so each
- can be A/B'd alone — docs/PLAN_LOOP.md),
+ can be A/B'd alone, + pin_plan_blockers() deciding when two arms may share
+ one plan — docs/PLAN_LOOP.md, docs/EVAL.md §8.1),
plan_examples.py (worked plans shown to the planner), depth.py + detailing.py
(per-part detail pass), envelope.py (bbox envelope), scene_asset_gen.py,
skills_hook.py (the round's view of codeverse/skills: attach before generating,
diff --git a/harness/docs/PLAN_LOOP.md b/harness/docs/PLAN_LOOP.md
new file mode 100644
index 00000000..295e2e09
--- /dev/null
+++ b/harness/docs/PLAN_LOOP.md
@@ -0,0 +1,101 @@
+# The plan-loop wave — what it measured, and why it kept nothing
+
+2026-08-24 → 25. Goal: stop hard-coding plan size ("6 to 14 parts") and let the plan come
+from the model's common-sense knowledge of the real object, with graph/hierarchical
+structure where it earns its keep — and prove the addition is a **guaranteed non-negative
+improvement** before shipping it on.
+
+**Outcome: nothing was kept.** Not because the ideas were wrong, but because the wave
+measured its own instrument first and found it could not resolve them. That measurement is
+the wave's real product and it now governs every A/B in this repo (`docs/EVAL.md` §8, §8.1).
+
+---
+
+## 1. What was designed
+
+The design agent mined 47 recorded plans and found the ordering fact for everything after:
+
+| plan-side quantity | vs assembly_fit | vs overall |
+|---|--:|--:|
+| **undeclared pairs overlapping > 10 mm** | **−0.50** | **−0.50** |
+| declared pairs with a planned gap | −0.38 | −0.28 |
+| number of parts | −0.13 | −0.11 |
+
+Plans with ≥ 1 pair of parts that overlap but that the plan never says touch: `assembly_fit`
+0.683, 1.82 gate errors. Plans with none: **0.811, 0.22**. **Part count does not predict
+fit; undeclared planned overlap does.** Six changes were ordered from that, C1…C6, each
+behind one switch (`codeverse/tracks/plan_features.py`, `CV3D_PLAN_FEATURES`).
+
+## 2. What happened to each
+
+| id | change | outcome |
+|---|---|---|
+| **C0** | (not a change) the A/A calibration | **the wave's most valuable result** — see §3 |
+| **C1** | `fit` — plan-level fit check, one repair re-ask | **REVERTED.** 2 regressions, mean Δ −0.020, and 3 of 8 variant cells died outright with `PlanningError: part OuterShell touches unknown part CarriageLever`. It turned a recoverable planning slip into a total loss. Patch kept at `bench/out/plan_loop/C1/C1_fit.patch`. |
+| **C2** | `contacts` — every declared contact rendered to the builder with computed mating faces | **NEVER MEASURED.** Implemented, offline-green (15 tests), reverted from the tree per the loop rule. Patch at `bench/out/plan_loop/C2/C2_contacts.patch`. |
+| C3–C6 | graph edges, graph budget, graph example, consistency | not reached |
+
+C1 was attempted four times. Attempt 1 gave the revert above; attempt 2 was discarded
+(`invalid_attempt1_maintree_import`) after the A/B was found to be running the MAIN tree's
+harness in **both** arms — comparing a tree with itself while looking healthy; attempt 3 was
+discarded (`invalid_attempt2_touches_hardfail`); attempt 4 lost **11 of 16 cells** to a
+provider 503 window and produced zero paired prompts.
+
+## 3. The instrument, measured
+
+An 8-prompt **A/A** — arms identical by construction — on the same battery, same judge:
+
+* paired **sd 0.202**, SE 0.082, 2 SE band **±0.165**;
+* the rig's own conclusion, printed in its summary: **~408 paired prompts** would be needed
+ before a +0.02 decision threshold is resolvable;
+* worst pair `mech_hard_pitcher_pump`, **identical settings**: one arm planned **1 part**
+ (`WoodenPlatform`) and scored **0.750 / passed**; the other planned **10** and scored
+ 0.600. 3 vs 13 built parts, 4,796 vs 61,340 triangles.
+
+So the variance is **in the planner**, not the judge — the two arms were not two
+measurements of one artifact, they were two different artifacts. More judge samples cannot
+touch it. This is why the wave's premise (8-prompt A/Bs screening ±0.02 changes) could
+never have worked, and why stopping was the right call rather than a fifth C1 attempt.
+
+## 4. What shipped from it anyway
+
+* **`bench/ab_plan.py`** — the paired control/variant rig: arms launched together so the
+ weather matches, `infra_failed` excluded not zeroed, `--aa` calibration mode, and a
+ **Confidence** block (paired sd, SE, 2 SE band, `separated from noise: yes|NO`, and
+ `n_for_power`) printed beside every verdict so a screen can never read as a proof.
+* **`bench/ab_gate_rates.py`** — the same run's *deterministic* readouts paired per prompt,
+ because a contact-table change targets penetrating pairs, not a judged score.
+* **`codeverse/tracks/plan_features.py`** — one switch per change, so each is A/B-able alone.
+* **`docs/EVAL.md` §8 and §8.1** — the noise floor and where it comes from.
+* Two defects found in the rig itself while trying to use it: an A/B launched from a
+ worktree silently ran the main tree's harness in both arms, and `--report-only` rebuilt
+ `summary.md` from bare flags, relabelling a real A/B as an A/A in the one file that
+ outlives the run.
+
+## 5. Next, precisely
+
+**Pin the plan.** For a change that acts *after* planning, plan once per prompt and seed
+both arms with the same `plan.json`; the paired delta then stops carrying the planner's
+spread. It costs one planner call *less* per pair, not k times more.
+
+The decision procedure is implemented and tested —
+`codeverse.tracks.plan_features.pin_plan_blockers(variant_env)` returns the switches that
+forbid sharing a plan, and refuses anything it cannot prove acts after planning (refusing
+costs one noisy A/B; pinning wrongly costs a confident wrong answer). By that rule **C2
+`contacts` is pinnable and C1 `fit` is not**.
+
+Still to build: the seeding itself in `ab_plan.py` (`--pin-plan`) — run the plan stage once
+into a shared dir, then copy `plan.json`, `stages/plan.json` and the `run_state.json` stage
+entry into each arm's run workspace before spawning it. The stage cache keys on an inputs
+hash computed from the spec, which is identical across arms, so a seeded arm skips planning
+and resumes from it. Then re-run C2 with `--pin-plan` and read out `ab_gate_rates.py`
+(penetrating pairs, max depth, floating parts) as the primary signal, with the judged score
+secondary under a sign test.
+
+Also open, from the same A/A: **7 of 122 recorded static-object runs shipped a plan with
+≤ 1 part**, all *after* the plan-budget gate landed. The gate is not broken —
+`plan_quality_complaint` fires on exactly that plan and emits `plan.thin` — but after
+`MAX_QUALITY_REASKS` the planner accepts what came back and tells nobody: that pitcher-pump
+run carries `status: passed`, `score: 0.75` and no trace of the complaint. The verdict
+should reach the record and the rubric the way `missing_must_acceptance` does. Explicitly
+**not** by failing the run — that was C1.
diff --git a/harness/tests/orchestrator_tracks/test_plan_features.py b/harness/tests/orchestrator_tracks/test_plan_features.py
index 04e9b3ae..64bca37a 100644
--- a/harness/tests/orchestrator_tracks/test_plan_features.py
+++ b/harness/tests/orchestrator_tracks/test_plan_features.py
@@ -113,3 +113,48 @@ def test_ab_plan_still_accepts_a_live_switch(monkeypatch, capsys):
assert len(seen) == 2, capsys.readouterr()
assert "CV3D_PLAN_BRIEF" in seen[0]
+
+
+# --------------------------------------------------------------------- pin-plan safety
+def test_a_generation_side_switch_may_share_one_plan():
+ """`contacts` renders a table into the BUILDER's prompt from an unchanged plan, so
+ both arms can be seeded with the same plan.json and the paired delta stops carrying
+ the planner's spread — the dominant variance term (docs/EVAL.md §8.1)."""
+ from codeverse.tracks.plan_features import pin_plan_blockers
+
+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "contacts"}) == []
+ assert pin_plan_blockers({"CV3D_SKILLS": "1", "CV3D_SKILLS_MAX": "3"}) == []
+ assert pin_plan_blockers({}) == []
+
+
+def test_a_plan_side_switch_is_refused_by_name():
+ """Pinning `fit` would hand both arms one plan and so silently delete the change
+ under test — the rig would then report "no effect" with confidence."""
+ from codeverse.tracks.plan_features import pin_plan_blockers
+
+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "fit"}) == [
+ "CV3D_PLAN_FEATURES=fit changes the plan itself"]
+ # one plan-side name in a list of otherwise-safe ones still blocks
+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "contacts,fit"}) == [
+ "CV3D_PLAN_FEATURES=fit changes the plan itself"]
+ assert len(pin_plan_blockers({"CV3D_PLAN_FEATURES": "all"})) == 5
+
+
+def test_an_unclassified_switch_defaults_to_refusing():
+ """Refusing to pin costs one noisy A/B; pinning wrongly costs a confident wrong
+ answer. So the default for anything unknown is: do not pin."""
+ from codeverse.tracks.plan_features import pin_plan_blockers
+
+ assert pin_plan_blockers({"CV3D_MYSTERY_KNOB": "1"}) == [
+ "CV3D_MYSTERY_KNOB is not known to act after planning"]
+ assert pin_plan_blockers({"CV3D_PLAN_BRIEF": "off"}) == [
+ "CV3D_PLAN_BRIEF is not known to act after planning"]
+
+
+def test_every_known_feature_is_classified():
+ """A new feature must be put on one side or the other in the same commit; otherwise
+ it silently inherits 'plan-side' and nobody notices the A/B got noisier."""
+ from codeverse.tracks.plan_features import GENERATION_SIDE, KNOWN_FEATURES, PLAN_SIDE
+
+ assert set(KNOWN_FEATURES) == PLAN_SIDE | GENERATION_SIDE
+ assert not (PLAN_SIDE & GENERATION_SIDE)