Skip to content

Official Codebase of 3DCodeVerse

Python

bench: pin one plan across both A/B arms — the variance the A/A found is the planner's

gaoypengbaae4b5d57e5
diff --git a/harness/bench/pin_plan.py b/harness/bench/pin_plan.py
new file mode 100644
index 00000000..7150cb1f
--- /dev/null
+++ b/harness/bench/pin_plan.py
@@ -0,0 +1,76 @@
+"""Share ONE plan between the two arms of an A/B.
+
+WHY (docs/EVAL.md §8.1). An 8-prompt A/A of this rig — arms identical by construction —
+measured paired sd 0.202 and needs ~408 paired prompts to resolve +0.02. Its worst pair,
+on identical settings, planned **1 part** in one arm and **10** in the other: 3 vs 13 built
+parts, 4,796 vs 61,340 triangles, 0.750 vs 0.600. The spread is the PLANNER's, so no
+amount of judge sampling touches it.
+
+For a switch that acts AFTER planning, both arms can be seeded with the same ``plan.json``
+and the paired difference stops carrying that spread. It costs one planner call per pair
+instead of two — cheaper, not more expensive, unlike averaging k generations.
+
+Only valid when the switch cannot change planning: sharing a plan across a plan-side switch
+would silently delete the thing under test and the rig would then report "no effect" with
+confidence. ``codeverse.tracks.plan_features.pin_plan_blockers`` decides that, and callers
+must consult it — this module refuses to guess.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+
+__all__ = ["PinError", "plan_artifacts", "seed_plan"]
+
+#: what a planned run leaves behind that a later run resumes from
+PLAN_JSON = "plan.json"
+STAGE_RESULT = Path("stages") / "plan.json"
+RUN_STATE = "run_state.json"
+STAGE_NAME = "plan"
+
+
+class PinError(RuntimeError):
+ """The source run has no reusable plan, or the target cannot take one."""
+
+
+def plan_artifacts(run: Path) -> tuple[Path, Path, Path]:
+ """``(plan.json, stages/plan.json, run_state.json)`` of a run that has planned."""
+ return Path(run) / PLAN_JSON, Path(run) / STAGE_RESULT, Path(run) / RUN_STATE
+
+
+def seed_plan(src_run: Path, dst_run: Path) -> str:
+ """Copy ``src_run``'s planning result into ``dst_run`` so it resumes from that plan.
+
+ Returns the ``inputs_hash`` that was carried across, which the caller should assert is
+ the same for both arms — it is derived from the spec, so a mismatch means the two arms
+ were not planning the same thing and pinning would be comparing different questions.
+
+ The stage cache is keyed by ``(inputs_hash, result file exists)``
+ (``orchestrator/runner.StageRunner.stage``), so writing all three artifacts makes the
+ plan stage a cache HIT and the run continues from generation without a planner call.
+ """
+ src_plan, src_stage, src_state = plan_artifacts(src_run)
+ for p in (src_plan, src_stage, src_state):
+ if not p.is_file():
+ raise PinError(f"{src_run} has not planned: {p.name} missing")
+
+ state = json.loads(src_state.read_text())
+ entry = (state.get("stages") or {}).get(STAGE_NAME)
+ if not entry or not entry.get("inputs_hash"):
+ raise PinError(f"{src_run}/run_state.json has no cached '{STAGE_NAME}' stage")
+
+ dst_plan, dst_stage, dst_state = plan_artifacts(dst_run)
+ dst_stage.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(src_plan, dst_plan)
+ shutil.copyfile(src_stage, dst_stage)
+
+ # merge into the TARGET's state: it owns its own status, rounds and result paths, and
+ # only the plan stage crosses over (with this workspace's path, not the source's)
+ target = json.loads(dst_state.read_text()) if dst_state.is_file() else {}
+ stages = dict(target.get("stages") or {})
+ stages[STAGE_NAME] = {**entry, "result_path": str(dst_stage.resolve())}
+ target["stages"] = stages
+ dst_state.write_text(json.dumps(target, indent=1))
+ return str(entry["inputs_hash"])
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index d0b1535e..dfee7c3c 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -173,6 +173,10 @@ codeverse/
bench/ run_bench.py, report.py (reuses flywheel gallery), compare_backends.py
(preflights every model it needs; --wait-for-provider / --no-preflight),
_infra.py (outage vs model failure: infra_failed / budget_exhausted, docs/EVAL.md §7),
+ ab_plan.py (the paired control/variant A/B rig, --aa calibration mode),
+ ab_gate_rates.py (the same run's deterministic readouts, paired per prompt),
+ pin_plan.py (seed one plan into both arms so the paired delta stops carrying
+ the planner's spread — permitted only by plan_features.pin_plan_blockers),
_compare_report.py (arm table incl. the `dropped` / `over budget` loss columns;
dedups the append-only rows per (prompt, arm) so every reader agrees),
_jsonl.py (the ONE tolerant reader/append-sealer for the resumable
diff --git a/harness/tests/compare_bench/test_pin_plan.py b/harness/tests/compare_bench/test_pin_plan.py
new file mode 100644
index 00000000..b886b573
--- /dev/null
+++ b/harness/tests/compare_bench/test_pin_plan.py
@@ -0,0 +1,104 @@
+"""Sharing one plan between A/B arms (bench/pin_plan.py + the rule that permits it).
+
+docs/EVAL.md §8.1: the A/A's worst pair planned 1 part in one arm and 10 in the other on
+IDENTICAL settings. Pinning removes that term; pinning the wrong switch deletes the
+experiment. Both halves are tested here.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from bench.pin_plan import PinError, plan_artifacts, seed_plan
+
+
+def _planned_run(root, *, parts, inputs_hash="abc123"):
+ """A run directory in the state a planned harness run leaves behind."""
+ run = root / "src_run"
+ (run / "stages").mkdir(parents=True)
+ plan = {"object_name": "Chair", "parts": [{"name": f"P{i}"} for i in range(parts)]}
+ (run / "plan.json").write_text(json.dumps(plan))
+ (run / "stages" / "plan.json").write_text(json.dumps(plan))
+ (run / "run_state.json").write_text(json.dumps({
+ "status": "generating",
+ "stages": {"plan": {"name": "plan", "inputs_hash": inputs_hash,
+ "result_path": str(run / "stages" / "plan.json")}},
+ }))
+ return run
+
+
+def test_the_target_resumes_from_the_source_plan(tmp_path):
+ src = _planned_run(tmp_path, parts=10)
+ dst = tmp_path / "dst_run"
+ (dst / "stages").mkdir(parents=True)
+ (dst / "run_state.json").write_text(json.dumps({"status": "created", "stages": {}}))
+
+ got = seed_plan(src, dst)
+ assert got == "abc123"
+
+ dst_plan, dst_stage, dst_state = plan_artifacts(dst)
+ assert json.loads(dst_plan.read_text())["parts"] == json.loads((src / "plan.json").read_text())["parts"]
+ assert dst_stage.is_file(), "the stage cache result must exist or the stage is a MISS"
+ entry = json.loads(dst_state.read_text())["stages"]["plan"]
+ assert entry["inputs_hash"] == "abc123", "the hash is what makes it a cache hit"
+ assert entry["result_path"] == str(dst_stage.resolve()), "must point at the TARGET's file"
+
+
+def test_the_targets_own_state_survives(tmp_path):
+ """Only the plan stage crosses over — status, rounds and other stages are the
+ target's own, or seeding would import the source run's history."""
+ src = _planned_run(tmp_path, parts=3)
+ dst = tmp_path / "dst_run"
+ (dst / "stages").mkdir(parents=True)
+ (dst / "run_state.json").write_text(json.dumps(
+ {"status": "created", "current_round": 0, "stages": {"skeleton": {"inputs_hash": "zzz"}}}))
+
+ seed_plan(src, dst)
+ state = json.loads((dst / "run_state.json").read_text())
+ assert state["status"] == "created" and state["current_round"] == 0
+ assert state["stages"]["skeleton"]["inputs_hash"] == "zzz", "the target's other stages were dropped"
+ assert "plan" in state["stages"]
+
+
+def test_an_unplanned_source_is_refused_by_name(tmp_path):
+ (tmp_path / "src_run" / "stages").mkdir(parents=True)
+ with pytest.raises(PinError, match="has not planned"):
+ seed_plan(tmp_path / "src_run", tmp_path / "dst")
+
+
+def test_a_source_without_a_cached_stage_is_refused(tmp_path):
+ """plan.json on disk but no stage entry: the target would re-plan and the pair would
+ silently stop being pinned — worse than not pinning, because the report would claim it."""
+ src = _planned_run(tmp_path, parts=4)
+ (src / "run_state.json").write_text(json.dumps({"status": "generating", "stages": {}}))
+ dst = tmp_path / "dst_run"
+ (dst / "stages").mkdir(parents=True)
+ with pytest.raises(PinError, match="no cached 'plan' stage"):
+ seed_plan(src, dst)
+
+
+# --------------------------------------------------------------- the permission rule
+def test_only_a_post_planning_switch_may_be_pinned():
+ from codeverse.tracks.plan_features import pin_plan_blockers
+
+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "contacts"}) == []
+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "fit"}), "a plan-side switch must block"
+
+
+def test_pinning_a_plan_side_switch_would_delete_the_experiment(tmp_path):
+ """The failure this guards: both arms get one plan, so a change that only alters
+ planning becomes a no-op — and the rig then reports 'no effect' with confidence."""
+ from codeverse.tracks.plan_features import pin_plan_blockers
+
+ src = _planned_run(tmp_path, parts=10)
+ dst = tmp_path / "dst_run"
+ (dst / "stages").mkdir(parents=True)
+ (dst / "run_state.json").write_text(json.dumps({"status": "created", "stages": {}}))
+ seed_plan(src, dst)
+ # identical plans in both arms is exactly what pinning means...
+ assert json.loads((dst / "plan.json").read_text()) == json.loads((src / "plan.json").read_text())
+ # ...which is why the caller must refuse it for a switch that changes planning
+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "fit"}) == [
+ "CV3D_PLAN_FEATURES=fit changes the plan itself"]