Official Codebase of 3DCodeVerse
Python
bench: pin one plan across both A/B arms — the variance the A/A found is the planner's
gaoypengbaae4b5d57e5
diff --git a/harness/bench/pin_plan.py b/harness/bench/pin_plan.pynew file mode 100644index 00000000..7150cb1f--- /dev/null+++ b/harness/bench/pin_plan.py@@ -0,0 +1,76 @@+"""Share ONE plan between the two arms of an A/B.++WHY (docs/EVAL.md §8.1). An 8-prompt A/A of this rig — arms identical by construction —+measured paired sd 0.202 and needs ~408 paired prompts to resolve +0.02. Its worst pair,+on identical settings, planned **1 part** in one arm and **10** in the other: 3 vs 13 built+parts, 4,796 vs 61,340 triangles, 0.750 vs 0.600. The spread is the PLANNER's, so no+amount of judge sampling touches it.++For a switch that acts AFTER planning, both arms can be seeded with the same ``plan.json``+and the paired difference stops carrying that spread. It costs one planner call per pair+instead of two — cheaper, not more expensive, unlike averaging k generations.++Only valid when the switch cannot change planning: sharing a plan across a plan-side switch+would silently delete the thing under test and the rig would then report "no effect" with+confidence. ``codeverse.tracks.plan_features.pin_plan_blockers`` decides that, and callers+must consult it — this module refuses to guess.+"""++from __future__ import annotations++import json+import shutil+from pathlib import Path++__all__ = ["PinError", "plan_artifacts", "seed_plan"]++#: what a planned run leaves behind that a later run resumes from+PLAN_JSON = "plan.json"+STAGE_RESULT = Path("stages") / "plan.json"+RUN_STATE = "run_state.json"+STAGE_NAME = "plan"+++class PinError(RuntimeError):+ """The source run has no reusable plan, or the target cannot take one."""+++def plan_artifacts(run: Path) -> tuple[Path, Path, Path]:+ """``(plan.json, stages/plan.json, run_state.json)`` of a run that has planned."""+ return Path(run) / PLAN_JSON, Path(run) / STAGE_RESULT, Path(run) / RUN_STATE+++def seed_plan(src_run: Path, dst_run: Path) -> str:+ """Copy ``src_run``'s planning result into ``dst_run`` so it resumes from that plan.++ Returns the ``inputs_hash`` that was carried across, which the caller should assert is+ the same for both arms — it is derived from the spec, so a mismatch means the two arms+ were not planning the same thing and pinning would be comparing different questions.++ The stage cache is keyed by ``(inputs_hash, result file exists)``+ (``orchestrator/runner.StageRunner.stage``), so writing all three artifacts makes the+ plan stage a cache HIT and the run continues from generation without a planner call.+ """+ src_plan, src_stage, src_state = plan_artifacts(src_run)+ for p in (src_plan, src_stage, src_state):+ if not p.is_file():+ raise PinError(f"{src_run} has not planned: {p.name} missing")++ state = json.loads(src_state.read_text())+ entry = (state.get("stages") or {}).get(STAGE_NAME)+ if not entry or not entry.get("inputs_hash"):+ raise PinError(f"{src_run}/run_state.json has no cached '{STAGE_NAME}' stage")++ dst_plan, dst_stage, dst_state = plan_artifacts(dst_run)+ dst_stage.parent.mkdir(parents=True, exist_ok=True)+ shutil.copyfile(src_plan, dst_plan)+ shutil.copyfile(src_stage, dst_stage)++ # merge into the TARGET's state: it owns its own status, rounds and result paths, and+ # only the plan stage crosses over (with this workspace's path, not the source's)+ target = json.loads(dst_state.read_text()) if dst_state.is_file() else {}+ stages = dict(target.get("stages") or {})+ stages[STAGE_NAME] = {**entry, "result_path": str(dst_stage.resolve())}+ target["stages"] = stages+ dst_state.write_text(json.dumps(target, indent=1))+ return str(entry["inputs_hash"])diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex d0b1535e..dfee7c3c 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -173,6 +173,10 @@ codeverse/bench/ run_bench.py, report.py (reuses flywheel gallery), compare_backends.py(preflights every model it needs; --wait-for-provider / --no-preflight),_infra.py (outage vs model failure: infra_failed / budget_exhausted, docs/EVAL.md §7),+ ab_plan.py (the paired control/variant A/B rig, --aa calibration mode),+ ab_gate_rates.py (the same run's deterministic readouts, paired per prompt),+ pin_plan.py (seed one plan into both arms so the paired delta stops carrying+ the planner's spread — permitted only by plan_features.pin_plan_blockers),_compare_report.py (arm table incl. the `dropped` / `over budget` loss columns;dedups the append-only rows per (prompt, arm) so every reader agrees),_jsonl.py (the ONE tolerant reader/append-sealer for the resumablediff --git a/harness/tests/compare_bench/test_pin_plan.py b/harness/tests/compare_bench/test_pin_plan.pynew file mode 100644index 00000000..b886b573--- /dev/null+++ b/harness/tests/compare_bench/test_pin_plan.py@@ -0,0 +1,104 @@+"""Sharing one plan between A/B arms (bench/pin_plan.py + the rule that permits it).++docs/EVAL.md §8.1: the A/A's worst pair planned 1 part in one arm and 10 in the other on+IDENTICAL settings. Pinning removes that term; pinning the wrong switch deletes the+experiment. Both halves are tested here.+"""++from __future__ import annotations++import json++import pytest++from bench.pin_plan import PinError, plan_artifacts, seed_plan+++def _planned_run(root, *, parts, inputs_hash="abc123"):+ """A run directory in the state a planned harness run leaves behind."""+ run = root / "src_run"+ (run / "stages").mkdir(parents=True)+ plan = {"object_name": "Chair", "parts": [{"name": f"P{i}"} for i in range(parts)]}+ (run / "plan.json").write_text(json.dumps(plan))+ (run / "stages" / "plan.json").write_text(json.dumps(plan))+ (run / "run_state.json").write_text(json.dumps({+ "status": "generating",+ "stages": {"plan": {"name": "plan", "inputs_hash": inputs_hash,+ "result_path": str(run / "stages" / "plan.json")}},+ }))+ return run+++def test_the_target_resumes_from_the_source_plan(tmp_path):+ src = _planned_run(tmp_path, parts=10)+ dst = tmp_path / "dst_run"+ (dst / "stages").mkdir(parents=True)+ (dst / "run_state.json").write_text(json.dumps({"status": "created", "stages": {}}))++ got = seed_plan(src, dst)+ assert got == "abc123"++ dst_plan, dst_stage, dst_state = plan_artifacts(dst)+ assert json.loads(dst_plan.read_text())["parts"] == json.loads((src / "plan.json").read_text())["parts"]+ assert dst_stage.is_file(), "the stage cache result must exist or the stage is a MISS"+ entry = json.loads(dst_state.read_text())["stages"]["plan"]+ assert entry["inputs_hash"] == "abc123", "the hash is what makes it a cache hit"+ assert entry["result_path"] == str(dst_stage.resolve()), "must point at the TARGET's file"+++def test_the_targets_own_state_survives(tmp_path):+ """Only the plan stage crosses over — status, rounds and other stages are the+ target's own, or seeding would import the source run's history."""+ src = _planned_run(tmp_path, parts=3)+ dst = tmp_path / "dst_run"+ (dst / "stages").mkdir(parents=True)+ (dst / "run_state.json").write_text(json.dumps(+ {"status": "created", "current_round": 0, "stages": {"skeleton": {"inputs_hash": "zzz"}}}))++ seed_plan(src, dst)+ state = json.loads((dst / "run_state.json").read_text())+ assert state["status"] == "created" and state["current_round"] == 0+ assert state["stages"]["skeleton"]["inputs_hash"] == "zzz", "the target's other stages were dropped"+ assert "plan" in state["stages"]+++def test_an_unplanned_source_is_refused_by_name(tmp_path):+ (tmp_path / "src_run" / "stages").mkdir(parents=True)+ with pytest.raises(PinError, match="has not planned"):+ seed_plan(tmp_path / "src_run", tmp_path / "dst")+++def test_a_source_without_a_cached_stage_is_refused(tmp_path):+ """plan.json on disk but no stage entry: the target would re-plan and the pair would+ silently stop being pinned — worse than not pinning, because the report would claim it."""+ src = _planned_run(tmp_path, parts=4)+ (src / "run_state.json").write_text(json.dumps({"status": "generating", "stages": {}}))+ dst = tmp_path / "dst_run"+ (dst / "stages").mkdir(parents=True)+ with pytest.raises(PinError, match="no cached 'plan' stage"):+ seed_plan(src, dst)+++# --------------------------------------------------------------- the permission rule+def test_only_a_post_planning_switch_may_be_pinned():+ from codeverse.tracks.plan_features import pin_plan_blockers++ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "contacts"}) == []+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "fit"}), "a plan-side switch must block"+++def test_pinning_a_plan_side_switch_would_delete_the_experiment(tmp_path):+ """The failure this guards: both arms get one plan, so a change that only alters+ planning becomes a no-op — and the rig then reports 'no effect' with confidence."""+ from codeverse.tracks.plan_features import pin_plan_blockers++ src = _planned_run(tmp_path, parts=10)+ dst = tmp_path / "dst_run"+ (dst / "stages").mkdir(parents=True)+ (dst / "run_state.json").write_text(json.dumps({"status": "created", "stages": {}}))+ seed_plan(src, dst)+ # identical plans in both arms is exactly what pinning means...+ assert json.loads((dst / "plan.json").read_text()) == json.loads((src / "plan.json").read_text())+ # ...which is why the caller must refuse it for a switch that changes planning+ assert pin_plan_blockers({"CV3D_PLAN_FEATURES": "fit"}) == [+ "CV3D_PLAN_FEATURES=fit changes the plan itself"]