Official Codebase of 3DCodeVerse
Python
ab_plan: a pinned plan lost to a storm records the pair, not the battery
gaoypeng193a69467060
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.pyindex 0f8dd6f3..5a342364 100644--- a/harness/bench/ab_plan.py+++ b/harness/bench/ab_plan.py@@ -44,6 +44,7 @@ from __future__ import annotationsimport argparseimport json+import loggingimport osimport subprocessimport sys@@ -87,7 +88,8 @@ def _assert_local_codeverse() -> None:f"refusing to run: `import codeverse` resolved to {got}, not {want}.\n"f" An editable install is shadowing this tree, so both arms would run the same\n"f" code and the A/B would measure nothing. Launch with:\n"- f" PYTHONPATH={REPO} python bench/ab_plan.py ...")+ f" PYTHONPATH={REPO} python bench/ab_plan.py ..."+ )_assert_local_codeverse()@@ -109,6 +111,8 @@ from bench.run_bench import Battery, BenchPrompt, select_prompts # noqa: E402from codeverse.tracks.plan_features import pin_plan_blockers # noqa: E402from codeverse.workspace import Workspace # noqa: E402+log = logging.getLogger(__name__)+DEFAULT_GENERATOR = "api-agent:gemini:gemini-3.7-flash"DEFAULT_JUDGE = "gemini:gemini-3.1-pro-preview"#: the per-child cap. The flat name is a first-class alias of ``CV3D_RATE__MAX_IN_FLIGHT``@@ -125,8 +129,12 @@ PARALLEL = 2class AbOptions(BaseModel):generator: str = DEFAULT_GENERATORjudge: str = DEFAULT_JUDGE- loop_judge: str | None = Field(default=None, description="in-loop judge for BOTH arms (None → settings default)")- planner: str | None = Field(default=None, description="planner for BOTH arms (None → settings default)")+ loop_judge: str | None = Field(+ default=None, description="in-loop judge for BOTH arms (None → settings default)"+ )+ planner: str | None = Field(+ default=None, description="planner for BOTH arms (None → settings default)"+ )rounds: int = 2max_usd: float = 2.5max_minutes: float = 45.0@@ -135,19 +143,39 @@ class AbOptions(BaseModel):tiers: list[str] = Field(default_factory=list)limit: int | None = Noneresume: bool = True- redo_status: list[str] = Field(default_factory=list, description="re-run prompts where EITHER arm has one of these")- variant_env: dict[str, str] = Field(default_factory=dict, description="applied to the variant arm only")+ redo_status: list[str] = Field(+ default_factory=list, description="re-run prompts where EITHER arm has one of these"+ )+ variant_env: dict[str, str] = Field(+ default_factory=dict, description="applied to the variant arm only"+ )max_in_flight: int = DEFAULT_MAX_IN_FLIGHT- aa: bool = Field(default=False, description="calibration: run BOTH arms as the control, so the delta IS the noise")- redo_fresh: bool = Field(default=True, description="a redo regenerates both arms instead of resuming them")- pin_plan: bool = Field(default=False, description="plan ONCE per prompt and seed both arms with it "- "(generation-side switches only: see pin_plan_blockers)")+ aa: bool = Field(+ default=False,+ description="calibration: run BOTH arms as the control, so the delta IS the noise",+ )+ redo_fresh: bool = Field(+ default=True, description="a redo regenerates both arms instead of resuming them"+ )+ pin_plan: bool = Field(+ default=False,+ description="plan ONCE per prompt and seed both arms with it "+ "(generation-side switches only: see pin_plan_blockers)",+ )def compare_options(self) -> CompareOptions:"""The per-cell options handed to ``compare_backends.run_cell`` (both arms identical)."""- return CompareOptions(judge=self.judge, loop_judge=self.loop_judge, planner=self.planner, rounds=self.rounds,- max_usd=self.max_usd, max_minutes=self.max_minutes, n_samples=self.n_samples,- resume=self.resume, pairwise=False)+ return CompareOptions(+ judge=self.judge,+ loop_judge=self.loop_judge,+ planner=self.planner,+ rounds=self.rounds,+ max_usd=self.max_usd,+ max_minutes=self.max_minutes,+ n_samples=self.n_samples,+ resume=self.resume,+ pairwise=False,+ )def inherited_max_in_flight() -> int:@@ -174,8 +202,10 @@ def parse_variant_env(items: Sequence[str]) -> dict[str, str]:if k.strip() in (MAX_IN_FLIGHT_ENV, NESTED_MAX_IN_FLIGHT_ENV):# child_env sets the cap authoritatively so the §23 admission check and the# children agree; accepting it here too would silently discard one of them.- raise ValueError(f"--variant-env {k.strip()} is not allowed: the in-flight cap is "- f"the driver's (--max-in-flight), and the admission check budgets on it")+ raise ValueError(+ f"--variant-env {k.strip()} is not allowed: the in-flight cap is "+ f"the driver's (--max-in-flight), and the admission check budgets on it"+ )out[k.strip()] = vreturn out@@ -221,16 +251,33 @@ def _harness_arm(generator: str) -> Arm:return Arm(raw=f"harness:{generator}", kind="harness", target=generator)-def worker_argv(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> list[str]:- return [sys.executable, str(Path(__file__).resolve()), "cell", "--prompts", str(battery_path), "--out", str(out),- "--id", item_id, "--arm", arm, "--options-json", opts.model_dump_json()]+def worker_argv(+ battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions+) -> list[str]:+ return [+ sys.executable,+ str(Path(__file__).resolve()),+ "cell",+ "--prompts",+ str(battery_path),+ "--out",+ str(out),+ "--id",+ item_id,+ "--arm",+ arm,+ "--options-json",+ opts.model_dump_json(),+ ]# ----------------------------------------------------------------------------- one cell, in a childCellRunner = Callable[[Path, Path, BenchPrompt, str, AbOptions], CellResult]-def spawn_cell(battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts: AbOptions) -> CellResult:+def spawn_cell(+ battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts: AbOptions+) -> CellResult:"""Run one (prompt, arm) cell in a child process and read back its ``cell.json``.A child that dies without writing a cell (killed, import error, unhandled outage)@@ -240,23 +287,39 @@ def spawn_cell(battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts:cell = cell_dir(out, arm, item.id, opts.generator)cell.mkdir(parents=True, exist_ok=True)log, marker = cell / "worker.log", cell / "cell.json"- marker.unlink(missing_ok=True) # a stale verdict from an earlier attempt must not stand in for this one+ marker.unlink(+ missing_ok=True+ ) # a stale verdict from an earlier attempt must not stand in for this onet0 = time.time()with log.open("a") as fh:fh.write(f"--- {datetime.now(UTC).isoformat()} {arm} {item.id}\n")fh.flush()- rc = subprocess.run(worker_argv(battery_path, out, item.id, arm, opts), cwd=REPO, env=child_env(arm, opts),- stdout=fh, stderr=subprocess.STDOUT, check=False).returncode+ rc = subprocess.run(+ worker_argv(battery_path, out, item.id, arm, opts),+ cwd=REPO,+ env=child_env(arm, opts),+ stdout=fh,+ stderr=subprocess.STDOUT,+ check=False,+ ).returncodeif marker.is_file():return CellResult.model_validate_json(marker.read_text())# errors="replace": the child's raw stdout lands in this log, and one non-UTF-8 byte# from an agent CLI used to raise UnicodeDecodeError *here* — on the very path whose# job is to classify an outage — killing the whole driver through fut.result().tail = log.read_text(errors="replace")[-1500:] if log.is_file() else ""- return CellResult(prompt_id=item.id, tier=item.tier, arm=arm, kind="harness", target=opts.generator,- judge=opts.judge, workspace=str(cell), wall_s=round(time.time() - t0, 1),- status="infra_failed" if is_infra_failure(tail) else "error",- error=f"worker exited {rc} without a cell.json; log tail: {tail[-400:]!r}")+ return CellResult(+ prompt_id=item.id,+ tier=item.tier,+ arm=arm,+ kind="harness",+ target=opts.generator,+ judge=opts.judge,+ workspace=str(cell),+ wall_s=round(time.time() - t0, 1),+ status="infra_failed" if is_infra_failure(tail) else "error",+ error=f"worker exited {rc} without a cell.json; log tail: {tail[-400:]!r}",+ )def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> int:@@ -269,10 +332,14 @@ def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptprint(f"no prompt {item_id!r} in {battery_path}", file=sys.stderr)return 2deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples))- res = run_cell(battery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps)+ res = run_cell(+ battery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps+ )res.arm, res.target = arm, opts.generator # the row is keyed by A/B arm, not by generator(Path(res.workspace) / "cell.json").write_text(res.model_dump_json(indent=1))- print(f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}")+ print(+ f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}"+ )return 0@@ -285,7 +352,9 @@ class Todo(NamedTuple):fresh: bool = False-def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[Todo]:+def _plan_todo(+ battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions+) -> list[Todo]:"""Which (prompt, arms) still need running.A prompt with EITHER arm in ``redo_status`` is re-run on BOTH arms — a redo exists@@ -334,7 +403,9 @@ def archive_cell(out: Path, arm: str, item_id: str, opts: AbOptions) -> Path | Nraise AssertionError("unreachable") # pragma: no cover — count() is infinite-def pin_pair(battery: Battery, item: BenchPrompt, out: Path, arms: Sequence[str], opts: AbOptions) -> str:+def pin_pair(+ battery: Battery, item: BenchPrompt, out: Path, arms: Sequence[str], opts: AbOptions+) -> str:"""Plan ``item`` ONCE and seed that plan into every arm about to run; return its hash.The A/A of this rig measured paired sd 0.202 on the judged score and traced it to the@@ -360,32 +431,85 @@ def pin_pair(battery: Battery, item: BenchPrompt, out: Path, arms: Sequence[str]dst.write_json(dst.spec_path, spec) # _run_harness only writes it when it creates the wsgot = seed_plan(src.root, dst.root)if got != want: # pragma: no cover — same spec both sides; the assert is the point- raise PinError(f"{item.id}/{arm}: seeded plan hash {got} != {want}; the pair would be unpinned")+ raise PinError(+ f"{item.id}/{arm}: seeded plan hash {got} != {want}; the pair would be unpinned"+ )return want-def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, run_cell_fn: CellRunner = spawn_cell,- on_result: Callable[[CellResult], None] | None = None) -> Verdict:+def run_ab(+ battery_path: Path | str,+ out_dir: Path | str,+ opts: AbOptions,+ *,+ run_cell_fn: CellRunner = spawn_cell,+ on_result: Callable[[CellResult], None] | None = None,+) -> Verdict:"""Run (or resume) the A/B, one prompt-pair at a time; write pairs.json + summary.md."""battery_path, out = Path(battery_path), Path(out_dir)battery = Battery.load(battery_path)out.mkdir(parents=True, exist_ok=True)- (out / "ab.json").write_text(json.dumps({"battery": battery.name, "prompts": str(battery_path),- "aa": opts.aa,- "arms": {CONTROL: {}, VARIANT: {} if opts.aa else opts.variant_env},- "options": opts.model_dump(mode="json"),- "started_at": datetime.now(UTC).isoformat()}, indent=2))+ (out / "ab.json").write_text(+ json.dumps(+ {+ "battery": battery.name,+ "prompts": str(battery_path),+ "aa": opts.aa,+ "arms": {CONTROL: {}, VARIANT: {} if opts.aa else opts.variant_env},+ "options": opts.model_dump(mode="json"),+ "started_at": datetime.now(UTC).isoformat(),+ },+ indent=2,+ )+ )results = out / "results.jsonl"done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}todo = _plan_todo(battery, done, opts)seal_for_append(results) # a kill left the last row unterminated; do not glue onto itwith ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:- for item, arms, fresh in todo: # one pair at a time: both arms of a prompt see the same weather+ for (+ item,+ arms,+ fresh,+ ) in todo: # one pair at a time: both arms of a prompt see the same weatherif fresh:for a in arms:archive_cell(out, a, item.id, opts)if opts.pin_plan:- pin_pair(battery, item, out, arms, opts)+ try:+ pin_pair(battery, item, out, arms, opts)+ except Exception as e: # noqa: BLE001 — one prompt's planner outage must not end the battery+ # The pinned plan is ONE model call with a 15-minute retry budget+ # (models/retry.py RETRY_DEADLINE_S). When a 503 storm outlasts it the call+ # raises, and this used to propagate straight out of the loop: measured+ # 2026-08-26, a driver of 3 prompts died on its first pin with 2 prompts+ # never attempted, while the drivers beside it waited the storm out. The+ # pair is recorded the way a dead cell is (docs/EVAL.md §7 — an outage is+ # not a score) so `--redo-status infra_failed` picks it up, and the loop+ # goes on to the next prompt.+ status = "infra_failed" if is_infra_failure(str(e)) else "error"+ log.warning(+ "%s: pinned plan failed (%s) — pair recorded %s, continuing",+ item.id,+ e,+ status,+ )+ for a in arms:+ r = CellResult(+ prompt_id=item.id,+ tier=item.tier,+ arm=a,+ kind="harness",+ target=opts.generator,+ status=status,+ error=f"pinned plan: {type(e).__name__}: {e}"[:600],+ )+ done[(r.prompt_id, r.arm)] = r+ fh.write(r.model_dump_json() + "\n")+ fh.flush()+ if on_result:+ on_result(r)+ continuefuts = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]for fut in futs:r = fut.result()@@ -395,14 +519,27 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, ruif on_result:on_result(r)order = [(p.id, p.tier) for p in battery.prompts]- return write_report(out, list(done.values()), order, title=f"{battery.name} / {out.name}",- variant_env={} if opts.aa else opts.variant_env, generator=opts.generator,- judge=opts.judge, rounds=opts.rounds, aa=opts.aa)+ return write_report(+ out,+ list(done.values()),+ order,+ title=f"{battery.name} / {out.name}",+ variant_env={} if opts.aa else opts.variant_env,+ generator=opts.generator,+ judge=opts.judge,+ rounds=opts.rounds,+ aa=opts.aa,+ )def preflight(opts: AbOptions, *, wait_minutes: float) -> bool:"""Probe every model either arm needs (planner, generator, loop judge, fixed judge)."""- return _preflight(opts.judge, [_harness_arm(opts.generator)], opts.compare_options(), wait_minutes=wait_minutes)+ return _preflight(+ opts.judge,+ [_harness_arm(opts.generator)],+ opts.compare_options(),+ wait_minutes=wait_minutes,+ )# ----------------------------------------------------------------------------- CLI@@ -417,8 +554,13 @@ def _parser() -> argparse.ArgumentParser:w.add_argument("--options-json", required=True)ap.add_argument("--prompts")ap.add_argument("--out")- ap.add_argument("--variant-env", action="append", default=[], metavar="KEY=VALUE",- help="env applied to the VARIANT arm only (repeatable)")+ ap.add_argument(+ "--variant-env",+ action="append",+ default=[],+ metavar="KEY=VALUE",+ help="env applied to the VARIANT arm only (repeatable)",+ )ap.add_argument("--generator", default=DEFAULT_GENERATOR)ap.add_argument("--judge", default=DEFAULT_JUDGE, help="FIXED judge model id (both arms)")ap.add_argument("--loop-judge", default=None)@@ -430,25 +572,51 @@ def _parser() -> argparse.ArgumentParser:ap.add_argument("--ids", default="")ap.add_argument("--tiers", default="")ap.add_argument("--limit", type=int, default=None)- ap.add_argument("--max-in-flight", type=int, default=None,- help=f"per child; wins over an inherited {MAX_IN_FLIGHT_ENV} "- f"(default: that variable, else {DEFAULT_MAX_IN_FLIGHT}). The preflight reserves 2x it")- ap.add_argument("--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)")- ap.add_argument("--redo-status", default="", help="comma list, e.g. infra_failed (re-runs both arms of the prompt)")- ap.add_argument("--redo-resume", action="store_true",- help="let a redo RESUME the old cells instead of regenerating them (cheap, but the pair then "- "spans two weathers — see archive_cell)")- ap.add_argument("--pin-plan", action="store_true",- help="plan once per prompt and seed BOTH arms with it, removing the planner's "- "spread from the pairing; refused for a plan-side --variant-env")- ap.add_argument("--aa", action="store_true",- help="calibration run: both arms identical, so the measured delta IS this rig's noise floor")+ ap.add_argument(+ "--max-in-flight",+ type=int,+ default=None,+ help=f"per child; wins over an inherited {MAX_IN_FLIGHT_ENV} "+ f"(default: that variable, else {DEFAULT_MAX_IN_FLIGHT}). The preflight reserves 2x it",+ )+ ap.add_argument(+ "--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)"+ )+ ap.add_argument(+ "--redo-status",+ default="",+ help="comma list, e.g. infra_failed (re-runs both arms of the prompt)",+ )+ ap.add_argument(+ "--redo-resume",+ action="store_true",+ help="let a redo RESUME the old cells instead of regenerating them (cheap, but the pair then "+ "spans two weathers — see archive_cell)",+ )+ ap.add_argument(+ "--pin-plan",+ action="store_true",+ help="plan once per prompt and seed BOTH arms with it, removing the planner's "+ "spread from the pairing; refused for a plan-side --variant-env",+ )+ ap.add_argument(+ "--aa",+ action="store_true",+ help="calibration run: both arms identical, so the measured delta IS this rig's noise floor",+ )ap.add_argument("--no-resume", action="store_true")- ap.add_argument("--no-preflight", action="store_true", help="skip the provider health check (do not)")+ ap.add_argument(+ "--no-preflight", action="store_true", help="skip the provider health check (do not)"+ )ap.add_argument("--wait-for-provider", type=float, default=0.0, metavar="MIN")- ap.add_argument("--allow-siblings", action="store_true",- help="launch even when other harness processes are running (they share the quota)")- ap.add_argument("--report-only", action="store_true", help="only rebuild pairs.json / summary.md")+ ap.add_argument(+ "--allow-siblings",+ action="store_true",+ help="launch even when other harness processes are running (they share the quota)",+ )+ ap.add_argument(+ "--report-only", action="store_true", help="only rebuild pairs.json / summary.md"+ )return ap@@ -466,18 +634,41 @@ def _stored_options(out: Path) -> AbOptions | None:def main(argv: Sequence[str] | None = None) -> int:ns = _parser().parse_args(argv)if ns.cmd == "cell":- return cell_main(Path(ns.prompts), Path(ns.out), ns.id, ns.arm, AbOptions.model_validate_json(ns.options_json))+ return cell_main(+ Path(ns.prompts),+ Path(ns.out),+ ns.id,+ ns.arm,+ AbOptions.model_validate_json(ns.options_json),+ )if not ns.prompts or not ns.out:_parser().error("--prompts and --out are required")if ns.parallel != PARALLEL:- _parser().error(f"--parallel must be {PARALLEL}: the arms of a prompt run together and nothing else does")- opts = AbOptions(generator=ns.generator, judge=ns.judge, loop_judge=ns.loop_judge, planner=ns.planner,- rounds=ns.rounds, max_usd=ns.max_usd, max_minutes=ns.max_minutes, n_samples=ns.n_samples,- ids=[i for i in ns.ids.split(",") if i], tiers=[t for t in ns.tiers.split(",") if t],- limit=ns.limit, resume=not ns.no_resume, redo_status=[s for s in ns.redo_status.split(",") if s],- variant_env=parse_variant_env(ns.variant_env),- max_in_flight=ns.max_in_flight if ns.max_in_flight is not None else inherited_max_in_flight(),- aa=ns.aa, redo_fresh=not ns.redo_resume, pin_plan=ns.pin_plan)+ _parser().error(+ f"--parallel must be {PARALLEL}: the arms of a prompt run together and nothing else does"+ )+ opts = AbOptions(+ generator=ns.generator,+ judge=ns.judge,+ loop_judge=ns.loop_judge,+ planner=ns.planner,+ rounds=ns.rounds,+ max_usd=ns.max_usd,+ max_minutes=ns.max_minutes,+ n_samples=ns.n_samples,+ ids=[i for i in ns.ids.split(",") if i],+ tiers=[t for t in ns.tiers.split(",") if t],+ limit=ns.limit,+ resume=not ns.no_resume,+ redo_status=[s for s in ns.redo_status.split(",") if s],+ variant_env=parse_variant_env(ns.variant_env),+ max_in_flight=ns.max_in_flight+ if ns.max_in_flight is not None+ else inherited_max_in_flight(),+ aa=ns.aa,+ redo_fresh=not ns.redo_resume,+ pin_plan=ns.pin_plan,+ )out = Path(ns.out)if ns.report_only:battery = Battery.load(ns.prompts)@@ -485,15 +676,28 @@ def main(argv: Sequence[str] | None = None) -> int:# typed with no --variant-env, and reporting the arms as identical would relabel a# real A/B as an A/A in the file everyone reads afterwards.opts = _stored_options(out) or opts- v = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],- title=f"{battery.name} / {out.name}", variant_env={} if opts.aa else opts.variant_env,- generator=opts.generator, judge=opts.judge, rounds=opts.rounds, aa=opts.aa)- print(f"verdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")- + f"\n{out / 'summary.md'}")+ v = write_report(+ out,+ load_jsonl(out / "results.jsonl", CellResult),+ [(p.id, p.tier) for p in battery.prompts],+ title=f"{battery.name} / {out.name}",+ variant_env={} if opts.aa else opts.variant_env,+ generator=opts.generator,+ judge=opts.judge,+ rounds=opts.rounds,+ aa=opts.aa,+ )+ print(+ f"verdict: {v.decision} — {v.reason}"+ + (f"\nCAUTION: {v.caution}" if v.caution else "")+ + f"\n{out / 'summary.md'}"+ )return 0if not opts.variant_env and not opts.aa:- _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the noise "- "(pass --aa if measuring the noise is the point)")+ _parser().error(+ "--variant-env KEY=VALUE is required: an A/B with identical arms measures only the noise "+ "(pass --aa if measuring the noise is the point)"+ )if not opts.aa:# A KEY no code path reads makes the variant arm byte-identical to the control, so# the battery costs a full run and yields a verdict about nothing. One such A/B is@@ -505,42 +709,73 @@ def main(argv: Sequence[str] | None = None) -> int:_parser().error("--variant-env only sets switches nothing reads, so both arms would be identical: "+ "; ".join(f"{k} ({DEAD_SWITCHES[k]})" for k in dead)- + " (pass --aa if an identical-arms calibration run is the point)")+ + " (pass --aa if an identical-arms calibration run is the point)"+ )if opts.pin_plan and (blockers := pin_plan_blockers(opts.variant_env)):_parser().error("--pin-plan is refused for a PLAN-side switch: sharing one plan across the arms would delete the very ""thing under test and the rig would report 'no effect' with confidence — "- + "; ".join(blockers))+ + "; ".join(blockers)+ )from codeverse.models.health import pool_budget# the rule is a BUDGET, not a head-count: the provider sees one machine, so the sum of# every process's in-flight cap must stay at the knee. Two children run at once here.need = 2 * opts.max_in_flightif not (pb := pool_budget()).fits(need) and not ns.allow_siblings:- print(f"refusing to start: {pb}; this A/B needs {need} (2 children x {opts.max_in_flight}) "- f"(docs/COST.md §23). Wait, lower --max-in-flight, or pass --allow-siblings.", flush=True)+ print(+ f"refusing to start: {pb}; this A/B needs {need} (2 children x {opts.max_in_flight}) "+ f"(docs/COST.md §23). Wait, lower --max-in-flight, or pass --allow-siblings.",+ flush=True,+ )return 3if not ns.no_preflight and not preflight(opts, wait_minutes=ns.wait_for_provider):return 2def _log(r: CellResult) -> None:- print(f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:8s} score={r.score} "- f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}", flush=True)+ print(+ f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:8s} score={r.score} "+ f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}",+ flush=True,+ )v = run_ab(ns.prompts, out, opts, on_result=_log)- print(f"\nverdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")- + f"\n{out / 'summary.md'}")+ print(+ f"\nverdict: {v.decision} — {v.reason}"+ + (f"\nCAUTION: {v.caution}" if v.caution else "")+ + f"\n{out / 'summary.md'}"+ )lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]if lost:- print(f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "- f"those prompts with: --redo-status infra_failed")+ print(+ f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "+ f"those prompts with: --redo-status infra_failed"+ )return 0-__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "NESTED_MAX_IN_FLIGHT_ENV", "PARALLEL",- "AbOptions", "CellRunner", "Todo", "archive_cell", "cell_dir", "cell_main", "child_env",- "inherited_max_in_flight", "main", "parse_variant_env", "pin_pair", "preflight", "run_ab", "spawn_cell",- "worker_argv"]+__all__ = [+ "DEFAULT_GENERATOR",+ "DEFAULT_JUDGE",+ "MAX_IN_FLIGHT_ENV",+ "NESTED_MAX_IN_FLIGHT_ENV",+ "PARALLEL",+ "AbOptions",+ "CellRunner",+ "Todo",+ "archive_cell",+ "cell_dir",+ "cell_main",+ "child_env",+ "inherited_max_in_flight",+ "main",+ "parse_variant_env",+ "pin_pair",+ "preflight",+ "run_ab",+ "spawn_cell",+ "worker_argv",+]if __name__ == "__main__":raise SystemExit(main())diff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.pyindex 5145858a..0713ece9 100644--- a/harness/tests/compare_bench/test_ab_plan.py+++ b/harness/tests/compare_bench/test_ab_plan.py@@ -456,3 +456,34 @@ def test_report_only_survives_a_run_dir_with_no_or_broken_ab_json(tmp_path):assert _stored_options(tmp_path) is None(tmp_path / "ab.json").write_text('{"no options key": 1}')assert _stored_options(tmp_path) is None+++def test_a_pinned_plan_that_dies_in_a_storm_records_the_pair_and_continues(tmp_path: Path, monkeypatch):+ """The pinned plan is ONE model call with a 15-minute retry budget. When a 503 storm+ outlasts it the call raises — and that used to propagate out of run_ab's loop and end+ the whole battery. Measured 2026-08-26: a 3-prompt driver died on its first pin, two+ prompts never attempted, while the drivers beside it waited the storm out. An outage is+ not a score (docs/EVAL.md §7): the pair is recorded infra_failed so --redo-status+ infra_failed picks it up, and the loop goes on."""+ from bench import ab_plan as ab+ from codeverse.models.base import ModelError++ b = Battery.load(BATTERY)+ ids = [p.id for p in b.prompts[:2]]+ fake = FakeCells({(ids[1], VARIANT): 0.6})+ dead = {ids[0]}++ def stormy_pin(battery, item, out, arms, opts):+ if item.id in dead:+ raise ModelError("Gemini API error 503: This model is currently experiencing high demand.")+ return "h1"++ monkeypatch.setattr(ab, "pin_pair", stormy_pin)+ opts = AbOptions(variant_env={"CV3D_SKILLS": "1"}, pin_plan=True, ids=ids)+ run_ab(BATTERY, tmp_path, opts, run_cell_fn=fake)++ rows = load_jsonl(tmp_path / "results.jsonl", CellResult)+ first = [r for r in rows if r.prompt_id == ids[0]]+ assert {r.arm for r in first} == {CONTROL, VARIANT} and all(r.status == "infra_failed" for r in first)+ assert all("pinned plan" in r.error for r in first)+ assert [c[0] for c in fake.calls] == [ids[1], ids[1]], "no cell was spent on the dead pair; the next prompt ran"