Skip to content

Official Codebase of 3DCodeVerse

Python

ab_plan: a pinned plan lost to a storm records the pair, not the battery

gaoypeng193a69467060
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.py
index 0f8dd6f3..5a342364 100644
--- a/harness/bench/ab_plan.py
+++ b/harness/bench/ab_plan.py
@@ -44,6 +44,7 @@ from __future__ import annotations
import argparse
import json
+import logging
import os
import subprocess
import sys
@@ -87,7 +88,8 @@ def _assert_local_codeverse() -> None:
f"refusing to run: `import codeverse` resolved to {got}, not {want}.\n"
f" An editable install is shadowing this tree, so both arms would run the same\n"
f" code and the A/B would measure nothing. Launch with:\n"
- f" PYTHONPATH={REPO} python bench/ab_plan.py ...")
+ f" PYTHONPATH={REPO} python bench/ab_plan.py ..."
+ )
_assert_local_codeverse()
@@ -109,6 +111,8 @@ from bench.run_bench import Battery, BenchPrompt, select_prompts # noqa: E402
from codeverse.tracks.plan_features import pin_plan_blockers # noqa: E402
from codeverse.workspace import Workspace # noqa: E402
+log = logging.getLogger(__name__)
+
DEFAULT_GENERATOR = "api-agent:gemini:gemini-3.7-flash"
DEFAULT_JUDGE = "gemini:gemini-3.1-pro-preview"
#: the per-child cap. The flat name is a first-class alias of ``CV3D_RATE__MAX_IN_FLIGHT``
@@ -125,8 +129,12 @@ PARALLEL = 2
class AbOptions(BaseModel):
generator: str = DEFAULT_GENERATOR
judge: str = DEFAULT_JUDGE
- loop_judge: str | None = Field(default=None, description="in-loop judge for BOTH arms (None → settings default)")
- planner: str | None = Field(default=None, description="planner for BOTH arms (None → settings default)")
+ loop_judge: str | None = Field(
+ default=None, description="in-loop judge for BOTH arms (None → settings default)"
+ )
+ planner: str | None = Field(
+ default=None, description="planner for BOTH arms (None → settings default)"
+ )
rounds: int = 2
max_usd: float = 2.5
max_minutes: float = 45.0
@@ -135,19 +143,39 @@ class AbOptions(BaseModel):
tiers: list[str] = Field(default_factory=list)
limit: int | None = None
resume: bool = True
- redo_status: list[str] = Field(default_factory=list, description="re-run prompts where EITHER arm has one of these")
- variant_env: dict[str, str] = Field(default_factory=dict, description="applied to the variant arm only")
+ redo_status: list[str] = Field(
+ default_factory=list, description="re-run prompts where EITHER arm has one of these"
+ )
+ variant_env: dict[str, str] = Field(
+ default_factory=dict, description="applied to the variant arm only"
+ )
max_in_flight: int = DEFAULT_MAX_IN_FLIGHT
- aa: bool = Field(default=False, description="calibration: run BOTH arms as the control, so the delta IS the noise")
- redo_fresh: bool = Field(default=True, description="a redo regenerates both arms instead of resuming them")
- pin_plan: bool = Field(default=False, description="plan ONCE per prompt and seed both arms with it "
- "(generation-side switches only: see pin_plan_blockers)")
+ aa: bool = Field(
+ default=False,
+ description="calibration: run BOTH arms as the control, so the delta IS the noise",
+ )
+ redo_fresh: bool = Field(
+ default=True, description="a redo regenerates both arms instead of resuming them"
+ )
+ pin_plan: bool = Field(
+ default=False,
+ description="plan ONCE per prompt and seed both arms with it "
+ "(generation-side switches only: see pin_plan_blockers)",
+ )
def compare_options(self) -> CompareOptions:
"""The per-cell options handed to ``compare_backends.run_cell`` (both arms identical)."""
- return CompareOptions(judge=self.judge, loop_judge=self.loop_judge, planner=self.planner, rounds=self.rounds,
- max_usd=self.max_usd, max_minutes=self.max_minutes, n_samples=self.n_samples,
- resume=self.resume, pairwise=False)
+ return CompareOptions(
+ judge=self.judge,
+ loop_judge=self.loop_judge,
+ planner=self.planner,
+ rounds=self.rounds,
+ max_usd=self.max_usd,
+ max_minutes=self.max_minutes,
+ n_samples=self.n_samples,
+ resume=self.resume,
+ pairwise=False,
+ )
def inherited_max_in_flight() -> int:
@@ -174,8 +202,10 @@ def parse_variant_env(items: Sequence[str]) -> dict[str, str]:
if k.strip() in (MAX_IN_FLIGHT_ENV, NESTED_MAX_IN_FLIGHT_ENV):
# child_env sets the cap authoritatively so the §23 admission check and the
# children agree; accepting it here too would silently discard one of them.
- raise ValueError(f"--variant-env {k.strip()} is not allowed: the in-flight cap is "
- f"the driver's (--max-in-flight), and the admission check budgets on it")
+ raise ValueError(
+ f"--variant-env {k.strip()} is not allowed: the in-flight cap is "
+ f"the driver's (--max-in-flight), and the admission check budgets on it"
+ )
out[k.strip()] = v
return out
@@ -221,16 +251,33 @@ def _harness_arm(generator: str) -> Arm:
return Arm(raw=f"harness:{generator}", kind="harness", target=generator)
-def worker_argv(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> list[str]:
- return [sys.executable, str(Path(__file__).resolve()), "cell", "--prompts", str(battery_path), "--out", str(out),
- "--id", item_id, "--arm", arm, "--options-json", opts.model_dump_json()]
+def worker_argv(
+ battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions
+) -> list[str]:
+ return [
+ sys.executable,
+ str(Path(__file__).resolve()),
+ "cell",
+ "--prompts",
+ str(battery_path),
+ "--out",
+ str(out),
+ "--id",
+ item_id,
+ "--arm",
+ arm,
+ "--options-json",
+ opts.model_dump_json(),
+ ]
# ----------------------------------------------------------------------------- one cell, in a child
CellRunner = Callable[[Path, Path, BenchPrompt, str, AbOptions], CellResult]
-def spawn_cell(battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts: AbOptions) -> CellResult:
+def spawn_cell(
+ battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts: AbOptions
+) -> CellResult:
"""Run one (prompt, arm) cell in a child process and read back its ``cell.json``.
A child that dies without writing a cell (killed, import error, unhandled outage)
@@ -240,23 +287,39 @@ def spawn_cell(battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts:
cell = cell_dir(out, arm, item.id, opts.generator)
cell.mkdir(parents=True, exist_ok=True)
log, marker = cell / "worker.log", cell / "cell.json"
- marker.unlink(missing_ok=True) # a stale verdict from an earlier attempt must not stand in for this one
+ marker.unlink(
+ missing_ok=True
+ ) # a stale verdict from an earlier attempt must not stand in for this one
t0 = time.time()
with log.open("a") as fh:
fh.write(f"--- {datetime.now(UTC).isoformat()} {arm} {item.id}\n")
fh.flush()
- rc = subprocess.run(worker_argv(battery_path, out, item.id, arm, opts), cwd=REPO, env=child_env(arm, opts),
- stdout=fh, stderr=subprocess.STDOUT, check=False).returncode
+ rc = subprocess.run(
+ worker_argv(battery_path, out, item.id, arm, opts),
+ cwd=REPO,
+ env=child_env(arm, opts),
+ stdout=fh,
+ stderr=subprocess.STDOUT,
+ check=False,
+ ).returncode
if marker.is_file():
return CellResult.model_validate_json(marker.read_text())
# errors="replace": the child's raw stdout lands in this log, and one non-UTF-8 byte
# from an agent CLI used to raise UnicodeDecodeError *here* — on the very path whose
# job is to classify an outage — killing the whole driver through fut.result().
tail = log.read_text(errors="replace")[-1500:] if log.is_file() else ""
- return CellResult(prompt_id=item.id, tier=item.tier, arm=arm, kind="harness", target=opts.generator,
- judge=opts.judge, workspace=str(cell), wall_s=round(time.time() - t0, 1),
- status="infra_failed" if is_infra_failure(tail) else "error",
- error=f"worker exited {rc} without a cell.json; log tail: {tail[-400:]!r}")
+ return CellResult(
+ prompt_id=item.id,
+ tier=item.tier,
+ arm=arm,
+ kind="harness",
+ target=opts.generator,
+ judge=opts.judge,
+ workspace=str(cell),
+ wall_s=round(time.time() - t0, 1),
+ status="infra_failed" if is_infra_failure(tail) else "error",
+ error=f"worker exited {rc} without a cell.json; log tail: {tail[-400:]!r}",
+ )
def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> int:
@@ -269,10 +332,14 @@ def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOpt
print(f"no prompt {item_id!r} in {battery_path}", file=sys.stderr)
return 2
deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples))
- res = run_cell(battery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps)
+ res = run_cell(
+ battery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps
+ )
res.arm, res.target = arm, opts.generator # the row is keyed by A/B arm, not by generator
(Path(res.workspace) / "cell.json").write_text(res.model_dump_json(indent=1))
- print(f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}")
+ print(
+ f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}"
+ )
return 0
@@ -285,7 +352,9 @@ class Todo(NamedTuple):
fresh: bool = False
-def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[Todo]:
+def _plan_todo(
+ battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions
+) -> list[Todo]:
"""Which (prompt, arms) still need running.
A prompt with EITHER arm in ``redo_status`` is re-run on BOTH arms — a redo exists
@@ -334,7 +403,9 @@ def archive_cell(out: Path, arm: str, item_id: str, opts: AbOptions) -> Path | N
raise AssertionError("unreachable") # pragma: no cover — count() is infinite
-def pin_pair(battery: Battery, item: BenchPrompt, out: Path, arms: Sequence[str], opts: AbOptions) -> str:
+def pin_pair(
+ battery: Battery, item: BenchPrompt, out: Path, arms: Sequence[str], opts: AbOptions
+) -> str:
"""Plan ``item`` ONCE and seed that plan into every arm about to run; return its hash.
The A/A of this rig measured paired sd 0.202 on the judged score and traced it to the
@@ -360,32 +431,85 @@ def pin_pair(battery: Battery, item: BenchPrompt, out: Path, arms: Sequence[str]
dst.write_json(dst.spec_path, spec) # _run_harness only writes it when it creates the ws
got = seed_plan(src.root, dst.root)
if got != want: # pragma: no cover — same spec both sides; the assert is the point
- raise PinError(f"{item.id}/{arm}: seeded plan hash {got} != {want}; the pair would be unpinned")
+ raise PinError(
+ f"{item.id}/{arm}: seeded plan hash {got} != {want}; the pair would be unpinned"
+ )
return want
-def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, run_cell_fn: CellRunner = spawn_cell,
- on_result: Callable[[CellResult], None] | None = None) -> Verdict:
+def run_ab(
+ battery_path: Path | str,
+ out_dir: Path | str,
+ opts: AbOptions,
+ *,
+ run_cell_fn: CellRunner = spawn_cell,
+ on_result: Callable[[CellResult], None] | None = None,
+) -> Verdict:
"""Run (or resume) the A/B, one prompt-pair at a time; write pairs.json + summary.md."""
battery_path, out = Path(battery_path), Path(out_dir)
battery = Battery.load(battery_path)
out.mkdir(parents=True, exist_ok=True)
- (out / "ab.json").write_text(json.dumps({"battery": battery.name, "prompts": str(battery_path),
- "aa": opts.aa,
- "arms": {CONTROL: {}, VARIANT: {} if opts.aa else opts.variant_env},
- "options": opts.model_dump(mode="json"),
- "started_at": datetime.now(UTC).isoformat()}, indent=2))
+ (out / "ab.json").write_text(
+ json.dumps(
+ {
+ "battery": battery.name,
+ "prompts": str(battery_path),
+ "aa": opts.aa,
+ "arms": {CONTROL: {}, VARIANT: {} if opts.aa else opts.variant_env},
+ "options": opts.model_dump(mode="json"),
+ "started_at": datetime.now(UTC).isoformat(),
+ },
+ indent=2,
+ )
+ )
results = out / "results.jsonl"
done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}
todo = _plan_todo(battery, done, opts)
seal_for_append(results) # a kill left the last row unterminated; do not glue onto it
with ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:
- for item, arms, fresh in todo: # one pair at a time: both arms of a prompt see the same weather
+ for (
+ item,
+ arms,
+ fresh,
+ ) in todo: # one pair at a time: both arms of a prompt see the same weather
if fresh:
for a in arms:
archive_cell(out, a, item.id, opts)
if opts.pin_plan:
- pin_pair(battery, item, out, arms, opts)
+ try:
+ pin_pair(battery, item, out, arms, opts)
+ except Exception as e: # noqa: BLE001 — one prompt's planner outage must not end the battery
+ # The pinned plan is ONE model call with a 15-minute retry budget
+ # (models/retry.py RETRY_DEADLINE_S). When a 503 storm outlasts it the call
+ # raises, and this used to propagate straight out of the loop: measured
+ # 2026-08-26, a driver of 3 prompts died on its first pin with 2 prompts
+ # never attempted, while the drivers beside it waited the storm out. The
+ # pair is recorded the way a dead cell is (docs/EVAL.md §7 — an outage is
+ # not a score) so `--redo-status infra_failed` picks it up, and the loop
+ # goes on to the next prompt.
+ status = "infra_failed" if is_infra_failure(str(e)) else "error"
+ log.warning(
+ "%s: pinned plan failed (%s) — pair recorded %s, continuing",
+ item.id,
+ e,
+ status,
+ )
+ for a in arms:
+ r = CellResult(
+ prompt_id=item.id,
+ tier=item.tier,
+ arm=a,
+ kind="harness",
+ target=opts.generator,
+ status=status,
+ error=f"pinned plan: {type(e).__name__}: {e}"[:600],
+ )
+ done[(r.prompt_id, r.arm)] = r
+ fh.write(r.model_dump_json() + "\n")
+ fh.flush()
+ if on_result:
+ on_result(r)
+ continue
futs = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]
for fut in futs:
r = fut.result()
@@ -395,14 +519,27 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, ru
if on_result:
on_result(r)
order = [(p.id, p.tier) for p in battery.prompts]
- return write_report(out, list(done.values()), order, title=f"{battery.name} / {out.name}",
- variant_env={} if opts.aa else opts.variant_env, generator=opts.generator,
- judge=opts.judge, rounds=opts.rounds, aa=opts.aa)
+ return write_report(
+ out,
+ list(done.values()),
+ order,
+ title=f"{battery.name} / {out.name}",
+ variant_env={} if opts.aa else opts.variant_env,
+ generator=opts.generator,
+ judge=opts.judge,
+ rounds=opts.rounds,
+ aa=opts.aa,
+ )
def preflight(opts: AbOptions, *, wait_minutes: float) -> bool:
"""Probe every model either arm needs (planner, generator, loop judge, fixed judge)."""
- return _preflight(opts.judge, [_harness_arm(opts.generator)], opts.compare_options(), wait_minutes=wait_minutes)
+ return _preflight(
+ opts.judge,
+ [_harness_arm(opts.generator)],
+ opts.compare_options(),
+ wait_minutes=wait_minutes,
+ )
# ----------------------------------------------------------------------------- CLI
@@ -417,8 +554,13 @@ def _parser() -> argparse.ArgumentParser:
w.add_argument("--options-json", required=True)
ap.add_argument("--prompts")
ap.add_argument("--out")
- ap.add_argument("--variant-env", action="append", default=[], metavar="KEY=VALUE",
- help="env applied to the VARIANT arm only (repeatable)")
+ ap.add_argument(
+ "--variant-env",
+ action="append",
+ default=[],
+ metavar="KEY=VALUE",
+ help="env applied to the VARIANT arm only (repeatable)",
+ )
ap.add_argument("--generator", default=DEFAULT_GENERATOR)
ap.add_argument("--judge", default=DEFAULT_JUDGE, help="FIXED judge model id (both arms)")
ap.add_argument("--loop-judge", default=None)
@@ -430,25 +572,51 @@ def _parser() -> argparse.ArgumentParser:
ap.add_argument("--ids", default="")
ap.add_argument("--tiers", default="")
ap.add_argument("--limit", type=int, default=None)
- ap.add_argument("--max-in-flight", type=int, default=None,
- help=f"per child; wins over an inherited {MAX_IN_FLIGHT_ENV} "
- f"(default: that variable, else {DEFAULT_MAX_IN_FLIGHT}). The preflight reserves 2x it")
- ap.add_argument("--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)")
- ap.add_argument("--redo-status", default="", help="comma list, e.g. infra_failed (re-runs both arms of the prompt)")
- ap.add_argument("--redo-resume", action="store_true",
- help="let a redo RESUME the old cells instead of regenerating them (cheap, but the pair then "
- "spans two weathers — see archive_cell)")
- ap.add_argument("--pin-plan", action="store_true",
- help="plan once per prompt and seed BOTH arms with it, removing the planner's "
- "spread from the pairing; refused for a plan-side --variant-env")
- ap.add_argument("--aa", action="store_true",
- help="calibration run: both arms identical, so the measured delta IS this rig's noise floor")
+ ap.add_argument(
+ "--max-in-flight",
+ type=int,
+ default=None,
+ help=f"per child; wins over an inherited {MAX_IN_FLIGHT_ENV} "
+ f"(default: that variable, else {DEFAULT_MAX_IN_FLIGHT}). The preflight reserves 2x it",
+ )
+ ap.add_argument(
+ "--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)"
+ )
+ ap.add_argument(
+ "--redo-status",
+ default="",
+ help="comma list, e.g. infra_failed (re-runs both arms of the prompt)",
+ )
+ ap.add_argument(
+ "--redo-resume",
+ action="store_true",
+ help="let a redo RESUME the old cells instead of regenerating them (cheap, but the pair then "
+ "spans two weathers — see archive_cell)",
+ )
+ ap.add_argument(
+ "--pin-plan",
+ action="store_true",
+ help="plan once per prompt and seed BOTH arms with it, removing the planner's "
+ "spread from the pairing; refused for a plan-side --variant-env",
+ )
+ ap.add_argument(
+ "--aa",
+ action="store_true",
+ help="calibration run: both arms identical, so the measured delta IS this rig's noise floor",
+ )
ap.add_argument("--no-resume", action="store_true")
- ap.add_argument("--no-preflight", action="store_true", help="skip the provider health check (do not)")
+ ap.add_argument(
+ "--no-preflight", action="store_true", help="skip the provider health check (do not)"
+ )
ap.add_argument("--wait-for-provider", type=float, default=0.0, metavar="MIN")
- ap.add_argument("--allow-siblings", action="store_true",
- help="launch even when other harness processes are running (they share the quota)")
- ap.add_argument("--report-only", action="store_true", help="only rebuild pairs.json / summary.md")
+ ap.add_argument(
+ "--allow-siblings",
+ action="store_true",
+ help="launch even when other harness processes are running (they share the quota)",
+ )
+ ap.add_argument(
+ "--report-only", action="store_true", help="only rebuild pairs.json / summary.md"
+ )
return ap
@@ -466,18 +634,41 @@ def _stored_options(out: Path) -> AbOptions | None:
def main(argv: Sequence[str] | None = None) -> int:
ns = _parser().parse_args(argv)
if ns.cmd == "cell":
- return cell_main(Path(ns.prompts), Path(ns.out), ns.id, ns.arm, AbOptions.model_validate_json(ns.options_json))
+ return cell_main(
+ Path(ns.prompts),
+ Path(ns.out),
+ ns.id,
+ ns.arm,
+ AbOptions.model_validate_json(ns.options_json),
+ )
if not ns.prompts or not ns.out:
_parser().error("--prompts and --out are required")
if ns.parallel != PARALLEL:
- _parser().error(f"--parallel must be {PARALLEL}: the arms of a prompt run together and nothing else does")
- opts = AbOptions(generator=ns.generator, judge=ns.judge, loop_judge=ns.loop_judge, planner=ns.planner,
- rounds=ns.rounds, max_usd=ns.max_usd, max_minutes=ns.max_minutes, n_samples=ns.n_samples,
- ids=[i for i in ns.ids.split(",") if i], tiers=[t for t in ns.tiers.split(",") if t],
- limit=ns.limit, resume=not ns.no_resume, redo_status=[s for s in ns.redo_status.split(",") if s],
- variant_env=parse_variant_env(ns.variant_env),
- max_in_flight=ns.max_in_flight if ns.max_in_flight is not None else inherited_max_in_flight(),
- aa=ns.aa, redo_fresh=not ns.redo_resume, pin_plan=ns.pin_plan)
+ _parser().error(
+ f"--parallel must be {PARALLEL}: the arms of a prompt run together and nothing else does"
+ )
+ opts = AbOptions(
+ generator=ns.generator,
+ judge=ns.judge,
+ loop_judge=ns.loop_judge,
+ planner=ns.planner,
+ rounds=ns.rounds,
+ max_usd=ns.max_usd,
+ max_minutes=ns.max_minutes,
+ n_samples=ns.n_samples,
+ ids=[i for i in ns.ids.split(",") if i],
+ tiers=[t for t in ns.tiers.split(",") if t],
+ limit=ns.limit,
+ resume=not ns.no_resume,
+ redo_status=[s for s in ns.redo_status.split(",") if s],
+ variant_env=parse_variant_env(ns.variant_env),
+ max_in_flight=ns.max_in_flight
+ if ns.max_in_flight is not None
+ else inherited_max_in_flight(),
+ aa=ns.aa,
+ redo_fresh=not ns.redo_resume,
+ pin_plan=ns.pin_plan,
+ )
out = Path(ns.out)
if ns.report_only:
battery = Battery.load(ns.prompts)
@@ -485,15 +676,28 @@ def main(argv: Sequence[str] | None = None) -> int:
# typed with no --variant-env, and reporting the arms as identical would relabel a
# real A/B as an A/A in the file everyone reads afterwards.
opts = _stored_options(out) or opts
- v = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],
- title=f"{battery.name} / {out.name}", variant_env={} if opts.aa else opts.variant_env,
- generator=opts.generator, judge=opts.judge, rounds=opts.rounds, aa=opts.aa)
- print(f"verdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")
- + f"\n{out / 'summary.md'}")
+ v = write_report(
+ out,
+ load_jsonl(out / "results.jsonl", CellResult),
+ [(p.id, p.tier) for p in battery.prompts],
+ title=f"{battery.name} / {out.name}",
+ variant_env={} if opts.aa else opts.variant_env,
+ generator=opts.generator,
+ judge=opts.judge,
+ rounds=opts.rounds,
+ aa=opts.aa,
+ )
+ print(
+ f"verdict: {v.decision} — {v.reason}"
+ + (f"\nCAUTION: {v.caution}" if v.caution else "")
+ + f"\n{out / 'summary.md'}"
+ )
return 0
if not opts.variant_env and not opts.aa:
- _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the noise "
- "(pass --aa if measuring the noise is the point)")
+ _parser().error(
+ "--variant-env KEY=VALUE is required: an A/B with identical arms measures only the noise "
+ "(pass --aa if measuring the noise is the point)"
+ )
if not opts.aa:
# A KEY no code path reads makes the variant arm byte-identical to the control, so
# the battery costs a full run and yields a verdict about nothing. One such A/B is
@@ -505,42 +709,73 @@ def main(argv: Sequence[str] | None = None) -> int:
_parser().error(
"--variant-env only sets switches nothing reads, so both arms would be identical: "
+ "; ".join(f"{k} ({DEAD_SWITCHES[k]})" for k in dead)
- + " (pass --aa if an identical-arms calibration run is the point)")
+ + " (pass --aa if an identical-arms calibration run is the point)"
+ )
if opts.pin_plan and (blockers := pin_plan_blockers(opts.variant_env)):
_parser().error(
"--pin-plan is refused for a PLAN-side switch: sharing one plan across the arms would delete the very "
"thing under test and the rig would report 'no effect' with confidence — "
- + "; ".join(blockers))
+ + "; ".join(blockers)
+ )
from codeverse.models.health import pool_budget
# the rule is a BUDGET, not a head-count: the provider sees one machine, so the sum of
# every process's in-flight cap must stay at the knee. Two children run at once here.
need = 2 * opts.max_in_flight
if not (pb := pool_budget()).fits(need) and not ns.allow_siblings:
- print(f"refusing to start: {pb}; this A/B needs {need} (2 children x {opts.max_in_flight}) "
- f"(docs/COST.md §23). Wait, lower --max-in-flight, or pass --allow-siblings.", flush=True)
+ print(
+ f"refusing to start: {pb}; this A/B needs {need} (2 children x {opts.max_in_flight}) "
+ f"(docs/COST.md §23). Wait, lower --max-in-flight, or pass --allow-siblings.",
+ flush=True,
+ )
return 3
if not ns.no_preflight and not preflight(opts, wait_minutes=ns.wait_for_provider):
return 2
def _log(r: CellResult) -> None:
- print(f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:8s} score={r.score} "
- f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}", flush=True)
+ print(
+ f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:8s} score={r.score} "
+ f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}",
+ flush=True,
+ )
v = run_ab(ns.prompts, out, opts, on_result=_log)
- print(f"\nverdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")
- + f"\n{out / 'summary.md'}")
+ print(
+ f"\nverdict: {v.decision} — {v.reason}"
+ + (f"\nCAUTION: {v.caution}" if v.caution else "")
+ + f"\n{out / 'summary.md'}"
+ )
lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]
if lost:
- print(f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "
- f"those prompts with: --redo-status infra_failed")
+ print(
+ f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "
+ f"those prompts with: --redo-status infra_failed"
+ )
return 0
-__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "NESTED_MAX_IN_FLIGHT_ENV", "PARALLEL",
- "AbOptions", "CellRunner", "Todo", "archive_cell", "cell_dir", "cell_main", "child_env",
- "inherited_max_in_flight", "main", "parse_variant_env", "pin_pair", "preflight", "run_ab", "spawn_cell",
- "worker_argv"]
+__all__ = [
+ "DEFAULT_GENERATOR",
+ "DEFAULT_JUDGE",
+ "MAX_IN_FLIGHT_ENV",
+ "NESTED_MAX_IN_FLIGHT_ENV",
+ "PARALLEL",
+ "AbOptions",
+ "CellRunner",
+ "Todo",
+ "archive_cell",
+ "cell_dir",
+ "cell_main",
+ "child_env",
+ "inherited_max_in_flight",
+ "main",
+ "parse_variant_env",
+ "pin_pair",
+ "preflight",
+ "run_ab",
+ "spawn_cell",
+ "worker_argv",
+]
if __name__ == "__main__":
raise SystemExit(main())
diff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.py
index 5145858a..0713ece9 100644
--- a/harness/tests/compare_bench/test_ab_plan.py
+++ b/harness/tests/compare_bench/test_ab_plan.py
@@ -456,3 +456,34 @@ def test_report_only_survives_a_run_dir_with_no_or_broken_ab_json(tmp_path):
assert _stored_options(tmp_path) is None
(tmp_path / "ab.json").write_text('{"no options key": 1}')
assert _stored_options(tmp_path) is None
+
+
+def test_a_pinned_plan_that_dies_in_a_storm_records_the_pair_and_continues(tmp_path: Path, monkeypatch):
+ """The pinned plan is ONE model call with a 15-minute retry budget. When a 503 storm
+ outlasts it the call raises — and that used to propagate out of run_ab's loop and end
+ the whole battery. Measured 2026-08-26: a 3-prompt driver died on its first pin, two
+ prompts never attempted, while the drivers beside it waited the storm out. An outage is
+ not a score (docs/EVAL.md §7): the pair is recorded infra_failed so --redo-status
+ infra_failed picks it up, and the loop goes on."""
+ from bench import ab_plan as ab
+ from codeverse.models.base import ModelError
+
+ b = Battery.load(BATTERY)
+ ids = [p.id for p in b.prompts[:2]]
+ fake = FakeCells({(ids[1], VARIANT): 0.6})
+ dead = {ids[0]}
+
+ def stormy_pin(battery, item, out, arms, opts):
+ if item.id in dead:
+ raise ModelError("Gemini API error 503: This model is currently experiencing high demand.")
+ return "h1"
+
+ monkeypatch.setattr(ab, "pin_pair", stormy_pin)
+ opts = AbOptions(variant_env={"CV3D_SKILLS": "1"}, pin_plan=True, ids=ids)
+ run_ab(BATTERY, tmp_path, opts, run_cell_fn=fake)
+
+ rows = load_jsonl(tmp_path / "results.jsonl", CellResult)
+ first = [r for r in rows if r.prompt_id == ids[0]]
+ assert {r.arm for r in first} == {CONTROL, VARIANT} and all(r.status == "infra_failed" for r in first)
+ assert all("pinned plan" in r.error for r in first)
+ assert [c[0] for c in fake.calls] == [ids[1], ids[1]], "no cell was spent on the dead pair; the next prompt ran"