Skip to content

Official Codebase of 3DCodeVerse

Python

bench: ab_plan.py — a paired control/variant A/B for plan and brief switches

gaoypeng68b763071b38
diff --git a/harness/bench/_ab_report.py b/harness/bench/_ab_report.py
new file mode 100644
index 00000000..fa6b4876
--- /dev/null
+++ b/harness/bench/_ab_report.py
@@ -0,0 +1,201 @@
+"""Pairing, verdict and ``summary.md`` for ``bench/ab_plan.py``.
+
+An A/B of a plan/brief change is decided on PAIRED deltas, one per prompt, because
+the two arms of a prompt ran side by side in the same weather and against the same
+fixed judge — that is the whole point of the rig. Anything that breaks the pair
+(an outage on either side, a budget blow-out, a judge error) removes the prompt from
+the decision instead of leaning it: a missing arm is a missing measurement, not a
+zero (``docs/EVAL.md`` §7).
+
+The verdict rule is deliberately blunt and stated once, in :func:`verdict_of`, so a
+report can never be argued into "keep" by hand.
+"""
+
+from __future__ import annotations
+
+import json
+import statistics
+from datetime import datetime
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from bench._compare_report import CellResult
+
+CONTROL = "control"
+VARIANT = "variant"
+ARMS = (CONTROL, VARIANT)
+
+#: a paired delta at or below this is a regression on that prompt. Wider than the
+#: fixed judge's own sample noise (n_samples 2 on pro, ~±0.02), so one unlucky verdict
+#: is not a regression but a real loss on one prompt is.
+REGRESSION_DELTA = -0.03
+#: mean paired delta needed to call the change a win / a loss (symmetric)
+KEEP_DELTA = 0.02
+REVERT_DELTA = -0.02
+#: this many regressing prompts is a revert even when the mean looks fine
+REVERT_REGRESSIONS = 2
+
+
+class PairOutcome(BaseModel):
+ prompt_id: str
+ tier: str = ""
+ control: CellResult | None = None
+ variant: CellResult | None = None
+ paired: bool = Field(default=False, description="both arms produced a fixed-judge score")
+ reason: str = Field(default="", description="why the prompt is NOT paired (empty when it is)")
+ delta: float | None = Field(default=None, description="variant - control, only when paired")
+
+ @property
+ def regression(self) -> bool:
+ return self.paired and self.delta is not None and self.delta <= REGRESSION_DELTA
+
+
+class Verdict(BaseModel):
+ decision: str = Field(description="keep | revert | inconclusive")
+ n_pairs: int
+ mean_delta: float | None
+ median_delta: float | None
+ regressions: list[str] = Field(default_factory=list, description="prompt ids with delta <= REGRESSION_DELTA")
+ reason: str = ""
+
+
+def _scored(r: CellResult | None) -> bool:
+ return r is not None and r.score is not None
+
+
+def pair_up(rows: list[CellResult], prompt_order: list[tuple[str, str]] | None = None) -> list[PairOutcome]:
+ """Fold (prompt, arm) cells into one :class:`PairOutcome` per prompt.
+
+ ``prompt_order`` (id, tier) fixes the row order of the report; only prompts that
+ have at least one cell appear (a prompt never selected is not "unpaired"), and
+ prompts absent from the order are appended first-seen. Latest row wins per
+ (prompt, arm) — the results file is append-only and a redo re-appends.
+ """
+ by: dict[tuple[str, str], CellResult] = {(r.prompt_id, r.arm): r for r in rows}
+ present = {r.prompt_id for r in rows}
+ ids: list[tuple[str, str]] = [(i, t) for i, t in (prompt_order or []) if i in present]
+ seen = {i for i, _ in ids}
+ for r in rows:
+ if r.prompt_id not in seen:
+ ids.append((r.prompt_id, r.tier))
+ seen.add(r.prompt_id)
+ out: list[PairOutcome] = []
+ for pid, tier in ids:
+ c, v = by.get((pid, CONTROL)), by.get((pid, VARIANT))
+ p = PairOutcome(prompt_id=pid, tier=tier, control=c, variant=v)
+ if _scored(c) and _scored(v):
+ p.paired, p.delta = True, round(v.score - c.score, 4) # type: ignore[union-attr]
+ else:
+ p.reason = "; ".join(f"{name} {'missing' if r is None else r.status}"
+ for name, r in ((CONTROL, c), (VARIANT, v)) if not _scored(r))
+ out.append(p)
+ return out
+
+
+def verdict_of(pairs: list[PairOutcome]) -> Verdict:
+ """keep iff mean delta >= +0.02 AND no regression; revert iff mean delta <= -0.02
+ OR >= 2 regressions; otherwise inconclusive. No pairs is inconclusive, never keep."""
+ deltas = [p.delta for p in pairs if p.paired and p.delta is not None]
+ regressions = [p.prompt_id for p in pairs if p.regression]
+ if not deltas:
+ return Verdict(decision="inconclusive", n_pairs=0, mean_delta=None, median_delta=None,
+ reason="no prompt has both arms scored")
+ mean, median = round(statistics.fmean(deltas), 4), round(statistics.median(deltas), 4)
+ v = Verdict(decision="inconclusive", n_pairs=len(deltas), mean_delta=mean, median_delta=median, regressions=regressions)
+ if mean <= REVERT_DELTA or len(regressions) >= REVERT_REGRESSIONS:
+ v.decision, v.reason = "revert", (f"mean delta {mean:+.3f} <= {REVERT_DELTA:+.2f}" if mean <= REVERT_DELTA
+ else f"{len(regressions)} regressions (>= {REVERT_REGRESSIONS})")
+ elif mean >= KEEP_DELTA and not regressions:
+ v.decision, v.reason = "keep", f"mean delta {mean:+.3f} >= {KEEP_DELTA:+.2f} and no regression"
+ else:
+ v.reason = (f"mean delta {mean:+.3f} inside ({REVERT_DELTA:+.2f}, {KEEP_DELTA:+.2f})" if not regressions
+ else f"mean delta {mean:+.3f} but {len(regressions)} regression(s): {', '.join(regressions)}")
+ return v
+
+
+class ArmSummary(BaseModel):
+ arm: str
+ n: int
+ n_scored: int
+ n_infra_failed: int
+ n_budget_exhausted: int
+ n_other_unscored: int
+ mean_score: float | None
+ median_score: float | None
+ cost_usd: float = Field(description="generation + fixed-judge spend over every cell that ran")
+ mean_minutes: float
+
+
+def arm_summary(rows: list[CellResult], arm: str) -> ArmSummary:
+ rs = [r for r in rows if r.arm == arm]
+ ran = [r for r in rs if r.status != "infra_failed"]
+ scores = [r.score for r in ran if r.score is not None]
+ return ArmSummary(
+ arm=arm, n=len(rs), n_scored=len(scores), n_infra_failed=len(rs) - len(ran),
+ n_budget_exhausted=sum(r.status == "budget_exhausted" for r in ran),
+ n_other_unscored=sum(r.score is None and r.status != "budget_exhausted" for r in ran),
+ mean_score=round(statistics.fmean(scores), 4) if scores else None,
+ median_score=round(statistics.median(scores), 4) if scores else None,
+ cost_usd=round(sum(r.gen_cost_usd + r.judge_cost_usd for r in ran), 4),
+ mean_minutes=round(statistics.fmean(r.wall_s for r in ran) / 60, 2) if ran else 0.0,
+ )
+
+
+def _fmt(x: float | None, signed: bool = False) -> str:
+ if x is None:
+ return "—"
+ return f"{x:+.3f}" if signed else f"{x:.3f}"
+
+
+def _cell(r: CellResult | None) -> str:
+ if r is None:
+ return "missing"
+ return _fmt(r.score) if r.score is not None else r.status
+
+
+def render_summary(pairs: list[PairOutcome], rows: list[CellResult], *, title: str, variant_env: dict[str, str],
+ generator: str, judge: str, rounds: int) -> str:
+ """The Markdown report; pure so tests can read it without a filesystem."""
+ v = verdict_of(pairs)
+ arms = [arm_summary(rows, a) for a in ARMS]
+ lines = [f"# A/B: {title}", "",
+ f"variant env: `{' '.join(f'{k}={val}' for k, val in variant_env.items()) or '(none)'}` ",
+ f"generator: `{generator}` · fixed judge: `{judge}` · rounds: {rounds} · "
+ f"written {datetime.now().strftime('%Y-%m-%d %H:%M')}", "",
+ f"## Verdict: **{v.decision}**", "",
+ f"{v.reason}. Paired prompts: {v.n_pairs} · mean delta {_fmt(v.mean_delta, True)} · "
+ f"median delta {_fmt(v.median_delta, True)} · regressions (delta <= {REGRESSION_DELTA:+.2f}): "
+ f"{', '.join(v.regressions) or 'none'}", "",
+ "## Arms", "",
+ "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | cost $ | mean min |",
+ "|---|---|---|---|---|---|---|---|---|---|"]
+ for a in arms:
+ lines.append(f"| {a.arm} | {a.n} | {a.n_scored} | {a.n_infra_failed} | {a.n_budget_exhausted} | "
+ f"{a.n_other_unscored} | {_fmt(a.mean_score)} | {_fmt(a.median_score)} | {a.cost_usd:.2f} | {a.mean_minutes:.1f} |")
+ lines += ["", "## Per prompt", "", "| prompt | tier | control | variant | delta | note |", "|---|---|---|---|---|---|"]
+ for p in pairs:
+ note = "REGRESSION" if p.regression else ("" if p.paired else f"unpaired: {p.reason}")
+ lines.append(f"| {p.prompt_id} | {p.tier} | {_cell(p.control)} | {_cell(p.variant)} | {_fmt(p.delta, True)} | {note} |")
+ dropped = sum(a.n_infra_failed for a in arms)
+ lines += ["", f"n_infra_failed: {dropped}" + (" — re-run with `--redo-status infra_failed` before trusting the verdict"
+ if dropped else ""), "",
+ "Rule: keep iff mean delta >= +0.02 and no regression; revert iff mean delta <= -0.02 or >= 2 regressions; "
+ "else inconclusive. A prompt counts only when BOTH arms were scored by the fixed judge.", ""]
+ return "\n".join(lines)
+
+
+def write_report(out: Path, rows: list[CellResult], prompt_order: list[tuple[str, str]], **meta: object) -> Verdict:
+ """Write ``pairs.json`` + ``summary.md`` under ``out`` and return the verdict."""
+ pairs = pair_up(rows, prompt_order)
+ v = verdict_of(pairs)
+ (out / "pairs.json").write_text(
+ json.dumps({"verdict": v.model_dump(mode="json"),
+ "pairs": [p.model_dump(mode="json") for p in pairs]}, indent=2))
+ (out / "summary.md").write_text(render_summary(pairs, rows, **meta)) # type: ignore[arg-type]
+ return v
+
+
+__all__ = ["ARMS", "CONTROL", "KEEP_DELTA", "REGRESSION_DELTA", "REVERT_DELTA", "REVERT_REGRESSIONS", "VARIANT",
+ "ArmSummary", "PairOutcome", "Verdict", "arm_summary", "pair_up", "render_summary", "verdict_of",
+ "write_report"]
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.py
new file mode 100644
index 00000000..d52deba6
--- /dev/null
+++ b/harness/bench/ab_plan.py
@@ -0,0 +1,341 @@
+"""Controlled A/B of a plan / brief change: control vs variant, paired per prompt.
+
+ python bench/ab_plan.py --prompts bench/prompts/static_objects_v2.yaml \\
+ --variant-env CV3D_PLAN_BRIEF=on --out bench/out/ab_brief \\
+ [--generator api-agent:gemini:gemini-3.7-flash] [--judge gemini:gemini-3.1-pro-preview] \\
+ [--rounds 2] [--max-usd 2.5] [--ids a,b] [--wait-for-provider 60] [--redo-status infra_failed]
+
+The only thing that differs between the arms is the ``--variant-env`` block, applied to
+the VARIANT arm alone. Everything else is held fixed on both sides: the generator, the
+planner, the in-loop judge, the rounds and dollars, and the fixed judge that produces
+the reported score (``bench/_fixed_eval.py``, ``n_samples`` 2).
+
+Why each arm is a child PROCESS rather than a thread: the switches under test are read
+from the environment (``codeverse.tracks.brief.brief_enabled`` reads ``os.environ`` at
+call time; anything under ``Settings`` is read once through an ``lru_cache``), so two
+threads in one interpreter cannot hold different values of them. A child gets exactly
+the env its arm needs and nothing leaks across. Each child owns a key pool, so the
+driver caps every child at ``CV3D_RATE__MAX_IN_FLIGHT`` (default 16) — two children at
+16 stay inside the 64 knee measured for one process (``docs/COST.md`` §23).
+
+Why the arms launch as simultaneous PAIRS: provider weather changes by the hour, and a
+verdict that compares an arm run in a storm against one run in the clear measures the
+storm. Prompt N's control and variant start together and the next prompt waits for
+both (``--parallel`` is fixed at 2 by construction). A cell lost to an outage is
+``infra_failed`` and removes its prompt from the pairing, never scores it 0
+(``docs/EVAL.md`` §7); ``--redo-status infra_failed`` re-runs BOTH arms of that prompt so
+the pair is re-measured together.
+
+Layout of ``--out``: ``ab.json`` (the arms, options and variant env) ·
+``results.jsonl`` (one ``CellResult`` per (prompt, arm); resume source) ·
+``arms/<arm>/cells/<prompt>/<slug>/{run,eval,cell.json,worker.log}`` · ``pairs.json`` ·
+``summary.md``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+from collections.abc import Callable, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from codeverse._compat import UTC
+
+REPO = Path(__file__).resolve().parents[1]
+if str(REPO) not in sys.path: # `python bench/ab_plan.py` from anywhere
+ sys.path.insert(0, str(REPO))
+
+from bench._ab_report import ARMS, CONTROL, VARIANT, Verdict, write_report # noqa: E402
+from bench._compare_report import CellResult, load_jsonl # noqa: E402
+from bench._infra import is_infra_failure # noqa: E402
+from bench.compare_backends import ( # noqa: E402
+ Arm,
+ CompareDeps,
+ CompareOptions,
+ _preflight,
+ run_cell,
+)
+from bench.run_bench import Battery, BenchPrompt, select_prompts # noqa: E402
+
+DEFAULT_GENERATOR = "api-agent:gemini:gemini-3.7-flash"
+DEFAULT_JUDGE = "gemini:gemini-3.1-pro-preview"
+#: the env name Settings actually reads (``Rate.max_in_flight`` under ``CV3D_`` + ``__``).
+#: NOT ``CV3D_MAX_IN_FLIGHT``: that flat name is read by nothing.
+MAX_IN_FLIGHT_ENV = "CV3D_RATE__MAX_IN_FLIGHT"
+DEFAULT_MAX_IN_FLIGHT = 16
+#: only ever 2 — one control + one variant, launched together (see module docstring)
+PARALLEL = 2
+
+
+class AbOptions(BaseModel):
+ generator: str = DEFAULT_GENERATOR
+ judge: str = DEFAULT_JUDGE
+ loop_judge: str | None = Field(default=None, description="in-loop judge for BOTH arms (None → settings default)")
+ planner: str | None = Field(default=None, description="planner for BOTH arms (None → settings default)")
+ rounds: int = 2
+ max_usd: float = 2.5
+ max_minutes: float = 45.0
+ n_samples: int = 2
+ ids: list[str] = Field(default_factory=list)
+ tiers: list[str] = Field(default_factory=list)
+ limit: int | None = None
+ resume: bool = True
+ redo_status: list[str] = Field(default_factory=list, description="re-run prompts where EITHER arm has one of these")
+ variant_env: dict[str, str] = Field(default_factory=dict, description="applied to the variant arm only")
+ max_in_flight: int = DEFAULT_MAX_IN_FLIGHT
+
+ def compare_options(self) -> CompareOptions:
+ """The per-cell options handed to ``compare_backends.run_cell`` (both arms identical)."""
+ return CompareOptions(judge=self.judge, loop_judge=self.loop_judge, planner=self.planner, rounds=self.rounds,
+ max_usd=self.max_usd, max_minutes=self.max_minutes, n_samples=self.n_samples,
+ resume=self.resume, pairwise=False)
+
+
+def parse_variant_env(items: Sequence[str]) -> dict[str, str]:
+ """``["K=V", "K2=V2"]`` → ``{K: V, K2: V2}``; a bare ``K`` is rejected, not defaulted."""
+ out: dict[str, str] = {}
+ for it in items:
+ k, sep, v = it.partition("=")
+ if not sep or not k.strip():
+ raise ValueError(f"--variant-env needs KEY=VALUE, got {it!r}")
+ out[k.strip()] = v
+ return out
+
+
+def child_env(arm: str, opts: AbOptions, base: dict[str, str] | None = None) -> dict[str, str]:
+ """The environment of one arm's child process.
+
+ The variant gets ``variant_env`` on top of the driver's env; the control gets the
+ driver's env with those same keys REMOVED, so a switch that happens to be set in
+ the launching shell cannot silently turn the control into a second variant. Both
+ get the in-flight cap unless the driver already set one.
+ """
+ env = dict(os.environ if base is None else base)
+ for k in opts.variant_env:
+ env.pop(k, None)
+ if arm == VARIANT:
+ env.update(opts.variant_env)
+ env.setdefault(MAX_IN_FLIGHT_ENV, str(opts.max_in_flight))
+ env["PYTHONUNBUFFERED"] = "1"
+ return env
+
+
+def arm_dir(out: Path, arm: str) -> Path:
+ return out / "arms" / arm
+
+
+def cell_dir(out: Path, arm: str, item_id: str, generator: str) -> Path:
+ """Where ``compare_backends.run_cell`` puts this arm's cell (its layout, our root)."""
+ return arm_dir(out, arm) / "cells" / item_id / _harness_arm(generator).slug
+
+
+def _harness_arm(generator: str) -> Arm:
+ return Arm(raw=f"harness:{generator}", kind="harness", target=generator)
+
+
+def worker_argv(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> list[str]:
+ return [sys.executable, str(Path(__file__).resolve()), "cell", "--prompts", str(battery_path), "--out", str(out),
+ "--id", item_id, "--arm", arm, "--options-json", opts.model_dump_json()]
+
+
+# ----------------------------------------------------------------------------- one cell, in a child
+CellRunner = Callable[[Path, Path, BenchPrompt, str, AbOptions], CellResult]
+
+
+def spawn_cell(battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts: AbOptions) -> CellResult:
+ """Run one (prompt, arm) cell in a child process and read back its ``cell.json``.
+
+ A child that dies without writing a cell (killed, import error, unhandled outage)
+ is classified by the tail of its log with the same rule every other failure path
+ uses, so an outage stays ``infra_failed`` and never becomes a scored zero.
+ """
+ cell = cell_dir(out, arm, item.id, opts.generator)
+ cell.mkdir(parents=True, exist_ok=True)
+ log, marker = cell / "worker.log", cell / "cell.json"
+ marker.unlink(missing_ok=True) # a stale verdict from an earlier attempt must not stand in for this one
+ t0 = time.time()
+ with log.open("a") as fh:
+ fh.write(f"--- {datetime.now(UTC).isoformat()} {arm} {item.id}\n")
+ fh.flush()
+ rc = subprocess.run(worker_argv(battery_path, out, item.id, arm, opts), cwd=REPO, env=child_env(arm, opts),
+ stdout=fh, stderr=subprocess.STDOUT, check=False).returncode
+ if marker.is_file():
+ return CellResult.model_validate_json(marker.read_text())
+ tail = log.read_text()[-1500:] if log.is_file() else ""
+ return CellResult(prompt_id=item.id, tier=item.tier, arm=arm, kind="harness", target=opts.generator,
+ judge=opts.judge, workspace=str(cell), wall_s=round(time.time() - t0, 1),
+ status="infra_failed" if is_infra_failure(tail) else "error",
+ error=f"worker exited {rc} without a cell.json; log tail: {tail[-400:]!r}")
+
+
+def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> int:
+ """Child entry: the harness run + fixed evaluation of one arm, via ``run_cell``."""
+ from bench._fixed_eval import FixedEvaluator
+
+ battery = Battery.load(battery_path)
+ item = next((p for p in battery.prompts if p.id == item_id), None)
+ if item is None:
+ print(f"no prompt {item_id!r} in {battery_path}", file=sys.stderr)
+ return 2
+ deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples))
+ res = run_cell(battery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps)
+ res.arm, res.target = arm, opts.generator # the row is keyed by A/B arm, not by generator
+ (Path(res.workspace) / "cell.json").write_text(res.model_dump_json(indent=1))
+ print(f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}")
+ return 0
+
+
+# ----------------------------------------------------------------------------- the driver
+def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[tuple[BenchPrompt, list[str]]]:
+ """Which (prompt, arms) still need running.
+
+ A prompt with EITHER arm in ``redo_status`` is re-run on BOTH arms — a redo exists
+ to re-measure the pair together, not to patch one side in different weather. A
+ prompt with only one arm recorded (the driver was interrupted mid-pair) runs only
+ the missing arm; its partner already finished and re-running it would pay twice
+ for the same generation without making the weather match.
+ """
+ todo: list[tuple[BenchPrompt, list[str]]] = []
+ redo = set(opts.redo_status)
+ for p in select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit):
+ rows = {a: done.get((p.id, a)) for a in ARMS}
+ if redo and any(r is not None and r.status in redo for r in rows.values()):
+ for a in ARMS:
+ done.pop((p.id, a), None)
+ todo.append((p, list(ARMS)))
+ continue
+ missing = [a for a in ARMS if rows[a] is None]
+ if missing:
+ todo.append((p, missing))
+ return todo
+
+
+def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, run_cell_fn: CellRunner = spawn_cell,
+ on_result: Callable[[CellResult], None] | None = None) -> Verdict:
+ """Run (or resume) the A/B, one prompt-pair at a time; write pairs.json + summary.md."""
+ battery_path, out = Path(battery_path), Path(out_dir)
+ battery = Battery.load(battery_path)
+ out.mkdir(parents=True, exist_ok=True)
+ (out / "ab.json").write_text(json.dumps({"battery": battery.name, "prompts": str(battery_path),
+ "arms": {CONTROL: {}, VARIANT: opts.variant_env},
+ "options": opts.model_dump(mode="json"),
+ "started_at": datetime.now(UTC).isoformat()}, indent=2))
+ results = out / "results.jsonl"
+ done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}
+ todo = _plan_todo(battery, done, opts)
+ with ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:
+ for item, arms in todo: # one pair at a time: both arms of a prompt see the same weather
+ futs = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]
+ for fut in futs:
+ r = fut.result()
+ done[(r.prompt_id, r.arm)] = r
+ fh.write(r.model_dump_json() + "\n")
+ fh.flush()
+ if on_result:
+ on_result(r)
+ order = [(p.id, p.tier) for p in battery.prompts]
+ return write_report(out, list(done.values()), order, title=f"{battery.name} / {out.name}",
+ variant_env=opts.variant_env, generator=opts.generator, judge=opts.judge, rounds=opts.rounds)
+
+
+def preflight(opts: AbOptions, *, wait_minutes: float) -> bool:
+ """Probe every model either arm needs (planner, generator, loop judge, fixed judge)."""
+ return _preflight(opts.judge, [_harness_arm(opts.generator)], opts.compare_options(), wait_minutes=wait_minutes)
+
+
+# ----------------------------------------------------------------------------- CLI
+def _parser() -> argparse.ArgumentParser:
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ sub = ap.add_subparsers(dest="cmd")
+ w = sub.add_parser("cell", help="(internal) run one arm of one prompt in this process")
+ w.add_argument("--prompts", required=True)
+ w.add_argument("--out", required=True)
+ w.add_argument("--id", required=True)
+ w.add_argument("--arm", required=True, choices=ARMS)
+ w.add_argument("--options-json", required=True)
+ ap.add_argument("--prompts")
+ ap.add_argument("--out")
+ ap.add_argument("--variant-env", action="append", default=[], metavar="KEY=VALUE",
+ help="env applied to the VARIANT arm only (repeatable)")
+ ap.add_argument("--generator", default=DEFAULT_GENERATOR)
+ ap.add_argument("--judge", default=DEFAULT_JUDGE, help="FIXED judge model id (both arms)")
+ ap.add_argument("--loop-judge", default=None)
+ ap.add_argument("--planner", default=None)
+ ap.add_argument("--rounds", type=int, default=2)
+ ap.add_argument("--max-usd", type=float, default=2.5)
+ ap.add_argument("--max-minutes", type=float, default=45.0)
+ ap.add_argument("--n-samples", type=int, default=2)
+ ap.add_argument("--ids", default="")
+ ap.add_argument("--tiers", default="")
+ ap.add_argument("--limit", type=int, default=None)
+ ap.add_argument("--max-in-flight", type=int, default=DEFAULT_MAX_IN_FLIGHT, help=f"per child ({MAX_IN_FLIGHT_ENV})")
+ ap.add_argument("--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)")
+ ap.add_argument("--redo-status", default="", help="comma list, e.g. infra_failed (re-runs both arms of the prompt)")
+ ap.add_argument("--no-resume", action="store_true")
+ ap.add_argument("--no-preflight", action="store_true", help="skip the provider health check (do not)")
+ ap.add_argument("--wait-for-provider", type=float, default=0.0, metavar="MIN")
+ ap.add_argument("--allow-siblings", action="store_true",
+ help="launch even when other harness processes are running (they share the quota)")
+ ap.add_argument("--report-only", action="store_true", help="only rebuild pairs.json / summary.md")
+ return ap
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ ns = _parser().parse_args(argv)
+ if ns.cmd == "cell":
+ return cell_main(Path(ns.prompts), Path(ns.out), ns.id, ns.arm, AbOptions.model_validate_json(ns.options_json))
+ if not ns.prompts or not ns.out:
+ _parser().error("--prompts and --out are required")
+ if ns.parallel != PARALLEL:
+ _parser().error(f"--parallel must be {PARALLEL}: the arms of a prompt run together and nothing else does")
+ opts = AbOptions(generator=ns.generator, judge=ns.judge, loop_judge=ns.loop_judge, planner=ns.planner,
+ rounds=ns.rounds, max_usd=ns.max_usd, max_minutes=ns.max_minutes, n_samples=ns.n_samples,
+ ids=[i for i in ns.ids.split(",") if i], tiers=[t for t in ns.tiers.split(",") if t],
+ limit=ns.limit, resume=not ns.no_resume, redo_status=[s for s in ns.redo_status.split(",") if s],
+ variant_env=parse_variant_env(ns.variant_env), max_in_flight=ns.max_in_flight)
+ out = Path(ns.out)
+ if ns.report_only:
+ battery = Battery.load(ns.prompts)
+ v = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],
+ title=f"{battery.name} / {out.name}", variant_env=opts.variant_env, generator=opts.generator,
+ judge=opts.judge, rounds=opts.rounds)
+ print(f"verdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")
+ return 0
+ if not opts.variant_env:
+ _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the judge")
+ from codeverse.models.health import sibling_processes
+
+ if (sib := sibling_processes()) and not ns.allow_siblings:
+ print(f"refusing to start: {sib} other harness process(es) are running and would share the key quota "
+ "(docs/COST.md §23). Wait for them, or pass --allow-siblings.", flush=True)
+ return 3
+ if not ns.no_preflight and not preflight(opts, wait_minutes=ns.wait_for_provider):
+ return 2
+
+ def _log(r: CellResult) -> None:
+ print(f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:8s} score={r.score} "
+ f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}", flush=True)
+
+ v = run_ab(ns.prompts, out, opts, on_result=_log)
+ print(f"\nverdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")
+ lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]
+ if lost:
+ print(f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "
+ f"those prompts with: --redo-status infra_failed")
+ return 0
+
+
+__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "PARALLEL", "AbOptions", "CellRunner",
+ "cell_dir", "cell_main", "child_env", "main", "parse_variant_env", "preflight", "run_ab", "spawn_cell",
+ "worker_argv"]
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/harness/codeverse/cli/doctor.py b/harness/codeverse/cli/doctor.py
index 38087906..f4d84b81 100644
--- a/harness/codeverse/cli/doctor.py
+++ b/harness/codeverse/cli/doctor.py
@@ -190,7 +190,7 @@ def check_keys(live: bool) -> list[Row]:
"OK" if sib == 0 else "WARN",
f"{sib} other harness process(es) running — each keeps its OWN key pool, so the real "
f"concurrency is ~{(sib + 1) * cap} in-flight against one shared quota "
- f"(docs/COST.md §23; run one battery at a time or set CV3D_MAX_IN_FLIGHT={max(1, cap // (sib + 1))})"
+ f"(docs/COST.md §23; run one battery at a time or set CV3D_RATE__MAX_IN_FLIGHT={max(1, cap // (sib + 1))})"
if sib else "sole harness process: max_in_flight applies as configured",
))
rows.append(("anthropic key", "OK" if s.anthropic_api_key else "WARN", "set" if s.anthropic_api_key else "not set (anthropic:* backends unavailable)"))
diff --git a/harness/codeverse/models/health.py b/harness/codeverse/models/health.py
index 6a602bd9..c04ecada 100644
--- a/harness/codeverse/models/health.py
+++ b/harness/codeverse/models/health.py
@@ -117,7 +117,7 @@ def _bare_model(model_id: str, timeout_s: float):
#: whole command line: the repo path itself contains "3dcodeverse", so a substring test
#: counts every shell that merely `cd`s into the tree.
_MODULE = "codeverse.cli.main"
-_SCRIPTS = ("compare_backends.py", "run_bench.py")
+_SCRIPTS = ("compare_backends.py", "run_bench.py", "ab_plan.py")
_ENTRY_POINTS = ("3dcv", "3dcodeverse")
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 9025fbae..39277867 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1082,7 +1082,8 @@ Three consequences:
setting where it could not win. Its negative result stands for the configuration
tested and should NOT be read as "back-pressure does not help".
3. **Operational rule until a cross-process limiter exists**: run ONE battery at a time,
- or set `CV3D_MAX_IN_FLIGHT` to `64 / (concurrent processes)`.
+ or set `CV3D_RATE__MAX_IN_FLIGHT` (the name `Settings.rate` actually reads; the flat
+ `CV3D_MAX_IN_FLIGHT` is read by nothing) to `64 / (concurrent processes)`.
The real fix is a machine-wide limiter — a file-locked token bucket under
`~/.cache/codeverse/` that every process shares — so the quota is enforced where it
diff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.py
new file mode 100644
index 00000000..2e1d2a70
--- /dev/null
+++ b/harness/tests/compare_bench/test_ab_plan.py
@@ -0,0 +1,259 @@
+"""bench/ab_plan.py — pairing, verdict rule, outage exclusion, env isolation (offline)."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[2]
+if str(REPO) not in sys.path:
+ sys.path.insert(0, str(REPO))
+
+from bench._ab_report import ( # noqa: E402
+ CONTROL,
+ VARIANT,
+ PairOutcome,
+ arm_summary,
+ pair_up,
+ render_summary,
+ verdict_of,
+)
+from bench._compare_report import CellResult, load_jsonl # noqa: E402
+from bench.ab_plan import ( # noqa: E402
+ MAX_IN_FLIGHT_ENV,
+ AbOptions,
+ _plan_todo,
+ cell_dir,
+ child_env,
+ main,
+ parse_variant_env,
+ run_ab,
+ worker_argv,
+)
+from bench.run_bench import Battery # noqa: E402
+
+BATTERY = REPO / "bench" / "prompts" / "compare_v1.yaml"
+
+
+def _row(pid: str, arm: str, score: float | None, status: str = "scored", **kw) -> CellResult:
+ return CellResult(prompt_id=pid, arm=arm, kind="harness", score=score, status=status,
+ gen_cost_usd=kw.pop("cost", 1.0), judge_cost_usd=0.1, wall_s=kw.pop("wall", 120.0), **kw)
+
+
+def _pairs(*deltas: float) -> list[PairOutcome]:
+ return [PairOutcome(prompt_id=f"p{i}", paired=True, delta=d) for i, d in enumerate(deltas)]
+
+
+# ----------------------------------------------------------------------------- verdict rule
+def test_keep_needs_mean_gain_and_no_regression():
+ assert verdict_of(_pairs(0.03, 0.02, 0.01, 0.04)).decision == "keep"
+ # one regression blocks a keep even when the mean clears the bar
+ v = verdict_of(_pairs(0.10, 0.10, 0.10, -0.03))
+ assert v.decision == "inconclusive" and v.regressions == ["p3"]
+ # a mean just under the bar is not a keep
+ assert verdict_of(_pairs(0.01, 0.02, 0.02, 0.01)).decision == "inconclusive"
+
+
+def test_revert_on_mean_loss_or_two_regressions():
+ assert verdict_of(_pairs(-0.02, -0.02, -0.02)).decision == "revert"
+ v = verdict_of(_pairs(0.20, 0.20, -0.03, -0.05)) # mean +0.08 but two prompts lost
+ assert v.decision == "revert" and v.regressions == ["p2", "p3"]
+ assert verdict_of(_pairs(-0.019, 0.0)).decision == "inconclusive"
+
+
+def test_no_pairs_is_inconclusive_never_keep():
+ v = verdict_of([PairOutcome(prompt_id="p0", paired=False, reason="control infra_failed")])
+ assert v.decision == "inconclusive" and v.n_pairs == 0 and v.mean_delta is None
+
+
+# ----------------------------------------------------------------------------- pairing
+def test_a_pair_counts_only_when_both_arms_scored():
+ rows = [_row("a", CONTROL, 0.70), _row("a", VARIANT, 0.75),
+ _row("b", CONTROL, 0.80), _row("b", VARIANT, None, "infra_failed"),
+ _row("c", CONTROL, None, "budget_exhausted"), _row("c", VARIANT, 0.9),
+ _row("d", CONTROL, 0.5)]
+ pairs = {p.prompt_id: p for p in pair_up(rows)}
+ assert pairs["a"].paired and pairs["a"].delta == pytest.approx(0.05)
+ assert not pairs["b"].paired and pairs["b"].reason == "variant infra_failed"
+ assert not pairs["c"].paired and pairs["c"].reason == "control budget_exhausted"
+ assert not pairs["d"].paired and pairs["d"].reason == "variant missing"
+ v = verdict_of(list(pairs.values()))
+ assert v.n_pairs == 1 and v.mean_delta == pytest.approx(0.05)
+
+
+def test_latest_row_wins_and_order_follows_the_battery():
+ rows = [_row("a", CONTROL, 0.1), _row("b", CONTROL, 0.2), _row("b", VARIANT, 0.2),
+ _row("a", VARIANT, 0.1), _row("a", CONTROL, 0.9)] # re-appended on a redo
+ pairs = pair_up(rows, [("b", "easy"), ("a", "hard")])
+ assert [p.prompt_id for p in pairs] == ["b", "a"] and pairs[1].tier == "hard"
+ assert pairs[1].delta == pytest.approx(0.1 - 0.9)
+
+
+def test_outage_cells_are_excluded_from_every_arm_rate():
+ """An arm hit by weather must summarise the same as one that ran in the clear, and
+ the loss must stay visible (docs/EVAL.md §7)."""
+ clear = [_row(f"p{i}", CONTROL, 0.8) for i in range(4)]
+ unlucky = [_row(f"p{i}", VARIANT, 0.8) for i in range(4)] + [_row("p9", VARIANT, None, "infra_failed", wall=3600, cost=0.0)]
+ c, v = arm_summary(clear + unlucky, CONTROL), arm_summary(clear + unlucky, VARIANT)
+ assert c.mean_score == v.mean_score and c.mean_minutes == v.mean_minutes and c.cost_usd == v.cost_usd
+ assert v.n_infra_failed == 1 and v.n == 5 and v.n_scored == 4
+ md = render_summary(pair_up(clear + unlucky), clear + unlucky, title="t", variant_env={"X": "1"},
+ generator="g", judge="j", rounds=2)
+ assert "n_infra_failed: 1" in md and "--redo-status infra_failed" in md and "variant env: `X=1`" in md
+ assert "| p9 | | missing | infra_failed | — | unpaired: control missing; variant infra_failed |" in md
+
+
+def test_summary_marks_regressions_and_states_the_rule():
+ rows = [_row("a", CONTROL, 0.8), _row("a", VARIANT, 0.7), _row("b", CONTROL, 0.5), _row("b", VARIANT, 0.6)]
+ md = render_summary(pair_up(rows), rows, title="t", variant_env={}, generator="g", judge="j", rounds=1)
+ assert "| a | | 0.800 | 0.700 | -0.100 | REGRESSION |" in md and "## Verdict: **inconclusive**" in md
+ assert "keep iff mean delta >= +0.02" in md and "(none)" in md
+
+
+# ----------------------------------------------------------------------------- env isolation
+def test_variant_env_is_applied_to_the_variant_arm_only():
+ opts = AbOptions(variant_env={"CV3D_PLAN_BRIEF": "on"})
+ base = {"PATH": "/bin", "CV3D_PLAN_BRIEF": "on"} # leaked from the launching shell
+ c, v = child_env(CONTROL, opts, base), child_env(VARIANT, opts, base)
+ assert v["CV3D_PLAN_BRIEF"] == "on"
+ assert "CV3D_PLAN_BRIEF" not in c, "the control must never inherit the switch under test"
+ assert c["PATH"] == v["PATH"] == "/bin"
+ assert c[MAX_IN_FLIGHT_ENV] == v[MAX_IN_FLIGHT_ENV] == "16"
+ assert MAX_IN_FLIGHT_ENV == "CV3D_RATE__MAX_IN_FLIGHT", "the name Settings.rate actually reads"
+ # a cap set by the driver's own environment is respected, not overwritten
+ assert child_env(VARIANT, opts, {MAX_IN_FLIGHT_ENV: "4"})[MAX_IN_FLIGHT_ENV] == "4"
+ assert child_env(VARIANT, opts, {})["PYTHONUNBUFFERED"] == "1"
+
+
+def test_the_in_flight_env_name_is_one_settings_reads(monkeypatch):
+ from codeverse.config import get_settings
+
+ monkeypatch.setenv(MAX_IN_FLIGHT_ENV, "7")
+ get_settings.cache_clear()
+ try:
+ assert get_settings().rate.max_in_flight == 7
+ finally:
+ get_settings.cache_clear()
+
+
+def test_parse_variant_env():
+ assert parse_variant_env(["A=1", "B=x=y", "C="]) == {"A": "1", "B": "x=y", "C": ""}
+ for bad in (["A"], ["=1"]):
+ with pytest.raises(ValueError):
+ parse_variant_env(bad)
+
+
+def test_worker_argv_carries_the_whole_option_block(tmp_path: Path):
+ opts = AbOptions(variant_env={"K": "v"}, rounds=1)
+ argv = worker_argv(BATTERY, tmp_path, "cmp_easy_stool", VARIANT, opts)
+ assert argv[1].endswith("ab_plan.py") and argv[2] == "cell" and "--arm" in argv
+ assert AbOptions.model_validate_json(argv[-1]) == opts
+ assert cell_dir(tmp_path, CONTROL, "x", "api-agent:gemini:g").parent.parts[-4:] == ("arms", "control", "cells", "x")
+
+
+def test_a_child_that_dies_without_a_cell_is_an_error_not_a_score(tmp_path: Path):
+ """Real subprocess, no network: an unknown prompt id makes the worker exit 2 before any
+ model is touched, and the driver must record that as a scoreless error cell."""
+ from bench.ab_plan import spawn_cell
+ from bench.run_bench import BenchPrompt
+
+ ghost = BenchPrompt(id="no_such_prompt", prompt="x", tier="easy")
+ res = spawn_cell(BATTERY, tmp_path, ghost, CONTROL, AbOptions(variant_env={"K": "v"}))
+ assert res.status == "error" and res.score is None and res.arm == CONTROL
+ assert "worker exited 2" in res.error and "no prompt 'no_such_prompt'" in res.error
+ assert (Path(res.workspace) / "worker.log").is_file()
+
+
+# ----------------------------------------------------------------------------- the driver, with a fake cell runner
+class FakeCells:
+ """Records (prompt, arm, env-at-call) and answers from a script; the pair order is observable."""
+
+ def __init__(self, scores: dict[tuple[str, str], float | None | str]):
+ self.scores = scores
+ self.calls: list[tuple[str, str]] = []
+
+ def __call__(self, battery_path, out, item, arm, opts) -> CellResult:
+ self.calls.append((item.id, arm))
+ env = child_env(arm, opts, {})
+ s = self.scores.get((item.id, arm), 0.5)
+ if isinstance(s, str):
+ return _row(item.id, arm, None, s, tier=item.tier, workspace=env.get("CV3D_PLAN_BRIEF", ""))
+ return _row(item.id, arm, s, tier=item.tier, workspace=env.get("CV3D_PLAN_BRIEF", ""))
+
+
+def test_driver_runs_pairs_in_prompt_order_and_writes_the_verdict(tmp_path: Path):
+ b = Battery.load(BATTERY)
+ ids = [p.id for p in b.prompts[:3]]
+ fake = FakeCells({(ids[0], VARIANT): 0.55, (ids[1], VARIANT): 0.53, (ids[2], VARIANT): 0.52})
+ opts = AbOptions(variant_env={"CV3D_PLAN_BRIEF": "on"}, ids=ids)
+ v = run_ab(BATTERY, tmp_path, opts, run_cell_fn=fake)
+ # both arms of prompt N before anything of prompt N+1
+ assert [c[0] for c in fake.calls] == [i for i in ids for _ in range(2)]
+ assert {c[1] for c in fake.calls} == {CONTROL, VARIANT}
+ assert v.decision == "keep" and v.n_pairs == 3 and v.mean_delta == pytest.approx(0.0333, abs=1e-3)
+ rows = load_jsonl(tmp_path / "results.jsonl", CellResult)
+ assert len(rows) == 6
+ # the env reached only the variant cells
+ assert {r.workspace for r in rows if r.arm == VARIANT} == {"on"}
+ assert {r.workspace for r in rows if r.arm == CONTROL} == {""}
+ pairs = json.loads((tmp_path / "pairs.json").read_text())
+ assert pairs["verdict"]["decision"] == "keep" and len(pairs["pairs"]) == 3
+ assert (tmp_path / "summary.md").read_text().startswith("# A/B: compare_v1")
+ assert json.loads((tmp_path / "ab.json").read_text())["arms"] == {"control": {}, "variant": {"CV3D_PLAN_BRIEF": "on"}}
+
+
+def test_resume_skips_done_pairs_and_redo_reruns_both_arms(tmp_path: Path):
+ b = Battery.load(BATTERY)
+ ids = [p.id for p in b.prompts[:2]]
+ first = FakeCells({(ids[1], VARIANT): "infra_failed"})
+ opts = AbOptions(variant_env={"K": "v"}, ids=ids)
+ v = run_ab(BATTERY, tmp_path, opts, run_cell_fn=first)
+ assert v.n_pairs == 1, "the outage pair must not count"
+ # plain resume: nothing left to do
+ again = FakeCells({})
+ run_ab(BATTERY, tmp_path, opts, run_cell_fn=again)
+ assert again.calls == []
+ # redo: the whole pair of the hit prompt, nothing else
+ redo = FakeCells({})
+ v = run_ab(BATTERY, tmp_path, opts.model_copy(update={"redo_status": ["infra_failed"]}), run_cell_fn=redo)
+ assert sorted(redo.calls) == [(ids[1], CONTROL), (ids[1], VARIANT)]
+ assert v.n_pairs == 2
+ rows = load_jsonl(tmp_path / "results.jsonl", CellResult)
+ latest = {(r.prompt_id, r.arm): r for r in rows}
+ assert latest[(ids[1], VARIANT)].status == "scored" and len(rows) == 6
+
+
+def test_an_interrupted_pair_runs_only_its_missing_arm():
+ b = Battery.load(BATTERY)
+ p = b.prompts[0]
+ done = {(p.id, CONTROL): _row(p.id, CONTROL, 0.5)}
+ todo = _plan_todo(b, done, AbOptions(ids=[p.id]))
+ assert todo == [(p, [VARIANT])]
+ assert _plan_todo(b, {(p.id, CONTROL): _row(p.id, CONTROL, 0.5), (p.id, VARIANT): _row(p.id, VARIANT, 0.5)},
+ AbOptions(ids=[p.id])) == []
+
+
+def test_cli_refuses_identical_arms_and_wrong_parallel(tmp_path: Path, capsys):
+ with pytest.raises(SystemExit):
+ main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--no-preflight"])
+ with pytest.raises(SystemExit):
+ main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--variant-env", "K=v", "--parallel", "8"])
+
+
+def test_cli_report_only_rebuilds_from_results(tmp_path: Path, capsys):
+ (tmp_path / "results.jsonl").write_text("".join(r.model_dump_json() + "\n" for r in
+ [_row("cmp_easy_stool", CONTROL, 0.5), _row("cmp_easy_stool", VARIANT, 0.4)]))
+ assert main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--report-only"]) == 0
+ assert "verdict: revert" in capsys.readouterr().out
+ assert (tmp_path / "pairs.json").is_file()
+
+
+def test_ab_plan_children_count_as_harness_processes():
+ from codeverse.models.health import _is_harness_argv
+
+ assert _is_harness_argv([sys.executable, str(REPO / "bench" / "ab_plan.py"), "cell", "--arm", "control"])
+ assert os.environ is not None