Official Codebase of 3DCodeVerse
Python
bench: ab_plan.py — a paired control/variant A/B for plan and brief switches
gaoypeng68b763071b38
diff --git a/harness/bench/_ab_report.py b/harness/bench/_ab_report.pynew file mode 100644index 00000000..fa6b4876--- /dev/null+++ b/harness/bench/_ab_report.py@@ -0,0 +1,201 @@+"""Pairing, verdict and ``summary.md`` for ``bench/ab_plan.py``.++An A/B of a plan/brief change is decided on PAIRED deltas, one per prompt, because+the two arms of a prompt ran side by side in the same weather and against the same+fixed judge — that is the whole point of the rig. Anything that breaks the pair+(an outage on either side, a budget blow-out, a judge error) removes the prompt from+the decision instead of leaning it: a missing arm is a missing measurement, not a+zero (``docs/EVAL.md`` §7).++The verdict rule is deliberately blunt and stated once, in :func:`verdict_of`, so a+report can never be argued into "keep" by hand.+"""++from __future__ import annotations++import json+import statistics+from datetime import datetime+from pathlib import Path++from pydantic import BaseModel, Field++from bench._compare_report import CellResult++CONTROL = "control"+VARIANT = "variant"+ARMS = (CONTROL, VARIANT)++#: a paired delta at or below this is a regression on that prompt. Wider than the+#: fixed judge's own sample noise (n_samples 2 on pro, ~±0.02), so one unlucky verdict+#: is not a regression but a real loss on one prompt is.+REGRESSION_DELTA = -0.03+#: mean paired delta needed to call the change a win / a loss (symmetric)+KEEP_DELTA = 0.02+REVERT_DELTA = -0.02+#: this many regressing prompts is a revert even when the mean looks fine+REVERT_REGRESSIONS = 2+++class PairOutcome(BaseModel):+ prompt_id: str+ tier: str = ""+ control: CellResult | None = None+ variant: CellResult | None = None+ paired: bool = Field(default=False, description="both arms produced a fixed-judge score")+ reason: str = Field(default="", description="why the prompt is NOT paired (empty when it is)")+ delta: float | None = Field(default=None, description="variant - control, only when paired")++ @property+ def regression(self) -> bool:+ return self.paired and self.delta is not None and self.delta <= REGRESSION_DELTA+++class Verdict(BaseModel):+ decision: str = Field(description="keep | revert | inconclusive")+ n_pairs: int+ mean_delta: float | None+ median_delta: float | None+ regressions: list[str] = Field(default_factory=list, description="prompt ids with delta <= REGRESSION_DELTA")+ reason: str = ""+++def _scored(r: CellResult | None) -> bool:+ return r is not None and r.score is not None+++def pair_up(rows: list[CellResult], prompt_order: list[tuple[str, str]] | None = None) -> list[PairOutcome]:+ """Fold (prompt, arm) cells into one :class:`PairOutcome` per prompt.++ ``prompt_order`` (id, tier) fixes the row order of the report; only prompts that+ have at least one cell appear (a prompt never selected is not "unpaired"), and+ prompts absent from the order are appended first-seen. Latest row wins per+ (prompt, arm) — the results file is append-only and a redo re-appends.+ """+ by: dict[tuple[str, str], CellResult] = {(r.prompt_id, r.arm): r for r in rows}+ present = {r.prompt_id for r in rows}+ ids: list[tuple[str, str]] = [(i, t) for i, t in (prompt_order or []) if i in present]+ seen = {i for i, _ in ids}+ for r in rows:+ if r.prompt_id not in seen:+ ids.append((r.prompt_id, r.tier))+ seen.add(r.prompt_id)+ out: list[PairOutcome] = []+ for pid, tier in ids:+ c, v = by.get((pid, CONTROL)), by.get((pid, VARIANT))+ p = PairOutcome(prompt_id=pid, tier=tier, control=c, variant=v)+ if _scored(c) and _scored(v):+ p.paired, p.delta = True, round(v.score - c.score, 4) # type: ignore[union-attr]+ else:+ p.reason = "; ".join(f"{name} {'missing' if r is None else r.status}"+ for name, r in ((CONTROL, c), (VARIANT, v)) if not _scored(r))+ out.append(p)+ return out+++def verdict_of(pairs: list[PairOutcome]) -> Verdict:+ """keep iff mean delta >= +0.02 AND no regression; revert iff mean delta <= -0.02+ OR >= 2 regressions; otherwise inconclusive. No pairs is inconclusive, never keep."""+ deltas = [p.delta for p in pairs if p.paired and p.delta is not None]+ regressions = [p.prompt_id for p in pairs if p.regression]+ if not deltas:+ return Verdict(decision="inconclusive", n_pairs=0, mean_delta=None, median_delta=None,+ reason="no prompt has both arms scored")+ mean, median = round(statistics.fmean(deltas), 4), round(statistics.median(deltas), 4)+ v = Verdict(decision="inconclusive", n_pairs=len(deltas), mean_delta=mean, median_delta=median, regressions=regressions)+ if mean <= REVERT_DELTA or len(regressions) >= REVERT_REGRESSIONS:+ v.decision, v.reason = "revert", (f"mean delta {mean:+.3f} <= {REVERT_DELTA:+.2f}" if mean <= REVERT_DELTA+ else f"{len(regressions)} regressions (>= {REVERT_REGRESSIONS})")+ elif mean >= KEEP_DELTA and not regressions:+ v.decision, v.reason = "keep", f"mean delta {mean:+.3f} >= {KEEP_DELTA:+.2f} and no regression"+ else:+ v.reason = (f"mean delta {mean:+.3f} inside ({REVERT_DELTA:+.2f}, {KEEP_DELTA:+.2f})" if not regressions+ else f"mean delta {mean:+.3f} but {len(regressions)} regression(s): {', '.join(regressions)}")+ return v+++class ArmSummary(BaseModel):+ arm: str+ n: int+ n_scored: int+ n_infra_failed: int+ n_budget_exhausted: int+ n_other_unscored: int+ mean_score: float | None+ median_score: float | None+ cost_usd: float = Field(description="generation + fixed-judge spend over every cell that ran")+ mean_minutes: float+++def arm_summary(rows: list[CellResult], arm: str) -> ArmSummary:+ rs = [r for r in rows if r.arm == arm]+ ran = [r for r in rs if r.status != "infra_failed"]+ scores = [r.score for r in ran if r.score is not None]+ return ArmSummary(+ arm=arm, n=len(rs), n_scored=len(scores), n_infra_failed=len(rs) - len(ran),+ n_budget_exhausted=sum(r.status == "budget_exhausted" for r in ran),+ n_other_unscored=sum(r.score is None and r.status != "budget_exhausted" for r in ran),+ mean_score=round(statistics.fmean(scores), 4) if scores else None,+ median_score=round(statistics.median(scores), 4) if scores else None,+ cost_usd=round(sum(r.gen_cost_usd + r.judge_cost_usd for r in ran), 4),+ mean_minutes=round(statistics.fmean(r.wall_s for r in ran) / 60, 2) if ran else 0.0,+ )+++def _fmt(x: float | None, signed: bool = False) -> str:+ if x is None:+ return "—"+ return f"{x:+.3f}" if signed else f"{x:.3f}"+++def _cell(r: CellResult | None) -> str:+ if r is None:+ return "missing"+ return _fmt(r.score) if r.score is not None else r.status+++def render_summary(pairs: list[PairOutcome], rows: list[CellResult], *, title: str, variant_env: dict[str, str],+ generator: str, judge: str, rounds: int) -> str:+ """The Markdown report; pure so tests can read it without a filesystem."""+ v = verdict_of(pairs)+ arms = [arm_summary(rows, a) for a in ARMS]+ lines = [f"# A/B: {title}", "",+ f"variant env: `{' '.join(f'{k}={val}' for k, val in variant_env.items()) or '(none)'}` ",+ f"generator: `{generator}` · fixed judge: `{judge}` · rounds: {rounds} · "+ f"written {datetime.now().strftime('%Y-%m-%d %H:%M')}", "",+ f"## Verdict: **{v.decision}**", "",+ f"{v.reason}. Paired prompts: {v.n_pairs} · mean delta {_fmt(v.mean_delta, True)} · "+ f"median delta {_fmt(v.median_delta, True)} · regressions (delta <= {REGRESSION_DELTA:+.2f}): "+ f"{', '.join(v.regressions) or 'none'}", "",+ "## Arms", "",+ "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | cost $ | mean min |",+ "|---|---|---|---|---|---|---|---|---|---|"]+ for a in arms:+ lines.append(f"| {a.arm} | {a.n} | {a.n_scored} | {a.n_infra_failed} | {a.n_budget_exhausted} | "+ f"{a.n_other_unscored} | {_fmt(a.mean_score)} | {_fmt(a.median_score)} | {a.cost_usd:.2f} | {a.mean_minutes:.1f} |")+ lines += ["", "## Per prompt", "", "| prompt | tier | control | variant | delta | note |", "|---|---|---|---|---|---|"]+ for p in pairs:+ note = "REGRESSION" if p.regression else ("" if p.paired else f"unpaired: {p.reason}")+ lines.append(f"| {p.prompt_id} | {p.tier} | {_cell(p.control)} | {_cell(p.variant)} | {_fmt(p.delta, True)} | {note} |")+ dropped = sum(a.n_infra_failed for a in arms)+ lines += ["", f"n_infra_failed: {dropped}" + (" — re-run with `--redo-status infra_failed` before trusting the verdict"+ if dropped else ""), "",+ "Rule: keep iff mean delta >= +0.02 and no regression; revert iff mean delta <= -0.02 or >= 2 regressions; "+ "else inconclusive. A prompt counts only when BOTH arms were scored by the fixed judge.", ""]+ return "\n".join(lines)+++def write_report(out: Path, rows: list[CellResult], prompt_order: list[tuple[str, str]], **meta: object) -> Verdict:+ """Write ``pairs.json`` + ``summary.md`` under ``out`` and return the verdict."""+ pairs = pair_up(rows, prompt_order)+ v = verdict_of(pairs)+ (out / "pairs.json").write_text(+ json.dumps({"verdict": v.model_dump(mode="json"),+ "pairs": [p.model_dump(mode="json") for p in pairs]}, indent=2))+ (out / "summary.md").write_text(render_summary(pairs, rows, **meta)) # type: ignore[arg-type]+ return v+++__all__ = ["ARMS", "CONTROL", "KEEP_DELTA", "REGRESSION_DELTA", "REVERT_DELTA", "REVERT_REGRESSIONS", "VARIANT",+ "ArmSummary", "PairOutcome", "Verdict", "arm_summary", "pair_up", "render_summary", "verdict_of",+ "write_report"]diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.pynew file mode 100644index 00000000..d52deba6--- /dev/null+++ b/harness/bench/ab_plan.py@@ -0,0 +1,341 @@+"""Controlled A/B of a plan / brief change: control vs variant, paired per prompt.++ python bench/ab_plan.py --prompts bench/prompts/static_objects_v2.yaml \\+ --variant-env CV3D_PLAN_BRIEF=on --out bench/out/ab_brief \\+ [--generator api-agent:gemini:gemini-3.7-flash] [--judge gemini:gemini-3.1-pro-preview] \\+ [--rounds 2] [--max-usd 2.5] [--ids a,b] [--wait-for-provider 60] [--redo-status infra_failed]++The only thing that differs between the arms is the ``--variant-env`` block, applied to+the VARIANT arm alone. Everything else is held fixed on both sides: the generator, the+planner, the in-loop judge, the rounds and dollars, and the fixed judge that produces+the reported score (``bench/_fixed_eval.py``, ``n_samples`` 2).++Why each arm is a child PROCESS rather than a thread: the switches under test are read+from the environment (``codeverse.tracks.brief.brief_enabled`` reads ``os.environ`` at+call time; anything under ``Settings`` is read once through an ``lru_cache``), so two+threads in one interpreter cannot hold different values of them. A child gets exactly+the env its arm needs and nothing leaks across. Each child owns a key pool, so the+driver caps every child at ``CV3D_RATE__MAX_IN_FLIGHT`` (default 16) — two children at+16 stay inside the 64 knee measured for one process (``docs/COST.md`` §23).++Why the arms launch as simultaneous PAIRS: provider weather changes by the hour, and a+verdict that compares an arm run in a storm against one run in the clear measures the+storm. Prompt N's control and variant start together and the next prompt waits for+both (``--parallel`` is fixed at 2 by construction). A cell lost to an outage is+``infra_failed`` and removes its prompt from the pairing, never scores it 0+(``docs/EVAL.md`` §7); ``--redo-status infra_failed`` re-runs BOTH arms of that prompt so+the pair is re-measured together.++Layout of ``--out``: ``ab.json`` (the arms, options and variant env) ·+``results.jsonl`` (one ``CellResult`` per (prompt, arm); resume source) ·+``arms/<arm>/cells/<prompt>/<slug>/{run,eval,cell.json,worker.log}`` · ``pairs.json`` ·+``summary.md``.+"""++from __future__ import annotations++import argparse+import json+import os+import subprocess+import sys+import time+from collections.abc import Callable, Sequence+from concurrent.futures import ThreadPoolExecutor+from datetime import datetime+from pathlib import Path++from pydantic import BaseModel, Field++from codeverse._compat import UTC++REPO = Path(__file__).resolve().parents[1]+if str(REPO) not in sys.path: # `python bench/ab_plan.py` from anywhere+ sys.path.insert(0, str(REPO))++from bench._ab_report import ARMS, CONTROL, VARIANT, Verdict, write_report # noqa: E402+from bench._compare_report import CellResult, load_jsonl # noqa: E402+from bench._infra import is_infra_failure # noqa: E402+from bench.compare_backends import ( # noqa: E402+ Arm,+ CompareDeps,+ CompareOptions,+ _preflight,+ run_cell,+)+from bench.run_bench import Battery, BenchPrompt, select_prompts # noqa: E402++DEFAULT_GENERATOR = "api-agent:gemini:gemini-3.7-flash"+DEFAULT_JUDGE = "gemini:gemini-3.1-pro-preview"+#: the env name Settings actually reads (``Rate.max_in_flight`` under ``CV3D_`` + ``__``).+#: NOT ``CV3D_MAX_IN_FLIGHT``: that flat name is read by nothing.+MAX_IN_FLIGHT_ENV = "CV3D_RATE__MAX_IN_FLIGHT"+DEFAULT_MAX_IN_FLIGHT = 16+#: only ever 2 — one control + one variant, launched together (see module docstring)+PARALLEL = 2+++class AbOptions(BaseModel):+ generator: str = DEFAULT_GENERATOR+ judge: str = DEFAULT_JUDGE+ loop_judge: str | None = Field(default=None, description="in-loop judge for BOTH arms (None → settings default)")+ planner: str | None = Field(default=None, description="planner for BOTH arms (None → settings default)")+ rounds: int = 2+ max_usd: float = 2.5+ max_minutes: float = 45.0+ n_samples: int = 2+ ids: list[str] = Field(default_factory=list)+ tiers: list[str] = Field(default_factory=list)+ limit: int | None = None+ resume: bool = True+ redo_status: list[str] = Field(default_factory=list, description="re-run prompts where EITHER arm has one of these")+ variant_env: dict[str, str] = Field(default_factory=dict, description="applied to the variant arm only")+ max_in_flight: int = DEFAULT_MAX_IN_FLIGHT++ def compare_options(self) -> CompareOptions:+ """The per-cell options handed to ``compare_backends.run_cell`` (both arms identical)."""+ return CompareOptions(judge=self.judge, loop_judge=self.loop_judge, planner=self.planner, rounds=self.rounds,+ max_usd=self.max_usd, max_minutes=self.max_minutes, n_samples=self.n_samples,+ resume=self.resume, pairwise=False)+++def parse_variant_env(items: Sequence[str]) -> dict[str, str]:+ """``["K=V", "K2=V2"]`` → ``{K: V, K2: V2}``; a bare ``K`` is rejected, not defaulted."""+ out: dict[str, str] = {}+ for it in items:+ k, sep, v = it.partition("=")+ if not sep or not k.strip():+ raise ValueError(f"--variant-env needs KEY=VALUE, got {it!r}")+ out[k.strip()] = v+ return out+++def child_env(arm: str, opts: AbOptions, base: dict[str, str] | None = None) -> dict[str, str]:+ """The environment of one arm's child process.++ The variant gets ``variant_env`` on top of the driver's env; the control gets the+ driver's env with those same keys REMOVED, so a switch that happens to be set in+ the launching shell cannot silently turn the control into a second variant. Both+ get the in-flight cap unless the driver already set one.+ """+ env = dict(os.environ if base is None else base)+ for k in opts.variant_env:+ env.pop(k, None)+ if arm == VARIANT:+ env.update(opts.variant_env)+ env.setdefault(MAX_IN_FLIGHT_ENV, str(opts.max_in_flight))+ env["PYTHONUNBUFFERED"] = "1"+ return env+++def arm_dir(out: Path, arm: str) -> Path:+ return out / "arms" / arm+++def cell_dir(out: Path, arm: str, item_id: str, generator: str) -> Path:+ """Where ``compare_backends.run_cell`` puts this arm's cell (its layout, our root)."""+ return arm_dir(out, arm) / "cells" / item_id / _harness_arm(generator).slug+++def _harness_arm(generator: str) -> Arm:+ return Arm(raw=f"harness:{generator}", kind="harness", target=generator)+++def worker_argv(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> list[str]:+ return [sys.executable, str(Path(__file__).resolve()), "cell", "--prompts", str(battery_path), "--out", str(out),+ "--id", item_id, "--arm", arm, "--options-json", opts.model_dump_json()]+++# ----------------------------------------------------------------------------- one cell, in a child+CellRunner = Callable[[Path, Path, BenchPrompt, str, AbOptions], CellResult]+++def spawn_cell(battery_path: Path, out: Path, item: BenchPrompt, arm: str, opts: AbOptions) -> CellResult:+ """Run one (prompt, arm) cell in a child process and read back its ``cell.json``.++ A child that dies without writing a cell (killed, import error, unhandled outage)+ is classified by the tail of its log with the same rule every other failure path+ uses, so an outage stays ``infra_failed`` and never becomes a scored zero.+ """+ cell = cell_dir(out, arm, item.id, opts.generator)+ cell.mkdir(parents=True, exist_ok=True)+ log, marker = cell / "worker.log", cell / "cell.json"+ marker.unlink(missing_ok=True) # a stale verdict from an earlier attempt must not stand in for this one+ t0 = time.time()+ with log.open("a") as fh:+ fh.write(f"--- {datetime.now(UTC).isoformat()} {arm} {item.id}\n")+ fh.flush()+ rc = subprocess.run(worker_argv(battery_path, out, item.id, arm, opts), cwd=REPO, env=child_env(arm, opts),+ stdout=fh, stderr=subprocess.STDOUT, check=False).returncode+ if marker.is_file():+ return CellResult.model_validate_json(marker.read_text())+ tail = log.read_text()[-1500:] if log.is_file() else ""+ return CellResult(prompt_id=item.id, tier=item.tier, arm=arm, kind="harness", target=opts.generator,+ judge=opts.judge, workspace=str(cell), wall_s=round(time.time() - t0, 1),+ status="infra_failed" if is_infra_failure(tail) else "error",+ error=f"worker exited {rc} without a cell.json; log tail: {tail[-400:]!r}")+++def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOptions) -> int:+ """Child entry: the harness run + fixed evaluation of one arm, via ``run_cell``."""+ from bench._fixed_eval import FixedEvaluator++ battery = Battery.load(battery_path)+ item = next((p for p in battery.prompts if p.id == item_id), None)+ if item is None:+ print(f"no prompt {item_id!r} in {battery_path}", file=sys.stderr)+ return 2+ deps = CompareDeps(FixedEvaluator(opts.judge, n_samples=opts.n_samples))+ res = run_cell(battery, item, _harness_arm(opts.generator), arm_dir(out, arm), opts.compare_options(), deps)+ res.arm, res.target = arm, opts.generator # the row is keyed by A/B arm, not by generator+ (Path(res.workspace) / "cell.json").write_text(res.model_dump_json(indent=1))+ print(f"cell {arm} {item_id}: status={res.status} score={res.score} ${res.gen_cost_usd + res.judge_cost_usd:.2f}")+ return 0+++# ----------------------------------------------------------------------------- the driver+def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[tuple[BenchPrompt, list[str]]]:+ """Which (prompt, arms) still need running.++ A prompt with EITHER arm in ``redo_status`` is re-run on BOTH arms — a redo exists+ to re-measure the pair together, not to patch one side in different weather. A+ prompt with only one arm recorded (the driver was interrupted mid-pair) runs only+ the missing arm; its partner already finished and re-running it would pay twice+ for the same generation without making the weather match.+ """+ todo: list[tuple[BenchPrompt, list[str]]] = []+ redo = set(opts.redo_status)+ for p in select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit):+ rows = {a: done.get((p.id, a)) for a in ARMS}+ if redo and any(r is not None and r.status in redo for r in rows.values()):+ for a in ARMS:+ done.pop((p.id, a), None)+ todo.append((p, list(ARMS)))+ continue+ missing = [a for a in ARMS if rows[a] is None]+ if missing:+ todo.append((p, missing))+ return todo+++def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, run_cell_fn: CellRunner = spawn_cell,+ on_result: Callable[[CellResult], None] | None = None) -> Verdict:+ """Run (or resume) the A/B, one prompt-pair at a time; write pairs.json + summary.md."""+ battery_path, out = Path(battery_path), Path(out_dir)+ battery = Battery.load(battery_path)+ out.mkdir(parents=True, exist_ok=True)+ (out / "ab.json").write_text(json.dumps({"battery": battery.name, "prompts": str(battery_path),+ "arms": {CONTROL: {}, VARIANT: opts.variant_env},+ "options": opts.model_dump(mode="json"),+ "started_at": datetime.now(UTC).isoformat()}, indent=2))+ results = out / "results.jsonl"+ done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}+ todo = _plan_todo(battery, done, opts)+ with ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:+ for item, arms in todo: # one pair at a time: both arms of a prompt see the same weather+ futs = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]+ for fut in futs:+ r = fut.result()+ done[(r.prompt_id, r.arm)] = r+ fh.write(r.model_dump_json() + "\n")+ fh.flush()+ if on_result:+ on_result(r)+ order = [(p.id, p.tier) for p in battery.prompts]+ return write_report(out, list(done.values()), order, title=f"{battery.name} / {out.name}",+ variant_env=opts.variant_env, generator=opts.generator, judge=opts.judge, rounds=opts.rounds)+++def preflight(opts: AbOptions, *, wait_minutes: float) -> bool:+ """Probe every model either arm needs (planner, generator, loop judge, fixed judge)."""+ return _preflight(opts.judge, [_harness_arm(opts.generator)], opts.compare_options(), wait_minutes=wait_minutes)+++# ----------------------------------------------------------------------------- CLI+def _parser() -> argparse.ArgumentParser:+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])+ sub = ap.add_subparsers(dest="cmd")+ w = sub.add_parser("cell", help="(internal) run one arm of one prompt in this process")+ w.add_argument("--prompts", required=True)+ w.add_argument("--out", required=True)+ w.add_argument("--id", required=True)+ w.add_argument("--arm", required=True, choices=ARMS)+ w.add_argument("--options-json", required=True)+ ap.add_argument("--prompts")+ ap.add_argument("--out")+ ap.add_argument("--variant-env", action="append", default=[], metavar="KEY=VALUE",+ help="env applied to the VARIANT arm only (repeatable)")+ ap.add_argument("--generator", default=DEFAULT_GENERATOR)+ ap.add_argument("--judge", default=DEFAULT_JUDGE, help="FIXED judge model id (both arms)")+ ap.add_argument("--loop-judge", default=None)+ ap.add_argument("--planner", default=None)+ ap.add_argument("--rounds", type=int, default=2)+ ap.add_argument("--max-usd", type=float, default=2.5)+ ap.add_argument("--max-minutes", type=float, default=45.0)+ ap.add_argument("--n-samples", type=int, default=2)+ ap.add_argument("--ids", default="")+ ap.add_argument("--tiers", default="")+ ap.add_argument("--limit", type=int, default=None)+ ap.add_argument("--max-in-flight", type=int, default=DEFAULT_MAX_IN_FLIGHT, help=f"per child ({MAX_IN_FLIGHT_ENV})")+ ap.add_argument("--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)")+ ap.add_argument("--redo-status", default="", help="comma list, e.g. infra_failed (re-runs both arms of the prompt)")+ ap.add_argument("--no-resume", action="store_true")+ ap.add_argument("--no-preflight", action="store_true", help="skip the provider health check (do not)")+ ap.add_argument("--wait-for-provider", type=float, default=0.0, metavar="MIN")+ ap.add_argument("--allow-siblings", action="store_true",+ help="launch even when other harness processes are running (they share the quota)")+ ap.add_argument("--report-only", action="store_true", help="only rebuild pairs.json / summary.md")+ return ap+++def main(argv: Sequence[str] | None = None) -> int:+ ns = _parser().parse_args(argv)+ if ns.cmd == "cell":+ return cell_main(Path(ns.prompts), Path(ns.out), ns.id, ns.arm, AbOptions.model_validate_json(ns.options_json))+ if not ns.prompts or not ns.out:+ _parser().error("--prompts and --out are required")+ if ns.parallel != PARALLEL:+ _parser().error(f"--parallel must be {PARALLEL}: the arms of a prompt run together and nothing else does")+ opts = AbOptions(generator=ns.generator, judge=ns.judge, loop_judge=ns.loop_judge, planner=ns.planner,+ rounds=ns.rounds, max_usd=ns.max_usd, max_minutes=ns.max_minutes, n_samples=ns.n_samples,+ ids=[i for i in ns.ids.split(",") if i], tiers=[t for t in ns.tiers.split(",") if t],+ limit=ns.limit, resume=not ns.no_resume, redo_status=[s for s in ns.redo_status.split(",") if s],+ variant_env=parse_variant_env(ns.variant_env), max_in_flight=ns.max_in_flight)+ out = Path(ns.out)+ if ns.report_only:+ battery = Battery.load(ns.prompts)+ v = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],+ title=f"{battery.name} / {out.name}", variant_env=opts.variant_env, generator=opts.generator,+ judge=opts.judge, rounds=opts.rounds)+ print(f"verdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")+ return 0+ if not opts.variant_env:+ _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the judge")+ from codeverse.models.health import sibling_processes++ if (sib := sibling_processes()) and not ns.allow_siblings:+ print(f"refusing to start: {sib} other harness process(es) are running and would share the key quota "+ "(docs/COST.md §23). Wait for them, or pass --allow-siblings.", flush=True)+ return 3+ if not ns.no_preflight and not preflight(opts, wait_minutes=ns.wait_for_provider):+ return 2++ def _log(r: CellResult) -> None:+ print(f"[{datetime.now().strftime('%H:%M:%S')}] {r.prompt_id:28s} {r.arm:8s} score={r.score} "+ f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}", flush=True)++ v = run_ab(ns.prompts, out, opts, on_result=_log)+ print(f"\nverdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")+ lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]+ if lost:+ print(f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "+ f"those prompts with: --redo-status infra_failed")+ return 0+++__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "PARALLEL", "AbOptions", "CellRunner",+ "cell_dir", "cell_main", "child_env", "main", "parse_variant_env", "preflight", "run_ab", "spawn_cell",+ "worker_argv"]++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/harness/codeverse/cli/doctor.py b/harness/codeverse/cli/doctor.pyindex 38087906..f4d84b81 100644--- a/harness/codeverse/cli/doctor.py+++ b/harness/codeverse/cli/doctor.py@@ -190,7 +190,7 @@ def check_keys(live: bool) -> list[Row]:"OK" if sib == 0 else "WARN",f"{sib} other harness process(es) running — each keeps its OWN key pool, so the real "f"concurrency is ~{(sib + 1) * cap} in-flight against one shared quota "- f"(docs/COST.md §23; run one battery at a time or set CV3D_MAX_IN_FLIGHT={max(1, cap // (sib + 1))})"+ f"(docs/COST.md §23; run one battery at a time or set CV3D_RATE__MAX_IN_FLIGHT={max(1, cap // (sib + 1))})"if sib else "sole harness process: max_in_flight applies as configured",))rows.append(("anthropic key", "OK" if s.anthropic_api_key else "WARN", "set" if s.anthropic_api_key else "not set (anthropic:* backends unavailable)"))diff --git a/harness/codeverse/models/health.py b/harness/codeverse/models/health.pyindex 6a602bd9..c04ecada 100644--- a/harness/codeverse/models/health.py+++ b/harness/codeverse/models/health.py@@ -117,7 +117,7 @@ def _bare_model(model_id: str, timeout_s: float):#: whole command line: the repo path itself contains "3dcodeverse", so a substring test#: counts every shell that merely `cd`s into the tree._MODULE = "codeverse.cli.main"-_SCRIPTS = ("compare_backends.py", "run_bench.py")+_SCRIPTS = ("compare_backends.py", "run_bench.py", "ab_plan.py")_ENTRY_POINTS = ("3dcv", "3dcodeverse")diff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 9025fbae..39277867 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1082,7 +1082,8 @@ Three consequences:setting where it could not win. Its negative result stands for the configurationtested and should NOT be read as "back-pressure does not help".3. **Operational rule until a cross-process limiter exists**: run ONE battery at a time,- or set `CV3D_MAX_IN_FLIGHT` to `64 / (concurrent processes)`.+ or set `CV3D_RATE__MAX_IN_FLIGHT` (the name `Settings.rate` actually reads; the flat+ `CV3D_MAX_IN_FLIGHT` is read by nothing) to `64 / (concurrent processes)`.The real fix is a machine-wide limiter — a file-locked token bucket under`~/.cache/codeverse/` that every process shares — so the quota is enforced where itdiff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.pynew file mode 100644index 00000000..2e1d2a70--- /dev/null+++ b/harness/tests/compare_bench/test_ab_plan.py@@ -0,0 +1,259 @@+"""bench/ab_plan.py — pairing, verdict rule, outage exclusion, env isolation (offline)."""++from __future__ import annotations++import json+import os+import sys+from pathlib import Path++import pytest++REPO = Path(__file__).resolve().parents[2]+if str(REPO) not in sys.path:+ sys.path.insert(0, str(REPO))++from bench._ab_report import ( # noqa: E402+ CONTROL,+ VARIANT,+ PairOutcome,+ arm_summary,+ pair_up,+ render_summary,+ verdict_of,+)+from bench._compare_report import CellResult, load_jsonl # noqa: E402+from bench.ab_plan import ( # noqa: E402+ MAX_IN_FLIGHT_ENV,+ AbOptions,+ _plan_todo,+ cell_dir,+ child_env,+ main,+ parse_variant_env,+ run_ab,+ worker_argv,+)+from bench.run_bench import Battery # noqa: E402++BATTERY = REPO / "bench" / "prompts" / "compare_v1.yaml"+++def _row(pid: str, arm: str, score: float | None, status: str = "scored", **kw) -> CellResult:+ return CellResult(prompt_id=pid, arm=arm, kind="harness", score=score, status=status,+ gen_cost_usd=kw.pop("cost", 1.0), judge_cost_usd=0.1, wall_s=kw.pop("wall", 120.0), **kw)+++def _pairs(*deltas: float) -> list[PairOutcome]:+ return [PairOutcome(prompt_id=f"p{i}", paired=True, delta=d) for i, d in enumerate(deltas)]+++# ----------------------------------------------------------------------------- verdict rule+def test_keep_needs_mean_gain_and_no_regression():+ assert verdict_of(_pairs(0.03, 0.02, 0.01, 0.04)).decision == "keep"+ # one regression blocks a keep even when the mean clears the bar+ v = verdict_of(_pairs(0.10, 0.10, 0.10, -0.03))+ assert v.decision == "inconclusive" and v.regressions == ["p3"]+ # a mean just under the bar is not a keep+ assert verdict_of(_pairs(0.01, 0.02, 0.02, 0.01)).decision == "inconclusive"+++def test_revert_on_mean_loss_or_two_regressions():+ assert verdict_of(_pairs(-0.02, -0.02, -0.02)).decision == "revert"+ v = verdict_of(_pairs(0.20, 0.20, -0.03, -0.05)) # mean +0.08 but two prompts lost+ assert v.decision == "revert" and v.regressions == ["p2", "p3"]+ assert verdict_of(_pairs(-0.019, 0.0)).decision == "inconclusive"+++def test_no_pairs_is_inconclusive_never_keep():+ v = verdict_of([PairOutcome(prompt_id="p0", paired=False, reason="control infra_failed")])+ assert v.decision == "inconclusive" and v.n_pairs == 0 and v.mean_delta is None+++# ----------------------------------------------------------------------------- pairing+def test_a_pair_counts_only_when_both_arms_scored():+ rows = [_row("a", CONTROL, 0.70), _row("a", VARIANT, 0.75),+ _row("b", CONTROL, 0.80), _row("b", VARIANT, None, "infra_failed"),+ _row("c", CONTROL, None, "budget_exhausted"), _row("c", VARIANT, 0.9),+ _row("d", CONTROL, 0.5)]+ pairs = {p.prompt_id: p for p in pair_up(rows)}+ assert pairs["a"].paired and pairs["a"].delta == pytest.approx(0.05)+ assert not pairs["b"].paired and pairs["b"].reason == "variant infra_failed"+ assert not pairs["c"].paired and pairs["c"].reason == "control budget_exhausted"+ assert not pairs["d"].paired and pairs["d"].reason == "variant missing"+ v = verdict_of(list(pairs.values()))+ assert v.n_pairs == 1 and v.mean_delta == pytest.approx(0.05)+++def test_latest_row_wins_and_order_follows_the_battery():+ rows = [_row("a", CONTROL, 0.1), _row("b", CONTROL, 0.2), _row("b", VARIANT, 0.2),+ _row("a", VARIANT, 0.1), _row("a", CONTROL, 0.9)] # re-appended on a redo+ pairs = pair_up(rows, [("b", "easy"), ("a", "hard")])+ assert [p.prompt_id for p in pairs] == ["b", "a"] and pairs[1].tier == "hard"+ assert pairs[1].delta == pytest.approx(0.1 - 0.9)+++def test_outage_cells_are_excluded_from_every_arm_rate():+ """An arm hit by weather must summarise the same as one that ran in the clear, and+ the loss must stay visible (docs/EVAL.md §7)."""+ clear = [_row(f"p{i}", CONTROL, 0.8) for i in range(4)]+ unlucky = [_row(f"p{i}", VARIANT, 0.8) for i in range(4)] + [_row("p9", VARIANT, None, "infra_failed", wall=3600, cost=0.0)]+ c, v = arm_summary(clear + unlucky, CONTROL), arm_summary(clear + unlucky, VARIANT)+ assert c.mean_score == v.mean_score and c.mean_minutes == v.mean_minutes and c.cost_usd == v.cost_usd+ assert v.n_infra_failed == 1 and v.n == 5 and v.n_scored == 4+ md = render_summary(pair_up(clear + unlucky), clear + unlucky, title="t", variant_env={"X": "1"},+ generator="g", judge="j", rounds=2)+ assert "n_infra_failed: 1" in md and "--redo-status infra_failed" in md and "variant env: `X=1`" in md+ assert "| p9 | | missing | infra_failed | — | unpaired: control missing; variant infra_failed |" in md+++def test_summary_marks_regressions_and_states_the_rule():+ rows = [_row("a", CONTROL, 0.8), _row("a", VARIANT, 0.7), _row("b", CONTROL, 0.5), _row("b", VARIANT, 0.6)]+ md = render_summary(pair_up(rows), rows, title="t", variant_env={}, generator="g", judge="j", rounds=1)+ assert "| a | | 0.800 | 0.700 | -0.100 | REGRESSION |" in md and "## Verdict: **inconclusive**" in md+ assert "keep iff mean delta >= +0.02" in md and "(none)" in md+++# ----------------------------------------------------------------------------- env isolation+def test_variant_env_is_applied_to_the_variant_arm_only():+ opts = AbOptions(variant_env={"CV3D_PLAN_BRIEF": "on"})+ base = {"PATH": "/bin", "CV3D_PLAN_BRIEF": "on"} # leaked from the launching shell+ c, v = child_env(CONTROL, opts, base), child_env(VARIANT, opts, base)+ assert v["CV3D_PLAN_BRIEF"] == "on"+ assert "CV3D_PLAN_BRIEF" not in c, "the control must never inherit the switch under test"+ assert c["PATH"] == v["PATH"] == "/bin"+ assert c[MAX_IN_FLIGHT_ENV] == v[MAX_IN_FLIGHT_ENV] == "16"+ assert MAX_IN_FLIGHT_ENV == "CV3D_RATE__MAX_IN_FLIGHT", "the name Settings.rate actually reads"+ # a cap set by the driver's own environment is respected, not overwritten+ assert child_env(VARIANT, opts, {MAX_IN_FLIGHT_ENV: "4"})[MAX_IN_FLIGHT_ENV] == "4"+ assert child_env(VARIANT, opts, {})["PYTHONUNBUFFERED"] == "1"+++def test_the_in_flight_env_name_is_one_settings_reads(monkeypatch):+ from codeverse.config import get_settings++ monkeypatch.setenv(MAX_IN_FLIGHT_ENV, "7")+ get_settings.cache_clear()+ try:+ assert get_settings().rate.max_in_flight == 7+ finally:+ get_settings.cache_clear()+++def test_parse_variant_env():+ assert parse_variant_env(["A=1", "B=x=y", "C="]) == {"A": "1", "B": "x=y", "C": ""}+ for bad in (["A"], ["=1"]):+ with pytest.raises(ValueError):+ parse_variant_env(bad)+++def test_worker_argv_carries_the_whole_option_block(tmp_path: Path):+ opts = AbOptions(variant_env={"K": "v"}, rounds=1)+ argv = worker_argv(BATTERY, tmp_path, "cmp_easy_stool", VARIANT, opts)+ assert argv[1].endswith("ab_plan.py") and argv[2] == "cell" and "--arm" in argv+ assert AbOptions.model_validate_json(argv[-1]) == opts+ assert cell_dir(tmp_path, CONTROL, "x", "api-agent:gemini:g").parent.parts[-4:] == ("arms", "control", "cells", "x")+++def test_a_child_that_dies_without_a_cell_is_an_error_not_a_score(tmp_path: Path):+ """Real subprocess, no network: an unknown prompt id makes the worker exit 2 before any+ model is touched, and the driver must record that as a scoreless error cell."""+ from bench.ab_plan import spawn_cell+ from bench.run_bench import BenchPrompt++ ghost = BenchPrompt(id="no_such_prompt", prompt="x", tier="easy")+ res = spawn_cell(BATTERY, tmp_path, ghost, CONTROL, AbOptions(variant_env={"K": "v"}))+ assert res.status == "error" and res.score is None and res.arm == CONTROL+ assert "worker exited 2" in res.error and "no prompt 'no_such_prompt'" in res.error+ assert (Path(res.workspace) / "worker.log").is_file()+++# ----------------------------------------------------------------------------- the driver, with a fake cell runner+class FakeCells:+ """Records (prompt, arm, env-at-call) and answers from a script; the pair order is observable."""++ def __init__(self, scores: dict[tuple[str, str], float | None | str]):+ self.scores = scores+ self.calls: list[tuple[str, str]] = []++ def __call__(self, battery_path, out, item, arm, opts) -> CellResult:+ self.calls.append((item.id, arm))+ env = child_env(arm, opts, {})+ s = self.scores.get((item.id, arm), 0.5)+ if isinstance(s, str):+ return _row(item.id, arm, None, s, tier=item.tier, workspace=env.get("CV3D_PLAN_BRIEF", ""))+ return _row(item.id, arm, s, tier=item.tier, workspace=env.get("CV3D_PLAN_BRIEF", ""))+++def test_driver_runs_pairs_in_prompt_order_and_writes_the_verdict(tmp_path: Path):+ b = Battery.load(BATTERY)+ ids = [p.id for p in b.prompts[:3]]+ fake = FakeCells({(ids[0], VARIANT): 0.55, (ids[1], VARIANT): 0.53, (ids[2], VARIANT): 0.52})+ opts = AbOptions(variant_env={"CV3D_PLAN_BRIEF": "on"}, ids=ids)+ v = run_ab(BATTERY, tmp_path, opts, run_cell_fn=fake)+ # both arms of prompt N before anything of prompt N+1+ assert [c[0] for c in fake.calls] == [i for i in ids for _ in range(2)]+ assert {c[1] for c in fake.calls} == {CONTROL, VARIANT}+ assert v.decision == "keep" and v.n_pairs == 3 and v.mean_delta == pytest.approx(0.0333, abs=1e-3)+ rows = load_jsonl(tmp_path / "results.jsonl", CellResult)+ assert len(rows) == 6+ # the env reached only the variant cells+ assert {r.workspace for r in rows if r.arm == VARIANT} == {"on"}+ assert {r.workspace for r in rows if r.arm == CONTROL} == {""}+ pairs = json.loads((tmp_path / "pairs.json").read_text())+ assert pairs["verdict"]["decision"] == "keep" and len(pairs["pairs"]) == 3+ assert (tmp_path / "summary.md").read_text().startswith("# A/B: compare_v1")+ assert json.loads((tmp_path / "ab.json").read_text())["arms"] == {"control": {}, "variant": {"CV3D_PLAN_BRIEF": "on"}}+++def test_resume_skips_done_pairs_and_redo_reruns_both_arms(tmp_path: Path):+ b = Battery.load(BATTERY)+ ids = [p.id for p in b.prompts[:2]]+ first = FakeCells({(ids[1], VARIANT): "infra_failed"})+ opts = AbOptions(variant_env={"K": "v"}, ids=ids)+ v = run_ab(BATTERY, tmp_path, opts, run_cell_fn=first)+ assert v.n_pairs == 1, "the outage pair must not count"+ # plain resume: nothing left to do+ again = FakeCells({})+ run_ab(BATTERY, tmp_path, opts, run_cell_fn=again)+ assert again.calls == []+ # redo: the whole pair of the hit prompt, nothing else+ redo = FakeCells({})+ v = run_ab(BATTERY, tmp_path, opts.model_copy(update={"redo_status": ["infra_failed"]}), run_cell_fn=redo)+ assert sorted(redo.calls) == [(ids[1], CONTROL), (ids[1], VARIANT)]+ assert v.n_pairs == 2+ rows = load_jsonl(tmp_path / "results.jsonl", CellResult)+ latest = {(r.prompt_id, r.arm): r for r in rows}+ assert latest[(ids[1], VARIANT)].status == "scored" and len(rows) == 6+++def test_an_interrupted_pair_runs_only_its_missing_arm():+ b = Battery.load(BATTERY)+ p = b.prompts[0]+ done = {(p.id, CONTROL): _row(p.id, CONTROL, 0.5)}+ todo = _plan_todo(b, done, AbOptions(ids=[p.id]))+ assert todo == [(p, [VARIANT])]+ assert _plan_todo(b, {(p.id, CONTROL): _row(p.id, CONTROL, 0.5), (p.id, VARIANT): _row(p.id, VARIANT, 0.5)},+ AbOptions(ids=[p.id])) == []+++def test_cli_refuses_identical_arms_and_wrong_parallel(tmp_path: Path, capsys):+ with pytest.raises(SystemExit):+ main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--no-preflight"])+ with pytest.raises(SystemExit):+ main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--variant-env", "K=v", "--parallel", "8"])+++def test_cli_report_only_rebuilds_from_results(tmp_path: Path, capsys):+ (tmp_path / "results.jsonl").write_text("".join(r.model_dump_json() + "\n" for r in+ [_row("cmp_easy_stool", CONTROL, 0.5), _row("cmp_easy_stool", VARIANT, 0.4)]))+ assert main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--report-only"]) == 0+ assert "verdict: revert" in capsys.readouterr().out+ assert (tmp_path / "pairs.json").is_file()+++def test_ab_plan_children_count_as_harness_processes():+ from codeverse.models.health import _is_harness_argv++ assert _is_harness_argv([sys.executable, str(REPO / "bench" / "ab_plan.py"), "cell", "--arm", "control"])+ assert os.environ is not None