Official Codebase of 3DCodeVerse
Python
ab_plan: the A/A run scored +0.344 and the verdict rule said "keep"
gaoypengece1f1dcb6a6
diff --git a/harness/bench/_ab_report.py b/harness/bench/_ab_report.pyindex fa6b4876..6d3d931c 100644--- a/harness/bench/_ab_report.py+++ b/harness/bench/_ab_report.py@@ -14,6 +14,7 @@ report can never be argued into "keep" by hand.from __future__ import annotationsimport json+import mathimport statisticsfrom datetime import datetimefrom pathlib import Path@@ -35,6 +36,19 @@ KEEP_DELTA = 0.02REVERT_DELTA = -0.02#: this many regressing prompts is a revert even when the mean looks fineREVERT_REGRESSIONS = 2+#: how many standard errors the mean delta must clear before the rig claims to have+#: SEPARATED the change from run-to-run noise. 2 is the usual ~95 % two-sided band.+#:+#: This is reported, never enforced: the keep/revert rule above is the protocol+#: (docs/EVAL.md §7) and a report must not be able to argue itself out of it. But the+#: rule alone is dangerously confident. Measured 2026-08-24: an A/A run of this rig —+#: `CV3D_PLAN_FEATURES=all` when nothing on main read that variable, so both arms were+#: byte-identical code — scored control 0.591 vs variant 0.934 on one prompt and the+#: verdict line read **keep, mean delta +0.344**. The generator is stochastic; a paired+#: delta carries the spread of two independent generations, not the judge's ±0.02 sample+#: noise. So every summary states the spread beside the verdict, and says plainly when+#: the decision threshold sits inside it.+NOISE_SIGMAS = 2.0class PairOutcome(BaseModel):@@ -58,6 +72,24 @@ class Verdict(BaseModel):median_delta: float | Noneregressions: list[str] = Field(default_factory=list, description="prompt ids with delta <= REGRESSION_DELTA")reason: str = ""+ sd_delta: float | None = Field(default=None, description="stdev of the paired deltas — the rig's own noise (None at n<2)")+ se_delta: float | None = Field(default=None, description="standard error of the mean delta, sd/sqrt(n)")+ separated: bool = Field(default=False, description="|mean delta| >= NOISE_SIGMAS * se: the decision is outside the noise")+ n_for_power: int | None = Field(default=None, description="pairs this spread needs for NOISE_SIGMAS*se to fit inside KEEP_DELTA")++ @property+ def caution(self) -> str:+ """The sentence a reader needs beside a decision that the data cannot support."""+ if self.n_pairs == 0:+ return ""+ if self.se_delta is None:+ return ("one pair cannot separate a change from run-to-run noise: an A/A run of this rig "+ "(identical arms) measured a paired delta of +0.344 on a single prompt.")+ if self.separated:+ return ""+ need = f"~{self.n_for_power} paired prompts" if self.n_for_power else "more paired prompts"+ return (f"NOT separated from noise: |mean delta| < {NOISE_SIGMAS:g} SE ({NOISE_SIGMAS * self.se_delta:+.3f}). "+ f"At this spread {need} would be needed to resolve {KEEP_DELTA:+.2f}. Treat the decision as a screen.")def _scored(r: CellResult | None) -> bool:@@ -102,7 +134,8 @@ def verdict_of(pairs: list[PairOutcome]) -> Verdict:return Verdict(decision="inconclusive", n_pairs=0, mean_delta=None, median_delta=None,reason="no prompt has both arms scored")mean, median = round(statistics.fmean(deltas), 4), round(statistics.median(deltas), 4)- v = Verdict(decision="inconclusive", n_pairs=len(deltas), mean_delta=mean, median_delta=median, regressions=regressions)+ v = Verdict(decision="inconclusive", n_pairs=len(deltas), mean_delta=mean, median_delta=median,+ regressions=regressions, **_spread(deltas, mean))if mean <= REVERT_DELTA or len(regressions) >= REVERT_REGRESSIONS:v.decision, v.reason = "revert", (f"mean delta {mean:+.3f} <= {REVERT_DELTA:+.2f}" if mean <= REVERT_DELTAelse f"{len(regressions)} regressions (>= {REVERT_REGRESSIONS})")@@ -114,6 +147,23 @@ def verdict_of(pairs: list[PairOutcome]) -> Verdict:return v+def _spread(deltas: list[float], mean: float) -> dict[str, object]:+ """The noise block of a :class:`Verdict`: how wide the paired deltas are, and what+ that width does to the decision.++ ``n_for_power`` inverts the standard error: to get ``NOISE_SIGMAS * sd/sqrt(n)`` down+ to ``KEEP_DELTA`` you need ``n = (NOISE_SIGMAS*sd/KEEP_DELTA)**2`` pairs. It is the+ honest answer to "would one more prompt settle this?" and it is usually brutal.+ """+ if len(deltas) < 2:+ return {"sd_delta": None, "se_delta": None, "separated": False, "n_for_power": None}+ sd = statistics.stdev(deltas)+ se = sd / math.sqrt(len(deltas))+ return {"sd_delta": round(sd, 4), "se_delta": round(se, 4),+ "separated": abs(mean) >= NOISE_SIGMAS * se,+ "n_for_power": math.ceil((NOISE_SIGMAS * sd / KEEP_DELTA) ** 2) if sd > 0 else 1}++class ArmSummary(BaseModel):arm: strn: int@@ -155,21 +205,36 @@ def _cell(r: CellResult | None) -> str:def render_summary(pairs: list[PairOutcome], rows: list[CellResult], *, title: str, variant_env: dict[str, str],- generator: str, judge: str, rounds: int) -> str:- """The Markdown report; pure so tests can read it without a filesystem."""+ generator: str, judge: str, rounds: int, aa: bool = False) -> str:+ """The Markdown report; pure so tests can read it without a filesystem.++ ``aa`` marks a calibration run whose arms are deliberately identical: there the+ measured delta IS the noise floor, and any decision word would be a lie.+ """v = verdict_of(pairs)arms = [arm_summary(rows, a) for a in ARMS]- lines = [f"# A/B: {title}", "",- f"variant env: `{' '.join(f'{k}={val}' for k, val in variant_env.items()) or '(none)'}` ",- f"generator: `{generator}` · fixed judge: `{judge}` · rounds: {rounds} · "- f"written {datetime.now().strftime('%Y-%m-%d %H:%M')}", "",- f"## Verdict: **{v.decision}**", "",- f"{v.reason}. Paired prompts: {v.n_pairs} · mean delta {_fmt(v.mean_delta, True)} · "- f"median delta {_fmt(v.median_delta, True)} · regressions (delta <= {REGRESSION_DELTA:+.2f}): "- f"{', '.join(v.regressions) or 'none'}", "",- "## Arms", "",- "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | cost $ | mean min |",- "|---|---|---|---|---|---|---|---|---|---|"]+ lines = [f"# A/{'A' if aa else 'B'}: {title}", ""]+ if aa:+ lines += ["> **A/A calibration — the arms are identical.** Every delta below is pure run-to-run",+ "> noise; the verdict word is printed only to show what this rig would have concluded",+ "> from nothing. Use the spread as the floor an A/B has to clear.", ""]+ lines += [+ f"variant env: `{' '.join(f'{k}={val}' for k, val in variant_env.items()) or '(none)'}` ",+ f"generator: `{generator}` · fixed judge: `{judge}` · rounds: {rounds} · "+ f"written {datetime.now().strftime('%Y-%m-%d %H:%M')}", "",+ f"## Verdict: **{v.decision}**", "",+ f"{v.reason}. Paired prompts: {v.n_pairs} · mean delta {_fmt(v.mean_delta, True)} · "+ f"median delta {_fmt(v.median_delta, True)} · regressions (delta <= {REGRESSION_DELTA:+.2f}): "+ f"{', '.join(v.regressions) or 'none'}", "",+ "## Confidence", "",+ f"paired sd {_fmt(v.sd_delta)} · SE {_fmt(v.se_delta)} · {NOISE_SIGMAS:g} SE band "+ f"{_fmt(v.mean_delta, True)} ± {_fmt(NOISE_SIGMAS * v.se_delta if v.se_delta else None)} · "+ f"separated from noise: {'yes' if v.separated else 'NO'}",+ *([f"\n**{v.caution}**"] if v.caution else []), "",+ "## Arms", "",+ "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | cost $ | mean min |",+ "|---|---|---|---|---|---|---|---|---|---|",+ ]for a in arms:lines.append(f"| {a.arm} | {a.n} | {a.n_scored} | {a.n_infra_failed} | {a.n_budget_exhausted} | "f"{a.n_other_unscored} | {_fmt(a.mean_score)} | {_fmt(a.median_score)} | {a.cost_usd:.2f} | {a.mean_minutes:.1f} |")@@ -196,6 +261,7 @@ def write_report(out: Path, rows: list[CellResult], prompt_order: list[tuple[strreturn v-__all__ = ["ARMS", "CONTROL", "KEEP_DELTA", "REGRESSION_DELTA", "REVERT_DELTA", "REVERT_REGRESSIONS", "VARIANT",+__all__ = ["ARMS", "CONTROL", "KEEP_DELTA", "NOISE_SIGMAS", "REGRESSION_DELTA", "REVERT_DELTA",+ "REVERT_REGRESSIONS", "VARIANT","ArmSummary", "PairOutcome", "Verdict", "arm_summary", "pair_up", "render_summary", "verdict_of","write_report"]diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.pyindex 4a949236..7cf100ac 100644--- a/harness/bench/ab_plan.py+++ b/harness/bench/ab_plan.py@@ -24,7 +24,15 @@ storm. Prompt N's control and variant start together and the next prompt waitsboth (``--parallel`` is fixed at 2 by construction). A cell lost to an outage is``infra_failed`` and removes its prompt from the pairing, never scores it 0(``docs/EVAL.md`` §7); ``--redo-status infra_failed`` re-runs BOTH arms of that prompt so-the pair is re-measured together.+the pair is re-measured together, archiving the previous attempt first so the surviving+arm regenerates instead of resuming its old answer (:func:`archive_cell`).++What the verdict does NOT tell you: the generator is stochastic, so a paired delta carries+the spread of two independent generations, not the judge's ±0.02 sampling noise. Run+``--aa`` (both arms identical) on the same battery to measure that floor; every summary+prints the paired sd, the 2 SE band and how many pairs this spread would need before+±0.02 is resolvable (``bench/_ab_report.NOISE_SIGMAS``). Measured 2026-08-24: one A/A+prompt came back +0.344 and the rule said "keep".Layout of ``--out``: ``ab.json`` (the arms, options and variant env) ·``results.jsonl`` (one ``CellResult`` per (prompt, arm); resume source) ·@@ -43,7 +51,9 @@ import timefrom collections.abc import Callable, Sequencefrom concurrent.futures import ThreadPoolExecutorfrom datetime import datetime+from itertools import countfrom pathlib import Path+from typing import NamedTuplefrom pydantic import BaseModel, Field@@ -93,6 +103,8 @@ class AbOptions(BaseModel):redo_status: list[str] = Field(default_factory=list, description="re-run prompts where EITHER arm has one of these")variant_env: dict[str, str] = Field(default_factory=dict, description="applied to the variant arm only")max_in_flight: int = DEFAULT_MAX_IN_FLIGHT+ aa: bool = Field(default=False, description="calibration: run BOTH arms as the control, so the delta IS the noise")+ redo_fresh: bool = Field(default=True, description="a redo regenerates both arms instead of resuming them")def compare_options(self) -> CompareOptions:"""The per-cell options handed to ``compare_backends.run_cell`` (both arms identical)."""@@ -119,11 +131,15 @@ def child_env(arm: str, opts: AbOptions, base: dict[str, str] | None = None) ->driver's env with those same keys REMOVED, so a switch that happens to be set inthe launching shell cannot silently turn the control into a second variant. Bothget the in-flight cap unless the driver already set one.++ Under ``--aa`` the variant arm gets the CONTROL environment: the two arms then run+ byte-identical code and the measured delta is the rig's own noise floor, which is the+ number every A/B verdict has to be read against (``bench/_ab_report.NOISE_SIGMAS``)."""env = dict(os.environ if base is None else base)for k in opts.variant_env:env.pop(k, None)- if arm == VARIANT:+ if arm == VARIANT and not opts.aa:env.update(opts.variant_env)env.setdefault(MAX_IN_FLIGHT_ENV, str(opts.max_in_flight))env["PYTHONUNBUFFERED"] = "1"@@ -196,7 +212,15 @@ def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOpt# ----------------------------------------------------------------------------- the driver-def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[tuple[BenchPrompt, list[str]]]:+class Todo(NamedTuple):+ """One unit of work: a prompt, the arms of it still owed, and whether to start over."""++ item: BenchPrompt+ arms: list[str]+ fresh: bool = False+++def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[Todo]:"""Which (prompt, arms) still need running.A prompt with EITHER arm in ``redo_status`` is re-run on BOTH arms — a redo exists@@ -205,21 +229,46 @@ def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts:the missing arm; its partner already finished and re-running it would pay twicefor the same generation without making the weather match."""- todo: list[tuple[BenchPrompt, list[str]]] = []+ todo: list[Todo] = []redo = set(opts.redo_status)for p in select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit):rows = {a: done.get((p.id, a)) for a in ARMS}if redo and any(r is not None and r.status in redo for r in rows.values()):for a in ARMS:done.pop((p.id, a), None)- todo.append((p, list(ARMS)))+ todo.append(Todo(p, list(ARMS), fresh=opts.redo_fresh))continuemissing = [a for a in ARMS if rows[a] is None]if missing:- todo.append((p, missing))+ todo.append(Todo(p, missing))return todo+def archive_cell(out: Path, arm: str, item_id: str, opts: AbOptions) -> Path | None:+ """Move a finished attempt aside so the next one REGENERATES; ``None`` if there was none.++ ``compare_backends._run_harness`` resumes whenever ``<cell>/run`` exists — it never+ consults ``resume`` — which is right for a harness run recovering from a crash and+ wrong for this rig. Without this, ``--redo-status infra_failed`` re-ran the pair on+ paper only: the arm that had SUCCEEDED resumed its finished workspace and handed back+ its old score, generated in the old weather, while its partner generated afresh in+ today's. That is precisely the cross-weather comparison the pairing exists to+ prevent, and it was invisible in the report.++ The old attempt is renamed, never deleted: it is the evidence for the outage that+ caused the redo, and it holds that attempt's cost ledger.+ """+ cell = cell_dir(out, arm, item_id, opts.generator)+ if not cell.exists():+ return None+ for n in count(1):+ dest = cell.with_name(f"{cell.name}.attempt{n}")+ if not dest.exists():+ cell.rename(dest)+ return dest+ raise AssertionError("unreachable") # pragma: no cover — count() is infinite++def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, run_cell_fn: CellRunner = spawn_cell,on_result: Callable[[CellResult], None] | None = None) -> Verdict:"""Run (or resume) the A/B, one prompt-pair at a time; write pairs.json + summary.md."""@@ -227,14 +276,18 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, rubattery = Battery.load(battery_path)out.mkdir(parents=True, exist_ok=True)(out / "ab.json").write_text(json.dumps({"battery": battery.name, "prompts": str(battery_path),- "arms": {CONTROL: {}, VARIANT: opts.variant_env},+ "aa": opts.aa,+ "arms": {CONTROL: {}, VARIANT: {} if opts.aa else opts.variant_env},"options": opts.model_dump(mode="json"),"started_at": datetime.now(UTC).isoformat()}, indent=2))results = out / "results.jsonl"done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}todo = _plan_todo(battery, done, opts)with ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:- for item, arms in todo: # one pair at a time: both arms of a prompt see the same weather+ for item, arms, fresh in todo: # one pair at a time: both arms of a prompt see the same weather+ if fresh:+ for a in arms:+ archive_cell(out, a, item.id, opts)futs = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]for fut in futs:r = fut.result()@@ -245,7 +298,8 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, ruon_result(r)order = [(p.id, p.tier) for p in battery.prompts]return write_report(out, list(done.values()), order, title=f"{battery.name} / {out.name}",- variant_env=opts.variant_env, generator=opts.generator, judge=opts.judge, rounds=opts.rounds)+ variant_env={} if opts.aa else opts.variant_env, generator=opts.generator,+ judge=opts.judge, rounds=opts.rounds, aa=opts.aa)def preflight(opts: AbOptions, *, wait_minutes: float) -> bool:@@ -281,6 +335,11 @@ def _parser() -> argparse.ArgumentParser:ap.add_argument("--max-in-flight", type=int, default=DEFAULT_MAX_IN_FLIGHT, help=f"per child ({MAX_IN_FLIGHT_ENV})")ap.add_argument("--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)")ap.add_argument("--redo-status", default="", help="comma list, e.g. infra_failed (re-runs both arms of the prompt)")+ ap.add_argument("--redo-resume", action="store_true",+ help="let a redo RESUME the old cells instead of regenerating them (cheap, but the pair then "+ "spans two weathers — see archive_cell)")+ ap.add_argument("--aa", action="store_true",+ help="calibration run: both arms identical, so the measured delta IS this rig's noise floor")ap.add_argument("--no-resume", action="store_true")ap.add_argument("--no-preflight", action="store_true", help="skip the provider health check (do not)")ap.add_argument("--wait-for-provider", type=float, default=0.0, metavar="MIN")@@ -302,17 +361,20 @@ def main(argv: Sequence[str] | None = None) -> int:rounds=ns.rounds, max_usd=ns.max_usd, max_minutes=ns.max_minutes, n_samples=ns.n_samples,ids=[i for i in ns.ids.split(",") if i], tiers=[t for t in ns.tiers.split(",") if t],limit=ns.limit, resume=not ns.no_resume, redo_status=[s for s in ns.redo_status.split(",") if s],- variant_env=parse_variant_env(ns.variant_env), max_in_flight=ns.max_in_flight)+ variant_env=parse_variant_env(ns.variant_env), max_in_flight=ns.max_in_flight,+ aa=ns.aa, redo_fresh=not ns.redo_resume)out = Path(ns.out)if ns.report_only:battery = Battery.load(ns.prompts)v = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],- title=f"{battery.name} / {out.name}", variant_env=opts.variant_env, generator=opts.generator,- judge=opts.judge, rounds=opts.rounds)- print(f"verdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")+ title=f"{battery.name} / {out.name}", variant_env={} if opts.aa else opts.variant_env,+ generator=opts.generator, judge=opts.judge, rounds=opts.rounds, aa=opts.aa)+ print(f"verdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")+ + f"\n{out / 'summary.md'}")return 0- if not opts.variant_env:- _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the judge")+ if not opts.variant_env and not opts.aa:+ _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the noise "+ "(pass --aa if measuring the noise is the point)")from codeverse.models.health import pool_budget# the rule is a BUDGET, not a head-count: the provider sees one machine, so the sum of@@ -330,7 +392,8 @@ def main(argv: Sequence[str] | None = None) -> int:f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}", flush=True)v = run_ab(ns.prompts, out, opts, on_result=_log)- print(f"\nverdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")+ print(f"\nverdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")+ + f"\n{out / 'summary.md'}")lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]if lost:print(f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "@@ -338,9 +401,9 @@ def main(argv: Sequence[str] | None = None) -> int:return 0-__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "PARALLEL", "AbOptions", "CellRunner",- "cell_dir", "cell_main", "child_env", "main", "parse_variant_env", "preflight", "run_ab", "spawn_cell",- "worker_argv"]+__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "PARALLEL", "AbOptions", "CellRunner", "Todo",+ "archive_cell", "cell_dir", "cell_main", "child_env", "main", "parse_variant_env", "preflight", "run_ab",+ "spawn_cell", "worker_argv"]if __name__ == "__main__":raise SystemExit(main())diff --git a/harness/docs/EVAL.md b/harness/docs/EVAL.mdindex 3a38cfd0..4cd60f4b 100644--- a/harness/docs/EVAL.md+++ b/harness/docs/EVAL.md@@ -216,7 +216,38 @@ Rules that follow from it:* `budget_exhausted` is deliberately *not* excused on the build rate: the provider isnot at fault for a model that cannot finish inside the cap.-## 8. Reporting checklist+## 8. The noise floor of a paired A/B (`bench/ab_plan.py`)++`bench/ab_plan.py` runs one switch as control-vs-variant, paired per prompt, both arms+launched together so the weather matches, both scored by the same fixed judge. The+verdict rule is blunt on purpose (`bench/_ab_report.verdict_of`, stated once):++> keep iff mean paired delta ≥ +0.02 **and** no prompt at ≤ −0.03; revert iff mean ≤ −0.02+> **or** ≥ 2 such prompts; else inconclusive. A prompt counts only when BOTH arms scored.++**That rule is a screen, not a proof, and the numbers say by how much.** Measured+2026-08-24: an A/A run — `--variant-env CV3D_PLAN_FEATURES=all` at a time when nothing on+`main` read that variable, so the two arms were byte-identical code — produced+control 0.591 vs variant 0.934 on `ctrl_med_dining_chair`, and the verdict line read+**keep, mean delta +0.344**. Nothing was under test and the rig said ship it.++The reason is structural, not a bug: a paired delta is the difference of two *independent+stochastic generations*, so it carries generation spread, not the fixed judge's ±0.02+sampling noise. The ±0.02 threshold was sized against the judge and is roughly an order+of magnitude too tight for what it is applied to. Consequences:++* every summary now prints a **Confidence** block — paired sd, SE, the 2 SE band, and+ `separated from noise: yes|NO` — beside the verdict, plus `n_for_power`, the number of+ paired prompts this spread would need before ±0.02 is resolvable. At a paired sd of+ ~0.18 that is *hundreds*, not eight;+* run `--aa` on the same battery and the same n to measure the floor before believing a+ win. Both arms get the control environment; the report is titled `A/A` and banners+ itself so no reader can mistake a calibration for a result;+* a `keep` that is not `separated` means "worth another look", never "ship it". Prefer+ changes whose per-prompt deltas are *consistent in sign* over ones with a big mean and+ a big spread — the sign pattern survives this noise where the mean does not.++## 9. Reporting checklistbattery name + git sha of prompts; judge id, rubric name + hash, `n_samples`; per-armgenerator/planner ids; rounds and budget caps; per-tier table (n, mean ± std, passdiff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.pyindex 96655c32..5353f4a0 100644--- a/harness/tests/compare_bench/test_ab_plan.py+++ b/harness/tests/compare_bench/test_ab_plan.py@@ -15,6 +15,8 @@ if str(REPO) not in sys.path:from bench._ab_report import ( # noqa: E402CONTROL,+ KEEP_DELTA,+ NOISE_SIGMAS,VARIANT,PairOutcome,arm_summary,@@ -26,7 +28,9 @@ from bench._compare_report import CellResult, load_jsonl # noqa: E402from bench.ab_plan import ( # noqa: E402MAX_IN_FLIGHT_ENV,AbOptions,+ Todo,_plan_todo,+ archive_cell,cell_dir,child_env,main,@@ -114,6 +118,52 @@ def test_summary_marks_regressions_and_states_the_rule():assert "keep iff mean delta >= +0.02" in md and "(none)" in md+# ----------------------------------------------------------------------------- noise floor+def test_a_verdict_states_the_spread_it_was_decided_on():+ """The rule fires on a mean, and a mean of stochastic generations has a spread. An+ A/A run of this rig (identical arms) returned +0.344 on one prompt and the rule said+ "keep" — so the report has to carry the noise beside the word."""+ v = verdict_of(_pairs(0.30, -0.20, 0.10, 0.05, -0.10, 0.25, 0.02, -0.15))+ assert v.sd_delta is not None and v.se_delta == pytest.approx(v.sd_delta / 8 ** 0.5, abs=1e-3)+ assert not v.separated and "NOT separated from noise" in v.caution+ # the honest answer to "would one more prompt settle this?": at this spread, hundreds+ assert v.n_for_power == pytest.approx((NOISE_SIGMAS * v.sd_delta / KEEP_DELTA) ** 2, rel=0.01)+ assert v.n_for_power > 100+++def test_a_tight_win_is_marked_separated_and_carries_no_caution():+ v = verdict_of(_pairs(0.10, 0.11, 0.09, 0.12))+ assert v.decision == "keep" and v.separated and v.caution == ""+++def test_one_pair_can_never_be_separated_from_noise():+ v = verdict_of(_pairs(0.344))+ assert v.decision == "keep", "the blunt rule still fires — that is exactly the danger"+ assert v.sd_delta is None and not v.separated and "one pair cannot separate" in v.caution+++def test_summary_prints_the_confidence_block():+ rows = [_row(f"p{i}", arm, s) for i, (c, x) in enumerate([(0.5, 0.9), (0.6, 0.3), (0.7, 0.72)])+ for arm, s in ((CONTROL, c), (VARIANT, x))]+ md = render_summary(pair_up(rows), rows, title="t", variant_env={"K": "v"}, generator="g", judge="j", rounds=1)+ assert "## Confidence" in md and "separated from noise: NO" in md and "NOT separated from noise" in md+++def test_an_aa_run_is_labelled_so_nobody_reads_it_as_a_decision():+ rows = [_row("a", CONTROL, 0.59), _row("a", VARIANT, 0.93)]+ md = render_summary(pair_up(rows), rows, title="t", variant_env={}, generator="g", judge="j", rounds=1, aa=True)+ assert md.startswith("# A/A: t") and "A/A calibration — the arms are identical" in md+++def test_aa_gives_both_arms_the_control_environment():+ """Otherwise the calibration measures the switch it is supposed to be blind to."""+ opts = AbOptions(variant_env={"CV3D_PLAN_FEATURES": "all"}, aa=True)+ base = {"PATH": "/bin", "CV3D_PLAN_FEATURES": "all"} # leaked from the launching shell+ c, v = child_env(CONTROL, opts, base), child_env(VARIANT, opts, base)+ assert "CV3D_PLAN_FEATURES" not in c and "CV3D_PLAN_FEATURES" not in v+ assert c == v++# ----------------------------------------------------------------------------- env isolationdef test_variant_env_is_applied_to_the_variant_arm_only():opts = AbOptions(variant_env={"CV3D_PLAN_BRIEF": "on"})@@ -233,11 +283,37 @@ def test_an_interrupted_pair_runs_only_its_missing_arm():p = b.prompts[0]done = {(p.id, CONTROL): _row(p.id, CONTROL, 0.5)}todo = _plan_todo(b, done, AbOptions(ids=[p.id]))- assert todo == [(p, [VARIANT])]+ assert todo == [Todo(p, [VARIANT], fresh=False)], "a finished partner is not regenerated"assert _plan_todo(b, {(p.id, CONTROL): _row(p.id, CONTROL, 0.5), (p.id, VARIANT): _row(p.id, VARIANT, 0.5)},AbOptions(ids=[p.id])) == []+def test_a_redo_is_planned_fresh_so_the_pair_is_regenerated():+ """compare_backends._run_harness resumes on the mere existence of <cell>/run, so+ without the fresh flag a redo would hand back the surviving arm's OLD score — the+ cross-weather comparison the pairing exists to prevent."""+ b = Battery.load(BATTERY)+ p = b.prompts[0]+ done = {(p.id, CONTROL): _row(p.id, CONTROL, 0.5), (p.id, VARIANT): _row(p.id, VARIANT, None, "infra_failed")}+ opts = AbOptions(ids=[p.id], redo_status=["infra_failed"])+ assert _plan_todo(b, dict(done), opts) == [Todo(p, [CONTROL, VARIANT], fresh=True)]+ # --redo-resume opts back into the cheap behaviour, explicitly+ assert _plan_todo(b, dict(done), opts.model_copy(update={"redo_fresh": False})) == [Todo(p, [CONTROL, VARIANT], False)]+++def test_archive_cell_moves_the_old_attempt_aside_and_keeps_it(tmp_path: Path):+ opts = AbOptions(variant_env={"K": "v"})+ cell = cell_dir(tmp_path, CONTROL, "p1", opts.generator)+ (cell / "run").mkdir(parents=True)+ (cell / "run" / "record.json").write_text("{}")+ first = archive_cell(tmp_path, CONTROL, "p1", opts)+ assert first is not None and first.name.endswith(".attempt1") and (first / "run" / "record.json").is_file()+ assert not cell.exists(), "the redo must start from an empty cell or the harness resumes"+ assert archive_cell(tmp_path, CONTROL, "p1", opts) is None # nothing left to archive+ cell.mkdir(parents=True)+ assert archive_cell(tmp_path, CONTROL, "p1", opts).name.endswith(".attempt2") # type: ignore[union-attr]++def test_cli_refuses_identical_arms_and_wrong_parallel(tmp_path: Path, capsys):with pytest.raises(SystemExit):main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--no-preflight"])@@ -245,6 +321,19 @@ def test_cli_refuses_identical_arms_and_wrong_parallel(tmp_path: Path, capsys):main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--variant-env", "K=v", "--parallel", "8"])+def test_aa_run_records_two_empty_arms(tmp_path: Path):+ """--aa is the ONLY sanctioned way to run identical arms, and the record must say so+ or a later reader will mistake a noise measurement for a result."""+ b = Battery.load(BATTERY)+ ids = [p.id for p in b.prompts[:2]]+ fake = FakeCells({(ids[0], VARIANT): 0.9})+ run_ab(BATTERY, tmp_path, AbOptions(variant_env={"CV3D_PLAN_FEATURES": "all"}, aa=True, ids=ids), run_cell_fn=fake)+ meta = json.loads((tmp_path / "ab.json").read_text())+ assert meta["aa"] is True and meta["arms"] == {"control": {}, "variant": {}}+ assert {r.workspace for r in load_jsonl(tmp_path / "results.jsonl", CellResult)} == {""}, "no arm saw the switch"+ assert (tmp_path / "summary.md").read_text().startswith("# A/A: ")++def test_cli_report_only_rebuilds_from_results(tmp_path: Path, capsys):(tmp_path / "results.jsonl").write_text("".join(r.model_dump_json() + "\n" for r in[_row("cmp_easy_stool", CONTROL, 0.5), _row("cmp_easy_stool", VARIANT, 0.4)]))