Skip to content

Official Codebase of 3DCodeVerse

Python

ab_plan: the A/A run scored +0.344 and the verdict rule said "keep"

gaoypengece1f1dcb6a6
diff --git a/harness/bench/_ab_report.py b/harness/bench/_ab_report.py
index fa6b4876..6d3d931c 100644
--- a/harness/bench/_ab_report.py
+++ b/harness/bench/_ab_report.py
@@ -14,6 +14,7 @@ report can never be argued into "keep" by hand.
from __future__ import annotations
import json
+import math
import statistics
from datetime import datetime
from pathlib import Path
@@ -35,6 +36,19 @@ KEEP_DELTA = 0.02
REVERT_DELTA = -0.02
#: this many regressing prompts is a revert even when the mean looks fine
REVERT_REGRESSIONS = 2
+#: how many standard errors the mean delta must clear before the rig claims to have
+#: SEPARATED the change from run-to-run noise. 2 is the usual ~95 % two-sided band.
+#:
+#: This is reported, never enforced: the keep/revert rule above is the protocol
+#: (docs/EVAL.md §7) and a report must not be able to argue itself out of it. But the
+#: rule alone is dangerously confident. Measured 2026-08-24: an A/A run of this rig —
+#: `CV3D_PLAN_FEATURES=all` when nothing on main read that variable, so both arms were
+#: byte-identical code — scored control 0.591 vs variant 0.934 on one prompt and the
+#: verdict line read **keep, mean delta +0.344**. The generator is stochastic; a paired
+#: delta carries the spread of two independent generations, not the judge's ±0.02 sample
+#: noise. So every summary states the spread beside the verdict, and says plainly when
+#: the decision threshold sits inside it.
+NOISE_SIGMAS = 2.0
class PairOutcome(BaseModel):
@@ -58,6 +72,24 @@ class Verdict(BaseModel):
median_delta: float | None
regressions: list[str] = Field(default_factory=list, description="prompt ids with delta <= REGRESSION_DELTA")
reason: str = ""
+ sd_delta: float | None = Field(default=None, description="stdev of the paired deltas — the rig's own noise (None at n<2)")
+ se_delta: float | None = Field(default=None, description="standard error of the mean delta, sd/sqrt(n)")
+ separated: bool = Field(default=False, description="|mean delta| >= NOISE_SIGMAS * se: the decision is outside the noise")
+ n_for_power: int | None = Field(default=None, description="pairs this spread needs for NOISE_SIGMAS*se to fit inside KEEP_DELTA")
+
+ @property
+ def caution(self) -> str:
+ """The sentence a reader needs beside a decision that the data cannot support."""
+ if self.n_pairs == 0:
+ return ""
+ if self.se_delta is None:
+ return ("one pair cannot separate a change from run-to-run noise: an A/A run of this rig "
+ "(identical arms) measured a paired delta of +0.344 on a single prompt.")
+ if self.separated:
+ return ""
+ need = f"~{self.n_for_power} paired prompts" if self.n_for_power else "more paired prompts"
+ return (f"NOT separated from noise: |mean delta| < {NOISE_SIGMAS:g} SE ({NOISE_SIGMAS * self.se_delta:+.3f}). "
+ f"At this spread {need} would be needed to resolve {KEEP_DELTA:+.2f}. Treat the decision as a screen.")
def _scored(r: CellResult | None) -> bool:
@@ -102,7 +134,8 @@ def verdict_of(pairs: list[PairOutcome]) -> Verdict:
return Verdict(decision="inconclusive", n_pairs=0, mean_delta=None, median_delta=None,
reason="no prompt has both arms scored")
mean, median = round(statistics.fmean(deltas), 4), round(statistics.median(deltas), 4)
- v = Verdict(decision="inconclusive", n_pairs=len(deltas), mean_delta=mean, median_delta=median, regressions=regressions)
+ v = Verdict(decision="inconclusive", n_pairs=len(deltas), mean_delta=mean, median_delta=median,
+ regressions=regressions, **_spread(deltas, mean))
if mean <= REVERT_DELTA or len(regressions) >= REVERT_REGRESSIONS:
v.decision, v.reason = "revert", (f"mean delta {mean:+.3f} <= {REVERT_DELTA:+.2f}" if mean <= REVERT_DELTA
else f"{len(regressions)} regressions (>= {REVERT_REGRESSIONS})")
@@ -114,6 +147,23 @@ def verdict_of(pairs: list[PairOutcome]) -> Verdict:
return v
+def _spread(deltas: list[float], mean: float) -> dict[str, object]:
+ """The noise block of a :class:`Verdict`: how wide the paired deltas are, and what
+ that width does to the decision.
+
+ ``n_for_power`` inverts the standard error: to get ``NOISE_SIGMAS * sd/sqrt(n)`` down
+ to ``KEEP_DELTA`` you need ``n = (NOISE_SIGMAS*sd/KEEP_DELTA)**2`` pairs. It is the
+ honest answer to "would one more prompt settle this?" and it is usually brutal.
+ """
+ if len(deltas) < 2:
+ return {"sd_delta": None, "se_delta": None, "separated": False, "n_for_power": None}
+ sd = statistics.stdev(deltas)
+ se = sd / math.sqrt(len(deltas))
+ return {"sd_delta": round(sd, 4), "se_delta": round(se, 4),
+ "separated": abs(mean) >= NOISE_SIGMAS * se,
+ "n_for_power": math.ceil((NOISE_SIGMAS * sd / KEEP_DELTA) ** 2) if sd > 0 else 1}
+
+
class ArmSummary(BaseModel):
arm: str
n: int
@@ -155,21 +205,36 @@ def _cell(r: CellResult | None) -> str:
def render_summary(pairs: list[PairOutcome], rows: list[CellResult], *, title: str, variant_env: dict[str, str],
- generator: str, judge: str, rounds: int) -> str:
- """The Markdown report; pure so tests can read it without a filesystem."""
+ generator: str, judge: str, rounds: int, aa: bool = False) -> str:
+ """The Markdown report; pure so tests can read it without a filesystem.
+
+ ``aa`` marks a calibration run whose arms are deliberately identical: there the
+ measured delta IS the noise floor, and any decision word would be a lie.
+ """
v = verdict_of(pairs)
arms = [arm_summary(rows, a) for a in ARMS]
- lines = [f"# A/B: {title}", "",
- f"variant env: `{' '.join(f'{k}={val}' for k, val in variant_env.items()) or '(none)'}` ",
- f"generator: `{generator}` · fixed judge: `{judge}` · rounds: {rounds} · "
- f"written {datetime.now().strftime('%Y-%m-%d %H:%M')}", "",
- f"## Verdict: **{v.decision}**", "",
- f"{v.reason}. Paired prompts: {v.n_pairs} · mean delta {_fmt(v.mean_delta, True)} · "
- f"median delta {_fmt(v.median_delta, True)} · regressions (delta <= {REGRESSION_DELTA:+.2f}): "
- f"{', '.join(v.regressions) or 'none'}", "",
- "## Arms", "",
- "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | cost $ | mean min |",
- "|---|---|---|---|---|---|---|---|---|---|"]
+ lines = [f"# A/{'A' if aa else 'B'}: {title}", ""]
+ if aa:
+ lines += ["> **A/A calibration — the arms are identical.** Every delta below is pure run-to-run",
+ "> noise; the verdict word is printed only to show what this rig would have concluded",
+ "> from nothing. Use the spread as the floor an A/B has to clear.", ""]
+ lines += [
+ f"variant env: `{' '.join(f'{k}={val}' for k, val in variant_env.items()) or '(none)'}` ",
+ f"generator: `{generator}` · fixed judge: `{judge}` · rounds: {rounds} · "
+ f"written {datetime.now().strftime('%Y-%m-%d %H:%M')}", "",
+ f"## Verdict: **{v.decision}**", "",
+ f"{v.reason}. Paired prompts: {v.n_pairs} · mean delta {_fmt(v.mean_delta, True)} · "
+ f"median delta {_fmt(v.median_delta, True)} · regressions (delta <= {REGRESSION_DELTA:+.2f}): "
+ f"{', '.join(v.regressions) or 'none'}", "",
+ "## Confidence", "",
+ f"paired sd {_fmt(v.sd_delta)} · SE {_fmt(v.se_delta)} · {NOISE_SIGMAS:g} SE band "
+ f"{_fmt(v.mean_delta, True)} ± {_fmt(NOISE_SIGMAS * v.se_delta if v.se_delta else None)} · "
+ f"separated from noise: {'yes' if v.separated else 'NO'}",
+ *([f"\n**{v.caution}**"] if v.caution else []), "",
+ "## Arms", "",
+ "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | cost $ | mean min |",
+ "|---|---|---|---|---|---|---|---|---|---|",
+ ]
for a in arms:
lines.append(f"| {a.arm} | {a.n} | {a.n_scored} | {a.n_infra_failed} | {a.n_budget_exhausted} | "
f"{a.n_other_unscored} | {_fmt(a.mean_score)} | {_fmt(a.median_score)} | {a.cost_usd:.2f} | {a.mean_minutes:.1f} |")
@@ -196,6 +261,7 @@ def write_report(out: Path, rows: list[CellResult], prompt_order: list[tuple[str
return v
-__all__ = ["ARMS", "CONTROL", "KEEP_DELTA", "REGRESSION_DELTA", "REVERT_DELTA", "REVERT_REGRESSIONS", "VARIANT",
+__all__ = ["ARMS", "CONTROL", "KEEP_DELTA", "NOISE_SIGMAS", "REGRESSION_DELTA", "REVERT_DELTA",
+ "REVERT_REGRESSIONS", "VARIANT",
"ArmSummary", "PairOutcome", "Verdict", "arm_summary", "pair_up", "render_summary", "verdict_of",
"write_report"]
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.py
index 4a949236..7cf100ac 100644
--- a/harness/bench/ab_plan.py
+++ b/harness/bench/ab_plan.py
@@ -24,7 +24,15 @@ storm. Prompt N's control and variant start together and the next prompt waits
both (``--parallel`` is fixed at 2 by construction). A cell lost to an outage is
``infra_failed`` and removes its prompt from the pairing, never scores it 0
(``docs/EVAL.md`` §7); ``--redo-status infra_failed`` re-runs BOTH arms of that prompt so
-the pair is re-measured together.
+the pair is re-measured together, archiving the previous attempt first so the surviving
+arm regenerates instead of resuming its old answer (:func:`archive_cell`).
+
+What the verdict does NOT tell you: the generator is stochastic, so a paired delta carries
+the spread of two independent generations, not the judge's ±0.02 sampling noise. Run
+``--aa`` (both arms identical) on the same battery to measure that floor; every summary
+prints the paired sd, the 2 SE band and how many pairs this spread would need before
+±0.02 is resolvable (``bench/_ab_report.NOISE_SIGMAS``). Measured 2026-08-24: one A/A
+prompt came back +0.344 and the rule said "keep".
Layout of ``--out``: ``ab.json`` (the arms, options and variant env) ·
``results.jsonl`` (one ``CellResult`` per (prompt, arm); resume source) ·
@@ -43,7 +51,9 @@ import time
from collections.abc import Callable, Sequence
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
+from itertools import count
from pathlib import Path
+from typing import NamedTuple
from pydantic import BaseModel, Field
@@ -93,6 +103,8 @@ class AbOptions(BaseModel):
redo_status: list[str] = Field(default_factory=list, description="re-run prompts where EITHER arm has one of these")
variant_env: dict[str, str] = Field(default_factory=dict, description="applied to the variant arm only")
max_in_flight: int = DEFAULT_MAX_IN_FLIGHT
+ aa: bool = Field(default=False, description="calibration: run BOTH arms as the control, so the delta IS the noise")
+ redo_fresh: bool = Field(default=True, description="a redo regenerates both arms instead of resuming them")
def compare_options(self) -> CompareOptions:
"""The per-cell options handed to ``compare_backends.run_cell`` (both arms identical)."""
@@ -119,11 +131,15 @@ def child_env(arm: str, opts: AbOptions, base: dict[str, str] | None = None) ->
driver's env with those same keys REMOVED, so a switch that happens to be set in
the launching shell cannot silently turn the control into a second variant. Both
get the in-flight cap unless the driver already set one.
+
+ Under ``--aa`` the variant arm gets the CONTROL environment: the two arms then run
+ byte-identical code and the measured delta is the rig's own noise floor, which is the
+ number every A/B verdict has to be read against (``bench/_ab_report.NOISE_SIGMAS``).
"""
env = dict(os.environ if base is None else base)
for k in opts.variant_env:
env.pop(k, None)
- if arm == VARIANT:
+ if arm == VARIANT and not opts.aa:
env.update(opts.variant_env)
env.setdefault(MAX_IN_FLIGHT_ENV, str(opts.max_in_flight))
env["PYTHONUNBUFFERED"] = "1"
@@ -196,7 +212,15 @@ def cell_main(battery_path: Path, out: Path, item_id: str, arm: str, opts: AbOpt
# ----------------------------------------------------------------------------- the driver
-def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[tuple[BenchPrompt, list[str]]]:
+class Todo(NamedTuple):
+ """One unit of work: a prompt, the arms of it still owed, and whether to start over."""
+
+ item: BenchPrompt
+ arms: list[str]
+ fresh: bool = False
+
+
+def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts: AbOptions) -> list[Todo]:
"""Which (prompt, arms) still need running.
A prompt with EITHER arm in ``redo_status`` is re-run on BOTH arms — a redo exists
@@ -205,21 +229,46 @@ def _plan_todo(battery: Battery, done: dict[tuple[str, str], CellResult], opts:
the missing arm; its partner already finished and re-running it would pay twice
for the same generation without making the weather match.
"""
- todo: list[tuple[BenchPrompt, list[str]]] = []
+ todo: list[Todo] = []
redo = set(opts.redo_status)
for p in select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit):
rows = {a: done.get((p.id, a)) for a in ARMS}
if redo and any(r is not None and r.status in redo for r in rows.values()):
for a in ARMS:
done.pop((p.id, a), None)
- todo.append((p, list(ARMS)))
+ todo.append(Todo(p, list(ARMS), fresh=opts.redo_fresh))
continue
missing = [a for a in ARMS if rows[a] is None]
if missing:
- todo.append((p, missing))
+ todo.append(Todo(p, missing))
return todo
+def archive_cell(out: Path, arm: str, item_id: str, opts: AbOptions) -> Path | None:
+ """Move a finished attempt aside so the next one REGENERATES; ``None`` if there was none.
+
+ ``compare_backends._run_harness`` resumes whenever ``<cell>/run`` exists — it never
+ consults ``resume`` — which is right for a harness run recovering from a crash and
+ wrong for this rig. Without this, ``--redo-status infra_failed`` re-ran the pair on
+ paper only: the arm that had SUCCEEDED resumed its finished workspace and handed back
+ its old score, generated in the old weather, while its partner generated afresh in
+ today's. That is precisely the cross-weather comparison the pairing exists to
+ prevent, and it was invisible in the report.
+
+ The old attempt is renamed, never deleted: it is the evidence for the outage that
+ caused the redo, and it holds that attempt's cost ledger.
+ """
+ cell = cell_dir(out, arm, item_id, opts.generator)
+ if not cell.exists():
+ return None
+ for n in count(1):
+ dest = cell.with_name(f"{cell.name}.attempt{n}")
+ if not dest.exists():
+ cell.rename(dest)
+ return dest
+ raise AssertionError("unreachable") # pragma: no cover — count() is infinite
+
+
def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, run_cell_fn: CellRunner = spawn_cell,
on_result: Callable[[CellResult], None] | None = None) -> Verdict:
"""Run (or resume) the A/B, one prompt-pair at a time; write pairs.json + summary.md."""
@@ -227,14 +276,18 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, ru
battery = Battery.load(battery_path)
out.mkdir(parents=True, exist_ok=True)
(out / "ab.json").write_text(json.dumps({"battery": battery.name, "prompts": str(battery_path),
- "arms": {CONTROL: {}, VARIANT: opts.variant_env},
+ "aa": opts.aa,
+ "arms": {CONTROL: {}, VARIANT: {} if opts.aa else opts.variant_env},
"options": opts.model_dump(mode="json"),
"started_at": datetime.now(UTC).isoformat()}, indent=2))
results = out / "results.jsonl"
done = {(r.prompt_id, r.arm): r for r in load_jsonl(results, CellResult)} if opts.resume else {}
todo = _plan_todo(battery, done, opts)
with ThreadPoolExecutor(max_workers=PARALLEL) as pool, results.open("a") as fh:
- for item, arms in todo: # one pair at a time: both arms of a prompt see the same weather
+ for item, arms, fresh in todo: # one pair at a time: both arms of a prompt see the same weather
+ if fresh:
+ for a in arms:
+ archive_cell(out, a, item.id, opts)
futs = [pool.submit(run_cell_fn, battery_path, out, item, a, opts) for a in arms]
for fut in futs:
r = fut.result()
@@ -245,7 +298,8 @@ def run_ab(battery_path: Path | str, out_dir: Path | str, opts: AbOptions, *, ru
on_result(r)
order = [(p.id, p.tier) for p in battery.prompts]
return write_report(out, list(done.values()), order, title=f"{battery.name} / {out.name}",
- variant_env=opts.variant_env, generator=opts.generator, judge=opts.judge, rounds=opts.rounds)
+ variant_env={} if opts.aa else opts.variant_env, generator=opts.generator,
+ judge=opts.judge, rounds=opts.rounds, aa=opts.aa)
def preflight(opts: AbOptions, *, wait_minutes: float) -> bool:
@@ -281,6 +335,11 @@ def _parser() -> argparse.ArgumentParser:
ap.add_argument("--max-in-flight", type=int, default=DEFAULT_MAX_IN_FLIGHT, help=f"per child ({MAX_IN_FLIGHT_ENV})")
ap.add_argument("--parallel", type=int, default=PARALLEL, help="accepted for symmetry; must be 2 (one pair)")
ap.add_argument("--redo-status", default="", help="comma list, e.g. infra_failed (re-runs both arms of the prompt)")
+ ap.add_argument("--redo-resume", action="store_true",
+ help="let a redo RESUME the old cells instead of regenerating them (cheap, but the pair then "
+ "spans two weathers — see archive_cell)")
+ ap.add_argument("--aa", action="store_true",
+ help="calibration run: both arms identical, so the measured delta IS this rig's noise floor")
ap.add_argument("--no-resume", action="store_true")
ap.add_argument("--no-preflight", action="store_true", help="skip the provider health check (do not)")
ap.add_argument("--wait-for-provider", type=float, default=0.0, metavar="MIN")
@@ -302,17 +361,20 @@ def main(argv: Sequence[str] | None = None) -> int:
rounds=ns.rounds, max_usd=ns.max_usd, max_minutes=ns.max_minutes, n_samples=ns.n_samples,
ids=[i for i in ns.ids.split(",") if i], tiers=[t for t in ns.tiers.split(",") if t],
limit=ns.limit, resume=not ns.no_resume, redo_status=[s for s in ns.redo_status.split(",") if s],
- variant_env=parse_variant_env(ns.variant_env), max_in_flight=ns.max_in_flight)
+ variant_env=parse_variant_env(ns.variant_env), max_in_flight=ns.max_in_flight,
+ aa=ns.aa, redo_fresh=not ns.redo_resume)
out = Path(ns.out)
if ns.report_only:
battery = Battery.load(ns.prompts)
v = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],
- title=f"{battery.name} / {out.name}", variant_env=opts.variant_env, generator=opts.generator,
- judge=opts.judge, rounds=opts.rounds)
- print(f"verdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")
+ title=f"{battery.name} / {out.name}", variant_env={} if opts.aa else opts.variant_env,
+ generator=opts.generator, judge=opts.judge, rounds=opts.rounds, aa=opts.aa)
+ print(f"verdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")
+ + f"\n{out / 'summary.md'}")
return 0
- if not opts.variant_env:
- _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the judge")
+ if not opts.variant_env and not opts.aa:
+ _parser().error("--variant-env KEY=VALUE is required: an A/B with identical arms measures only the noise "
+ "(pass --aa if measuring the noise is the point)")
from codeverse.models.health import pool_budget
# the rule is a BUDGET, not a head-count: the provider sees one machine, so the sum of
@@ -330,7 +392,8 @@ def main(argv: Sequence[str] | None = None) -> int:
f"${r.gen_cost_usd + r.judge_cost_usd:.2f} {r.wall_s / 60:.1f}min {r.status} {r.error[:80]!r}", flush=True)
v = run_ab(ns.prompts, out, opts, on_result=_log)
- print(f"\nverdict: {v.decision} — {v.reason}\n{out / 'summary.md'}")
+ print(f"\nverdict: {v.decision} — {v.reason}" + (f"\nCAUTION: {v.caution}" if v.caution else "")
+ + f"\n{out / 'summary.md'}")
lost = [r for r in load_jsonl(out / "results.jsonl", CellResult) if r.status == "infra_failed"]
if lost:
print(f"{len(lost)} cell(s) lost to provider outages and EXCLUDED from the pairing; re-run both arms of "
@@ -338,9 +401,9 @@ def main(argv: Sequence[str] | None = None) -> int:
return 0
-__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "PARALLEL", "AbOptions", "CellRunner",
- "cell_dir", "cell_main", "child_env", "main", "parse_variant_env", "preflight", "run_ab", "spawn_cell",
- "worker_argv"]
+__all__ = ["DEFAULT_GENERATOR", "DEFAULT_JUDGE", "MAX_IN_FLIGHT_ENV", "PARALLEL", "AbOptions", "CellRunner", "Todo",
+ "archive_cell", "cell_dir", "cell_main", "child_env", "main", "parse_variant_env", "preflight", "run_ab",
+ "spawn_cell", "worker_argv"]
if __name__ == "__main__":
raise SystemExit(main())
diff --git a/harness/docs/EVAL.md b/harness/docs/EVAL.md
index 3a38cfd0..4cd60f4b 100644
--- a/harness/docs/EVAL.md
+++ b/harness/docs/EVAL.md
@@ -216,7 +216,38 @@ Rules that follow from it:
* `budget_exhausted` is deliberately *not* excused on the build rate: the provider is
not at fault for a model that cannot finish inside the cap.
-## 8. Reporting checklist
+## 8. The noise floor of a paired A/B (`bench/ab_plan.py`)
+
+`bench/ab_plan.py` runs one switch as control-vs-variant, paired per prompt, both arms
+launched together so the weather matches, both scored by the same fixed judge. The
+verdict rule is blunt on purpose (`bench/_ab_report.verdict_of`, stated once):
+
+> keep iff mean paired delta ≥ +0.02 **and** no prompt at ≤ −0.03; revert iff mean ≤ −0.02
+> **or** ≥ 2 such prompts; else inconclusive. A prompt counts only when BOTH arms scored.
+
+**That rule is a screen, not a proof, and the numbers say by how much.** Measured
+2026-08-24: an A/A run — `--variant-env CV3D_PLAN_FEATURES=all` at a time when nothing on
+`main` read that variable, so the two arms were byte-identical code — produced
+control 0.591 vs variant 0.934 on `ctrl_med_dining_chair`, and the verdict line read
+**keep, mean delta +0.344**. Nothing was under test and the rig said ship it.
+
+The reason is structural, not a bug: a paired delta is the difference of two *independent
+stochastic generations*, so it carries generation spread, not the fixed judge's ±0.02
+sampling noise. The ±0.02 threshold was sized against the judge and is roughly an order
+of magnitude too tight for what it is applied to. Consequences:
+
+* every summary now prints a **Confidence** block — paired sd, SE, the 2 SE band, and
+ `separated from noise: yes|NO` — beside the verdict, plus `n_for_power`, the number of
+ paired prompts this spread would need before ±0.02 is resolvable. At a paired sd of
+ ~0.18 that is *hundreds*, not eight;
+* run `--aa` on the same battery and the same n to measure the floor before believing a
+ win. Both arms get the control environment; the report is titled `A/A` and banners
+ itself so no reader can mistake a calibration for a result;
+* a `keep` that is not `separated` means "worth another look", never "ship it". Prefer
+ changes whose per-prompt deltas are *consistent in sign* over ones with a big mean and
+ a big spread — the sign pattern survives this noise where the mean does not.
+
+## 9. Reporting checklist
battery name + git sha of prompts; judge id, rubric name + hash, `n_samples`; per-arm
generator/planner ids; rounds and budget caps; per-tier table (n, mean ± std, pass
diff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.py
index 96655c32..5353f4a0 100644
--- a/harness/tests/compare_bench/test_ab_plan.py
+++ b/harness/tests/compare_bench/test_ab_plan.py
@@ -15,6 +15,8 @@ if str(REPO) not in sys.path:
from bench._ab_report import ( # noqa: E402
CONTROL,
+ KEEP_DELTA,
+ NOISE_SIGMAS,
VARIANT,
PairOutcome,
arm_summary,
@@ -26,7 +28,9 @@ from bench._compare_report import CellResult, load_jsonl # noqa: E402
from bench.ab_plan import ( # noqa: E402
MAX_IN_FLIGHT_ENV,
AbOptions,
+ Todo,
_plan_todo,
+ archive_cell,
cell_dir,
child_env,
main,
@@ -114,6 +118,52 @@ def test_summary_marks_regressions_and_states_the_rule():
assert "keep iff mean delta >= +0.02" in md and "(none)" in md
+# ----------------------------------------------------------------------------- noise floor
+def test_a_verdict_states_the_spread_it_was_decided_on():
+ """The rule fires on a mean, and a mean of stochastic generations has a spread. An
+ A/A run of this rig (identical arms) returned +0.344 on one prompt and the rule said
+ "keep" — so the report has to carry the noise beside the word."""
+ v = verdict_of(_pairs(0.30, -0.20, 0.10, 0.05, -0.10, 0.25, 0.02, -0.15))
+ assert v.sd_delta is not None and v.se_delta == pytest.approx(v.sd_delta / 8 ** 0.5, abs=1e-3)
+ assert not v.separated and "NOT separated from noise" in v.caution
+ # the honest answer to "would one more prompt settle this?": at this spread, hundreds
+ assert v.n_for_power == pytest.approx((NOISE_SIGMAS * v.sd_delta / KEEP_DELTA) ** 2, rel=0.01)
+ assert v.n_for_power > 100
+
+
+def test_a_tight_win_is_marked_separated_and_carries_no_caution():
+ v = verdict_of(_pairs(0.10, 0.11, 0.09, 0.12))
+ assert v.decision == "keep" and v.separated and v.caution == ""
+
+
+def test_one_pair_can_never_be_separated_from_noise():
+ v = verdict_of(_pairs(0.344))
+ assert v.decision == "keep", "the blunt rule still fires — that is exactly the danger"
+ assert v.sd_delta is None and not v.separated and "one pair cannot separate" in v.caution
+
+
+def test_summary_prints_the_confidence_block():
+ rows = [_row(f"p{i}", arm, s) for i, (c, x) in enumerate([(0.5, 0.9), (0.6, 0.3), (0.7, 0.72)])
+ for arm, s in ((CONTROL, c), (VARIANT, x))]
+ md = render_summary(pair_up(rows), rows, title="t", variant_env={"K": "v"}, generator="g", judge="j", rounds=1)
+ assert "## Confidence" in md and "separated from noise: NO" in md and "NOT separated from noise" in md
+
+
+def test_an_aa_run_is_labelled_so_nobody_reads_it_as_a_decision():
+ rows = [_row("a", CONTROL, 0.59), _row("a", VARIANT, 0.93)]
+ md = render_summary(pair_up(rows), rows, title="t", variant_env={}, generator="g", judge="j", rounds=1, aa=True)
+ assert md.startswith("# A/A: t") and "A/A calibration — the arms are identical" in md
+
+
+def test_aa_gives_both_arms_the_control_environment():
+ """Otherwise the calibration measures the switch it is supposed to be blind to."""
+ opts = AbOptions(variant_env={"CV3D_PLAN_FEATURES": "all"}, aa=True)
+ base = {"PATH": "/bin", "CV3D_PLAN_FEATURES": "all"} # leaked from the launching shell
+ c, v = child_env(CONTROL, opts, base), child_env(VARIANT, opts, base)
+ assert "CV3D_PLAN_FEATURES" not in c and "CV3D_PLAN_FEATURES" not in v
+ assert c == v
+
+
# ----------------------------------------------------------------------------- env isolation
def test_variant_env_is_applied_to_the_variant_arm_only():
opts = AbOptions(variant_env={"CV3D_PLAN_BRIEF": "on"})
@@ -233,11 +283,37 @@ def test_an_interrupted_pair_runs_only_its_missing_arm():
p = b.prompts[0]
done = {(p.id, CONTROL): _row(p.id, CONTROL, 0.5)}
todo = _plan_todo(b, done, AbOptions(ids=[p.id]))
- assert todo == [(p, [VARIANT])]
+ assert todo == [Todo(p, [VARIANT], fresh=False)], "a finished partner is not regenerated"
assert _plan_todo(b, {(p.id, CONTROL): _row(p.id, CONTROL, 0.5), (p.id, VARIANT): _row(p.id, VARIANT, 0.5)},
AbOptions(ids=[p.id])) == []
+def test_a_redo_is_planned_fresh_so_the_pair_is_regenerated():
+ """compare_backends._run_harness resumes on the mere existence of <cell>/run, so
+ without the fresh flag a redo would hand back the surviving arm's OLD score — the
+ cross-weather comparison the pairing exists to prevent."""
+ b = Battery.load(BATTERY)
+ p = b.prompts[0]
+ done = {(p.id, CONTROL): _row(p.id, CONTROL, 0.5), (p.id, VARIANT): _row(p.id, VARIANT, None, "infra_failed")}
+ opts = AbOptions(ids=[p.id], redo_status=["infra_failed"])
+ assert _plan_todo(b, dict(done), opts) == [Todo(p, [CONTROL, VARIANT], fresh=True)]
+ # --redo-resume opts back into the cheap behaviour, explicitly
+ assert _plan_todo(b, dict(done), opts.model_copy(update={"redo_fresh": False})) == [Todo(p, [CONTROL, VARIANT], False)]
+
+
+def test_archive_cell_moves_the_old_attempt_aside_and_keeps_it(tmp_path: Path):
+ opts = AbOptions(variant_env={"K": "v"})
+ cell = cell_dir(tmp_path, CONTROL, "p1", opts.generator)
+ (cell / "run").mkdir(parents=True)
+ (cell / "run" / "record.json").write_text("{}")
+ first = archive_cell(tmp_path, CONTROL, "p1", opts)
+ assert first is not None and first.name.endswith(".attempt1") and (first / "run" / "record.json").is_file()
+ assert not cell.exists(), "the redo must start from an empty cell or the harness resumes"
+ assert archive_cell(tmp_path, CONTROL, "p1", opts) is None # nothing left to archive
+ cell.mkdir(parents=True)
+ assert archive_cell(tmp_path, CONTROL, "p1", opts).name.endswith(".attempt2") # type: ignore[union-attr]
+
+
def test_cli_refuses_identical_arms_and_wrong_parallel(tmp_path: Path, capsys):
with pytest.raises(SystemExit):
main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--no-preflight"])
@@ -245,6 +321,19 @@ def test_cli_refuses_identical_arms_and_wrong_parallel(tmp_path: Path, capsys):
main(["--prompts", str(BATTERY), "--out", str(tmp_path), "--variant-env", "K=v", "--parallel", "8"])
+def test_aa_run_records_two_empty_arms(tmp_path: Path):
+ """--aa is the ONLY sanctioned way to run identical arms, and the record must say so
+ or a later reader will mistake a noise measurement for a result."""
+ b = Battery.load(BATTERY)
+ ids = [p.id for p in b.prompts[:2]]
+ fake = FakeCells({(ids[0], VARIANT): 0.9})
+ run_ab(BATTERY, tmp_path, AbOptions(variant_env={"CV3D_PLAN_FEATURES": "all"}, aa=True, ids=ids), run_cell_fn=fake)
+ meta = json.loads((tmp_path / "ab.json").read_text())
+ assert meta["aa"] is True and meta["arms"] == {"control": {}, "variant": {}}
+ assert {r.workspace for r in load_jsonl(tmp_path / "results.jsonl", CellResult)} == {""}, "no arm saw the switch"
+ assert (tmp_path / "summary.md").read_text().startswith("# A/A: ")
+
+
def test_cli_report_only_rebuilds_from_results(tmp_path: Path, capsys):
(tmp_path / "results.jsonl").write_text("".join(r.model_dump_json() + "\n" for r in
[_row("cmp_easy_stool", CONTROL, 0.5), _row("cmp_easy_stool", VARIANT, 0.4)]))