Skip to content

Official Codebase of 3DCodeVerse

Python

bench: paired_compare — paired Δ, 95% t-interval and sign test per battery

ziyaocc4963bbd173
diff --git a/harness/bench/paired_compare.py b/harness/bench/paired_compare.py
new file mode 100644
index 00000000..e57d6f36
--- /dev/null
+++ b/harness/bench/paired_compare.py
@@ -0,0 +1,215 @@
+"""Paired analysis of a ``compare_backends`` battery: harness arm − one-shot arm per prompt.
+
+``bench/_compare_report.py`` tabulates arms; this answers the paper's question — *is the
+harness lift separated from noise?* — the way ``docs/report.html`` §6 states it: every
+comparison carries its paired standard error, a 95 % confidence interval and the exact
+two-sided sign test, and a comparison whose interval crosses zero is labelled
+``unsupported`` however good the mean looks.
+
+ python bench/paired_compare.py bench/out/compare_v4 # writes paired.md + paired.json
+
+Pairing rules (the same ones the arm table uses): the append-only ``results.jsonl`` is
+deduplicated per (prompt, arm) with the LAST row winning; a cell that ended
+``infra_failed`` (provider outage) tests nothing about the model and drops the pair; a
+failed build scored 0 by the fixed judge stays in (that is a real outcome); a cell with
+no score for any other reason (budget exhausted with no artifact) drops the pair and is
+counted. Tiers are reported separately because the lift is expected to shrink as
+prompts get easier.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import statistics
+import sys
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+REPO = Path(__file__).resolve().parents[1]
+if str(REPO) not in sys.path: # `python bench/paired_compare.py` from anywhere
+ sys.path.insert(0, str(REPO))
+
+from bench._ab_report import _sign_test # noqa: E402
+from bench._compare_report import CellResult # noqa: E402
+from bench._jsonl import read_jsonl # noqa: E402
+
+# two-sided 97.5 % Student-t quantiles by degrees of freedom (df 1..30, then 40/60/120, ∞);
+# a table, not scipy: the [urdf] extra is optional and this must run on a [dev] install
+_T975 = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262,
+ 10: 2.228, 11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.131, 16: 2.120, 17: 2.110,
+ 18: 2.101, 19: 2.093, 20: 2.086, 21: 2.080, 22: 2.074, 23: 2.069, 24: 2.064, 25: 2.060,
+ 26: 2.056, 27: 2.052, 28: 2.048, 29: 2.045, 30: 2.042, 40: 2.021, 60: 2.000, 120: 1.980}
+
+
+def t975(df: int) -> float:
+ if df <= 0:
+ return math.nan
+ if df in _T975:
+ return _T975[df]
+ for bound in (40, 60, 120):
+ if df < bound:
+ return _T975[bound]
+ return 1.960
+
+
+class PairedStats(BaseModel):
+ """One (harness arm, one-shot arm[, tier]) comparison."""
+
+ harness_arm: str
+ oneshot_arm: str
+ tier: str = Field(default="all")
+ n: int = Field(description="prompts with a score on BOTH arms")
+ dropped_infra: int = Field(default=0, description="pairs lost to a provider outage on either arm")
+ dropped_unscored: int = Field(default=0, description="pairs lost to an unscored, non-outage cell")
+ mean_harness: float | None = None
+ mean_oneshot: float | None = None
+ mean_delta: float | None = None
+ sd_delta: float | None = None
+ se_delta: float | None = None
+ ci95_low: float | None = None
+ ci95_high: float | None = None
+ wins: int = 0
+ losses: int = 0
+ ties: int = 0
+ sign_p: float | None = None
+ pass_rate_harness: float | None = None
+ pass_rate_oneshot: float | None = None
+ build_ok_harness: float | None = None
+ build_ok_oneshot: float | None = None
+ verdict: str = Field(default="", description="supported | unsupported | too few pairs")
+ deltas: dict[str, float] = Field(default_factory=dict, description="prompt id → harness − one-shot")
+
+
+def latest_cells(rows: list[CellResult]) -> dict[tuple[str, str], CellResult]:
+ """(prompt_id, arm) → last recorded row (the journal is append-only; a re-run supersedes)."""
+ out: dict[tuple[str, str], CellResult] = {}
+ for r in rows:
+ out[(r.prompt_id, r.arm)] = r
+ return out
+
+
+def _usable(r: CellResult | None) -> tuple[bool, str]:
+ if r is None:
+ return False, "missing"
+ if r.status == "infra_failed":
+ return False, "infra"
+ if r.score is None:
+ return False, "unscored"
+ return True, ""
+
+
+def paired(cells: dict[tuple[str, str], CellResult], harness_arm: str, oneshot_arm: str, *,
+ tier: str = "all") -> PairedStats:
+ prompts = sorted({p for (p, a) in cells if a in (harness_arm, oneshot_arm)})
+ st = PairedStats(harness_arm=harness_arm, oneshot_arm=oneshot_arm, tier=tier, n=0)
+ hs: list[float] = []
+ os_: list[float] = []
+ hp: list[bool] = []
+ op: list[bool] = []
+ hb: list[bool] = []
+ ob: list[bool] = []
+ for p in prompts:
+ h, o = cells.get((p, harness_arm)), cells.get((p, oneshot_arm))
+ if tier != "all" and (h or o).tier != tier:
+ continue
+ ok_h, why_h = _usable(h)
+ ok_o, why_o = _usable(o)
+ if not (ok_h and ok_o):
+ if "infra" in (why_h, why_o):
+ st.dropped_infra += 1
+ elif "missing" not in (why_h, why_o): # a cell that has not run yet is not a drop
+ st.dropped_unscored += 1
+ continue
+ assert h is not None and o is not None and h.score is not None and o.score is not None
+ hs.append(h.score)
+ os_.append(o.score)
+ st.deltas[p] = round(h.score - o.score, 4)
+ hp.append(bool(h.passed))
+ op.append(bool(o.passed))
+ hb.append(h.build_ok)
+ ob.append(o.build_ok)
+ st.n = len(hs)
+ if st.n == 0:
+ st.verdict = "too few pairs"
+ return st
+ deltas = list(st.deltas.values())
+ st.mean_harness = round(statistics.fmean(hs), 4)
+ st.mean_oneshot = round(statistics.fmean(os_), 4)
+ st.mean_delta = round(statistics.fmean(deltas), 4)
+ st.wins = sum(d > 0 for d in deltas)
+ st.losses = sum(d < 0 for d in deltas)
+ st.ties = st.n - st.wins - st.losses
+ st.sign_p = _sign_test(deltas)["sign_p"] # type: ignore[assignment]
+ st.pass_rate_harness = round(sum(hp) / st.n, 4)
+ st.pass_rate_oneshot = round(sum(op) / st.n, 4)
+ st.build_ok_harness = round(sum(hb) / st.n, 4)
+ st.build_ok_oneshot = round(sum(ob) / st.n, 4)
+ if st.n >= 2:
+ sd = statistics.stdev(deltas)
+ se = sd / math.sqrt(st.n)
+ half = t975(st.n - 1) * se
+ st.sd_delta, st.se_delta = round(sd, 4), round(se, 4)
+ st.ci95_low, st.ci95_high = round(st.mean_delta - half, 4), round(st.mean_delta + half, 4)
+ st.verdict = "supported" if (st.ci95_low > 0 or st.ci95_high < 0) else "unsupported"
+ else:
+ st.verdict = "too few pairs"
+ return st
+
+
+def analyse(rows: list[CellResult]) -> list[PairedStats]:
+ cells = latest_cells(rows)
+ arms = sorted({a for (_, a) in cells})
+ harness = [a for a in arms if a.startswith("harness:")]
+ oneshot = [a for a in arms if a.startswith("oneshot")]
+ tiers = sorted({r.tier for r in cells.values() if r.tier}, key=lambda t: ("easy", "medium", "hard").index(t) if t in ("easy", "medium", "hard") else 9)
+ out: list[PairedStats] = []
+ for ha in harness:
+ for oa in oneshot:
+ out.append(paired(cells, ha, oa))
+ for t in tiers:
+ out.append(paired(cells, ha, oa, tier=t))
+ return out
+
+
+def _f(x: float | None, signed: bool = False) -> str:
+ if x is None:
+ return "—"
+ return f"{x:+.3f}" if signed else f"{x:.3f}"
+
+
+def render_markdown(stats: list[PairedStats], title: str) -> str:
+ lines = [f"# paired harness − one-shot — {title}", "",
+ "Mean Δ with its paired 95 % t-interval and the exact two-sided sign test over prompts scored on BOTH arms; "
+ "`infra` = pairs dropped to a provider outage, `unscored` = pairs dropped to a cell with no score for any other reason. "
+ "A comparison is **supported** only when the interval excludes zero.", "",
+ "| harness arm | one-shot arm | tier | n | infra | unscored | harness | one-shot | Δ | 95 % CI | W/L/T | sign p | pass h/o | build h/o | verdict |",
+ "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
+ for s in stats:
+ ci = f"[{_f(s.ci95_low, True)}, {_f(s.ci95_high, True)}]" if s.ci95_low is not None else "—"
+ lines.append(
+ f"| {s.harness_arm} | {s.oneshot_arm} | {s.tier} | {s.n} | {s.dropped_infra} | {s.dropped_unscored} | "
+ f"{_f(s.mean_harness)} | {_f(s.mean_oneshot)} | {_f(s.mean_delta, True)} | {ci} | {s.wins}/{s.losses}/{s.ties} | "
+ f"{_f(s.sign_p)} | {_f(s.pass_rate_harness)}/{_f(s.pass_rate_oneshot)} | {_f(s.build_ok_harness)}/{_f(s.build_ok_oneshot)} | "
+ f"**{s.verdict}** |")
+ return "\n".join(lines) + "\n"
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ ap.add_argument("out_dir", help="a compare_backends output directory (holds results.jsonl)")
+ ns = ap.parse_args(argv)
+ out = Path(ns.out_dir)
+ rows = read_jsonl(out / "results.jsonl", CellResult)
+ stats = analyse(rows)
+ md = render_markdown(stats, out.name)
+ (out / "paired.md").write_text(md)
+ (out / "paired.json").write_text(json.dumps([s.model_dump(mode="json") for s in stats], indent=2))
+ print(md)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/harness/tests/compare_bench/test_paired_compare.py b/harness/tests/compare_bench/test_paired_compare.py
new file mode 100644
index 00000000..f47b7031
--- /dev/null
+++ b/harness/tests/compare_bench/test_paired_compare.py
@@ -0,0 +1,68 @@
+"""bench/paired_compare.py: paired Δ, t-interval and sign test over a compare battery."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+if str(REPO) not in sys.path:
+ sys.path.insert(0, str(REPO))
+
+from bench._compare_report import CellResult # noqa: E402
+from bench.paired_compare import analyse, latest_cells, main, paired, t975 # noqa: E402
+
+HA, OA = "harness:api-agent:gemini:x", "oneshot:gemini:x"
+
+
+def _cell(pid: str, arm: str, score: float | None, *, tier: str = "hard", status: str = "scored",
+ passed: bool = False, build_ok: bool = True) -> CellResult:
+ return CellResult(prompt_id=pid, tier=tier, arm=arm, kind="harness" if arm.startswith("harness") else "oneshot",
+ status=status, score=score, passed=passed, build_ok=build_ok)
+
+
+def test_paired_stats_ci_sign_test_and_drops():
+ rows = []
+ # 6 prompts: harness better on 5, worse on 1 → sign test 5 vs 1 (p = 0.2188)
+ for i, (h, o) in enumerate([(0.8, 0.6), (0.7, 0.5), (0.9, 0.6), (0.75, 0.7), (0.6, 0.65), (0.85, 0.5)]):
+ rows += [_cell(f"p{i}", HA, h, passed=h >= 0.72), _cell(f"p{i}", OA, o)]
+ # a provider outage on one arm drops the pair (not scored 0); an unscored budget cell drops it too
+ rows += [_cell("p6", HA, None, status="infra_failed"), _cell("p6", OA, 0.4)]
+ rows += [_cell("p7", HA, None, status="budget_exhausted"), _cell("p7", OA, 0.4)]
+ # the journal is append-only: a re-run of p0's harness cell supersedes the first row
+ rows += [_cell("p0", HA, 0.82, passed=True)]
+ st = paired(latest_cells(rows), HA, OA)
+ assert st.n == 6 and st.dropped_infra == 1 and st.dropped_unscored == 1
+ assert st.deltas["p0"] == 0.22 and st.wins == 5 and st.losses == 1 and st.ties == 0
+ assert st.sign_p == 0.2188
+ assert st.mean_delta == round((0.22 + 0.2 + 0.3 + 0.05 - 0.05 + 0.35) / 6, 4)
+ assert st.ci95_low is not None and st.ci95_high is not None and st.ci95_low < st.mean_delta < st.ci95_high
+ assert st.ci95_high - st.mean_delta == round(t975(5) * st.se_delta, 4) or abs((st.ci95_high - st.mean_delta) - t975(5) * st.se_delta) < 1e-3
+ assert st.verdict == "supported" and st.pass_rate_harness == round(4 / 6, 4) and st.pass_rate_oneshot == 0.0
+
+
+def test_interval_crossing_zero_is_unsupported_and_tiers_split():
+ rows = []
+ for i, (h, o, t) in enumerate([(0.8, 0.7, "easy"), (0.6, 0.7, "easy"), (0.9, 0.5, "hard"), (0.5, 0.8, "hard")]):
+ rows += [_cell(f"p{i}", HA, h, tier=t), _cell(f"p{i}", OA, o, tier=t)]
+ stats = analyse(rows)
+ by = {(s.tier): s for s in stats}
+ assert set(by) == {"all", "easy", "hard"}
+ assert by["all"].n == 4 and by["all"].verdict == "unsupported" and by["all"].ci95_low < 0 < by["all"].ci95_high
+ assert by["easy"].n == 2 and by["hard"].n == 2 and by["hard"].wins == 1 and by["hard"].losses == 1
+
+
+def test_t_quantiles_are_monotone_and_end_at_normal():
+ assert t975(1) > t975(5) > t975(30) > t975(59) > t975(1000) == 1.96
+ assert t975(0) != t975(0) # nan for no degrees of freedom
+
+
+def test_cli_writes_paired_md_and_json(tmp_path):
+ rows = [_cell("p0", HA, 0.8), _cell("p0", OA, 0.5), _cell("p1", HA, 0.7), _cell("p1", OA, 0.6)]
+ (tmp_path / "results.jsonl").write_text("\n".join(json.dumps(r.model_dump(mode="json")) for r in rows) + "\n")
+ assert main([str(tmp_path)]) == 0
+ md = (tmp_path / "paired.md").read_text()
+ assert "| harness arm |" in md and HA in md and "**" in md
+ data = json.loads((tmp_path / "paired.json").read_text())
+ assert data[0]["n"] == 2 and data[0]["wins"] == 2