Skip to content

Official Codebase of 3DCodeVerse

Python

bench: paired_compare pairs two `bench run` batteries, not only compare_backends arms

ziyaoed4681ce6db1
diff --git a/harness/bench/paired_compare.py b/harness/bench/paired_compare.py
index 5f5942ea..81d48e45 100644
--- a/harness/bench/paired_compare.py
+++ b/harness/bench/paired_compare.py
@@ -252,11 +252,42 @@ def render_markdown(stats: list[PairedStats], title: str, gates: list[ArmGateSta
return out
+def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:
+ """A ``bench run`` battery read as cells of one arm.
+
+ ``compare_backends`` writes one journal with an ``arm`` column; ``bench run`` writes a
+ directory per arm with ``id`` / ``score_final``. Two of those directories are a paired
+ comparison — same prompts, one thing different — and this lets the statistics below
+ (paired CI, exact sign test, the "unsupported when the interval crosses zero" rule) be
+ the same for both shapes rather than recomputed by hand.
+ """
+ rows: list[CellResult] = []
+ for line in (out_dir / "results.jsonl").read_text().splitlines():
+ if not line.strip():
+ continue
+ raw = json.loads(line)
+ rows.append(CellResult(prompt_id=raw["id"], arm=arm, tier=raw.get("tier", ""),
+ score=raw.get("score_final"), status=raw.get("status", ""),
+ build_ok=raw.get("score_final") is not None,
+ gen_cost_usd=float(raw.get("cost_usd") or 0.0), kind="harness"))
+ return rows
+
+
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("out_dir", help="a compare_backends output directory (holds results.jsonl)")
+ ap.add_argument("--against", type=Path, default=None,
+ help="a SECOND `bench run` output directory: pair the two by prompt id "
+ "(out_dir is then also read as a `bench run` battery)")
ns = ap.parse_args(argv)
out = Path(ns.out_dir)
+ if ns.against is not None:
+ rows = rows_from_bench_run(out, out.name) + rows_from_bench_run(ns.against, ns.against.name)
+ stats = [paired(latest_cells(rows), out.name, ns.against.name)]
+ md = render_markdown(stats, f"{out.name} vs {ns.against.name}")
+ (out / "paired.md").write_text(md)
+ print(md)
+ return 0
rows = read_jsonl(out / "results.jsonl", CellResult)
stats = analyse(rows)
gates = gate_stats(latest_cells(rows))
diff --git a/harness/tests/compare_bench/test_bench_shared.py b/harness/tests/compare_bench/test_bench_shared.py
index 2da7e503..4689d483 100644
--- a/harness/tests/compare_bench/test_bench_shared.py
+++ b/harness/tests/compare_bench/test_bench_shared.py
@@ -5,6 +5,8 @@ from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
+import pytest
+
import bench.compare_backends as cb
import bench.run_bench as rb
from bench.run_bench import Battery, BenchOptions, build_spec, select_prompts, spec_for
@@ -97,3 +99,29 @@ def test_a_run_that_scored_nothing_says_so_in_its_row() -> None:
judged = SimpleNamespace(index=0, judgment=SimpleNamespace(passed=True))
rec.rounds = [judged, rounds[1]]
assert result_from_record(item, rec, 16.0, ws).errors == "" # one verdict is enough
+
+
+def test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:
+ """`compare_backends` writes one journal with an arm column; `bench run` writes a
+ directory per arm. Two of those directories are a paired comparison, and it must go
+ through the same statistics — paired CI, exact sign test, "unsupported when the
+ interval crosses zero" — rather than being recomputed by hand."""
+ import json as _json
+
+ from bench.paired_compare import latest_cells, paired, rows_from_bench_run
+
+ def write(d: Path, scores: dict[str, float]) -> Path:
+ d.mkdir(parents=True)
+ (d / "results.jsonl").write_text("".join(
+ _json.dumps({"id": k, "tier": "hard", "score_final": v, "status": "plateau",
+ "cost_usd": 1.0}) + "\n" for k, v in scores.items()))
+ return d
+
+ a = write(tmp_path / "arm_a", {"p1": 0.6, "p2": 0.4, "p3": 0.5})
+ b = write(tmp_path / "arm_b", {"p1": 0.5, "p2": 0.3, "p3": 0.5})
+
+ rows = rows_from_bench_run(a, "arm_a") + rows_from_bench_run(b, "arm_b")
+ st = paired(latest_cells(rows), "arm_a", "arm_b")
+ assert st.n == 3 and st.mean_delta == pytest.approx(0.0667, abs=1e-3)
+ assert st.wins == 2 and st.losses == 0 and st.ties == 1
+ assert st.verdict in {"supported", "unsupported", "too few pairs"}