Official Codebase of 3DCodeVerse
Python
bench: paired_compare pairs two `bench run` batteries, not only compare_backends arms
ziyaoed4681ce6db1
diff --git a/harness/bench/paired_compare.py b/harness/bench/paired_compare.pyindex 5f5942ea..81d48e45 100644--- a/harness/bench/paired_compare.py+++ b/harness/bench/paired_compare.py@@ -252,11 +252,42 @@ def render_markdown(stats: list[PairedStats], title: str, gates: list[ArmGateStareturn out+def rows_from_bench_run(out_dir: Path, arm: str) -> list[CellResult]:+ """A ``bench run`` battery read as cells of one arm.++ ``compare_backends`` writes one journal with an ``arm`` column; ``bench run`` writes a+ directory per arm with ``id`` / ``score_final``. Two of those directories are a paired+ comparison — same prompts, one thing different — and this lets the statistics below+ (paired CI, exact sign test, the "unsupported when the interval crosses zero" rule) be+ the same for both shapes rather than recomputed by hand.+ """+ rows: list[CellResult] = []+ for line in (out_dir / "results.jsonl").read_text().splitlines():+ if not line.strip():+ continue+ raw = json.loads(line)+ rows.append(CellResult(prompt_id=raw["id"], arm=arm, tier=raw.get("tier", ""),+ score=raw.get("score_final"), status=raw.get("status", ""),+ build_ok=raw.get("score_final") is not None,+ gen_cost_usd=float(raw.get("cost_usd") or 0.0), kind="harness"))+ return rows++def main(argv: list[str] | None = None) -> int:ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])ap.add_argument("out_dir", help="a compare_backends output directory (holds results.jsonl)")+ ap.add_argument("--against", type=Path, default=None,+ help="a SECOND `bench run` output directory: pair the two by prompt id "+ "(out_dir is then also read as a `bench run` battery)")ns = ap.parse_args(argv)out = Path(ns.out_dir)+ if ns.against is not None:+ rows = rows_from_bench_run(out, out.name) + rows_from_bench_run(ns.against, ns.against.name)+ stats = [paired(latest_cells(rows), out.name, ns.against.name)]+ md = render_markdown(stats, f"{out.name} vs {ns.against.name}")+ (out / "paired.md").write_text(md)+ print(md)+ return 0rows = read_jsonl(out / "results.jsonl", CellResult)stats = analyse(rows)gates = gate_stats(latest_cells(rows))diff --git a/harness/tests/compare_bench/test_bench_shared.py b/harness/tests/compare_bench/test_bench_shared.pyindex 2da7e503..4689d483 100644--- a/harness/tests/compare_bench/test_bench_shared.py+++ b/harness/tests/compare_bench/test_bench_shared.py@@ -5,6 +5,8 @@ from __future__ import annotationsfrom pathlib import Pathfrom types import SimpleNamespace+import pytest+import bench.compare_backends as cbimport bench.run_bench as rbfrom bench.run_bench import Battery, BenchOptions, build_spec, select_prompts, spec_for@@ -97,3 +99,29 @@ def test_a_run_that_scored_nothing_says_so_in_its_row() -> None:judged = SimpleNamespace(index=0, judgment=SimpleNamespace(passed=True))rec.rounds = [judged, rounds[1]]assert result_from_record(item, rec, 16.0, ws).errors == "" # one verdict is enough+++def test_two_bench_run_batteries_pair_by_prompt(tmp_path: Path) -> None:+ """`compare_backends` writes one journal with an arm column; `bench run` writes a+ directory per arm. Two of those directories are a paired comparison, and it must go+ through the same statistics — paired CI, exact sign test, "unsupported when the+ interval crosses zero" — rather than being recomputed by hand."""+ import json as _json++ from bench.paired_compare import latest_cells, paired, rows_from_bench_run++ def write(d: Path, scores: dict[str, float]) -> Path:+ d.mkdir(parents=True)+ (d / "results.jsonl").write_text("".join(+ _json.dumps({"id": k, "tier": "hard", "score_final": v, "status": "plateau",+ "cost_usd": 1.0}) + "\n" for k, v in scores.items()))+ return d++ a = write(tmp_path / "arm_a", {"p1": 0.6, "p2": 0.4, "p3": 0.5})+ b = write(tmp_path / "arm_b", {"p1": 0.5, "p2": 0.3, "p3": 0.5})++ rows = rows_from_bench_run(a, "arm_a") + rows_from_bench_run(b, "arm_b")+ st = paired(latest_cells(rows), "arm_a", "arm_b")+ assert st.n == 3 and st.mean_delta == pytest.approx(0.0667, abs=1e-3)+ assert st.wins == 2 and st.losses == 0 and st.ties == 1+ assert st.verdict in {"supported", "unsupported", "too few pairs"}