Skip to content

Official Codebase of 3DCodeVerse

Python

bestofk: state the asymmetry on the harness side too

ziyaod3cf08ec0681
diff --git a/harness/bench/bestofk_report.py b/harness/bench/bestofk_report.py
index 4c8cde55..c35c1f16 100644
--- a/harness/bench/bestofk_report.py
+++ b/harness/bench/bestofk_report.py
@@ -71,6 +71,28 @@ def harness_scores(root: Path, *, arm_prefix: str = "harness:") -> dict[str, flo
return out
+def harness_repeat_spread(root: Path, *, arm_prefix: str = "harness:") -> tuple[int, int, float]:
+ """``(rows, prompts, worst within-prompt score spread)`` for the recorded arm.
+
+ The battery appends a row per report pass, so a prompt can carry several harness rows.
+ If those are re-runs the aggregation choice matters; if they are duplicates it cannot.
+ On `compare_v4` the spread is 0.000 on all 40 prompts, so last-row-wins is safe — and
+ it also means the harness has NO repeat measurement here, which is the asymmetry worth
+ stating: best-of-k selects over k draws while the harness gets one, carrying the
+ generator's own +-0.13 A/A repeat noise unaveraged.
+ """
+ per: dict[str, list[float]] = {}
+ rows = 0
+ for r in _rows(root / "results.jsonl"):
+ if not str(r.get("arm", "")).startswith(arm_prefix):
+ continue
+ rows += 1
+ if r.get("score") is not None:
+ per.setdefault(r["prompt_id"], []).append(float(r["score"]))
+ spreads = [max(v) - min(v) for v in per.values() if len(v) > 1]
+ return rows, len(per), (max(spreads) if spreads else 0.0)
+
+
def gen_costs(root: Path, arm_prefix: str) -> list[float]:
"""Generation cost per PROMPT for an arm — last row per prompt, the basis
`paired_compare` pairs on. Taken over every appended row instead, resumed and failed
@@ -129,6 +151,12 @@ def report(root: Path, against: Path, *, baseline_prefix: str = "oneshot:") -> s
mark = "reached" if eq[2] <= reps else f"NOT reached yet — {reps} of {eq[2]} reps"
lines += [f"Equal compute: harness ${eq[0]:.4f} per prompt against ${eq[1]:.4f}, so **k = {eq[2]}** ({mark}).",
"Costs are the last row per prompt, the basis `paired_compare` pairs on.", ""]
+ rows, prompts, spread = harness_repeat_spread(against)
+ if prompts:
+ lines += [f"Harness arm: {rows} rows over {prompts} prompts, worst within-prompt score spread "
+ f"{spread:.3f} — {'duplicates, so the aggregation choice cannot move this' if spread == 0 else 're-runs, so it can'}.",
+ "It is ONE draw against best-of-k's k, carrying the generator's own A/A repeat noise "
+ "(+-0.13) unaveraged. That asymmetry runs against the harness.", ""]
lines += [
f"Winner's-curse bound: judge sigma {JUDGE_SIGMA:.3f}, so a best-of-k maximum carries",
"about `sigma*sqrt(2 ln k)` of selection noise. It inflates the BASELINE, never the harness.",
diff --git a/harness/tests/compare_bench/test_bestofk_report.py b/harness/tests/compare_bench/test_bestofk_report.py
index 67fea5b5..455afa78 100644
--- a/harness/tests/compare_bench/test_bestofk_report.py
+++ b/harness/tests/compare_bench/test_bestofk_report.py
@@ -137,3 +137,26 @@ def test_the_report_says_when_k_has_not_been_reached(tmp_path: Path):
]))
text = report(tmp_path / "reps", tmp_path)
assert "**k = 40**" in text and "NOT reached yet — 1 of 40 reps" in text
+
+
+def test_repeated_harness_rows_are_reported_as_duplicates_or_reruns(tmp_path: Path):
+ """Whether the aggregation choice can move the comparison is a fact about the data, so
+ the report states it. On `compare_v4` the 117 harness rows over 40 prompts have a
+ within-prompt spread of exactly 0.000 — they are the same cell written by successive
+ report passes, not re-runs — so last-row-wins is safe there."""
+ from bench.bestofk_report import harness_repeat_spread
+
+ (tmp_path / "results.jsonl").write_text("\n".join(json.dumps(r) for r in [
+ {"prompt_id": "a", "arm": "harness:x", "score": 0.5},
+ {"prompt_id": "a", "arm": "harness:x", "score": 0.5}, # duplicate
+ {"prompt_id": "b", "arm": "harness:x", "score": 0.4},
+ {"prompt_id": "c", "arm": "oneshot:x", "score": 0.9}, # other arms ignored
+ ]))
+ assert harness_repeat_spread(tmp_path) == (3, 2, 0.0)
+
+ (tmp_path / "results.jsonl").write_text("\n".join(json.dumps(r) for r in [
+ {"prompt_id": "a", "arm": "harness:x", "score": 0.5},
+ {"prompt_id": "a", "arm": "harness:x", "score": 0.8}, # a real re-run
+ ]))
+ rows, prompts, spread = harness_repeat_spread(tmp_path)
+ assert (rows, prompts) == (2, 1) and spread == pytest.approx(0.3)