Official Codebase of 3DCodeVerse
Python
bestofk: state the asymmetry on the harness side too
ziyaod3cf08ec0681
diff --git a/harness/bench/bestofk_report.py b/harness/bench/bestofk_report.pyindex 4c8cde55..c35c1f16 100644--- a/harness/bench/bestofk_report.py+++ b/harness/bench/bestofk_report.py@@ -71,6 +71,28 @@ def harness_scores(root: Path, *, arm_prefix: str = "harness:") -> dict[str, floreturn out+def harness_repeat_spread(root: Path, *, arm_prefix: str = "harness:") -> tuple[int, int, float]:+ """``(rows, prompts, worst within-prompt score spread)`` for the recorded arm.++ The battery appends a row per report pass, so a prompt can carry several harness rows.+ If those are re-runs the aggregation choice matters; if they are duplicates it cannot.+ On `compare_v4` the spread is 0.000 on all 40 prompts, so last-row-wins is safe — and+ it also means the harness has NO repeat measurement here, which is the asymmetry worth+ stating: best-of-k selects over k draws while the harness gets one, carrying the+ generator's own +-0.13 A/A repeat noise unaveraged.+ """+ per: dict[str, list[float]] = {}+ rows = 0+ for r in _rows(root / "results.jsonl"):+ if not str(r.get("arm", "")).startswith(arm_prefix):+ continue+ rows += 1+ if r.get("score") is not None:+ per.setdefault(r["prompt_id"], []).append(float(r["score"]))+ spreads = [max(v) - min(v) for v in per.values() if len(v) > 1]+ return rows, len(per), (max(spreads) if spreads else 0.0)++def gen_costs(root: Path, arm_prefix: str) -> list[float]:"""Generation cost per PROMPT for an arm — last row per prompt, the basis`paired_compare` pairs on. Taken over every appended row instead, resumed and failed@@ -129,6 +151,12 @@ def report(root: Path, against: Path, *, baseline_prefix: str = "oneshot:") -> smark = "reached" if eq[2] <= reps else f"NOT reached yet — {reps} of {eq[2]} reps"lines += [f"Equal compute: harness ${eq[0]:.4f} per prompt against ${eq[1]:.4f}, so **k = {eq[2]}** ({mark}).","Costs are the last row per prompt, the basis `paired_compare` pairs on.", ""]+ rows, prompts, spread = harness_repeat_spread(against)+ if prompts:+ lines += [f"Harness arm: {rows} rows over {prompts} prompts, worst within-prompt score spread "+ f"{spread:.3f} — {'duplicates, so the aggregation choice cannot move this' if spread == 0 else 're-runs, so it can'}.",+ "It is ONE draw against best-of-k's k, carrying the generator's own A/A repeat noise "+ "(+-0.13) unaveraged. That asymmetry runs against the harness.", ""]lines += [f"Winner's-curse bound: judge sigma {JUDGE_SIGMA:.3f}, so a best-of-k maximum carries","about `sigma*sqrt(2 ln k)` of selection noise. It inflates the BASELINE, never the harness.",diff --git a/harness/tests/compare_bench/test_bestofk_report.py b/harness/tests/compare_bench/test_bestofk_report.pyindex 67fea5b5..455afa78 100644--- a/harness/tests/compare_bench/test_bestofk_report.py+++ b/harness/tests/compare_bench/test_bestofk_report.py@@ -137,3 +137,26 @@ def test_the_report_says_when_k_has_not_been_reached(tmp_path: Path):]))text = report(tmp_path / "reps", tmp_path)assert "**k = 40**" in text and "NOT reached yet — 1 of 40 reps" in text+++def test_repeated_harness_rows_are_reported_as_duplicates_or_reruns(tmp_path: Path):+ """Whether the aggregation choice can move the comparison is a fact about the data, so+ the report states it. On `compare_v4` the 117 harness rows over 40 prompts have a+ within-prompt spread of exactly 0.000 — they are the same cell written by successive+ report passes, not re-runs — so last-row-wins is safe there."""+ from bench.bestofk_report import harness_repeat_spread++ (tmp_path / "results.jsonl").write_text("\n".join(json.dumps(r) for r in [+ {"prompt_id": "a", "arm": "harness:x", "score": 0.5},+ {"prompt_id": "a", "arm": "harness:x", "score": 0.5}, # duplicate+ {"prompt_id": "b", "arm": "harness:x", "score": 0.4},+ {"prompt_id": "c", "arm": "oneshot:x", "score": 0.9}, # other arms ignored+ ]))+ assert harness_repeat_spread(tmp_path) == (3, 2, 0.0)++ (tmp_path / "results.jsonl").write_text("\n".join(json.dumps(r) for r in [+ {"prompt_id": "a", "arm": "harness:x", "score": 0.5},+ {"prompt_id": "a", "arm": "harness:x", "score": 0.8}, # a real re-run+ ]))+ rows, prompts, spread = harness_repeat_spread(tmp_path)+ assert (rows, prompts) == (2, 1) and spread == pytest.approx(0.3)