Official Codebase of 3DCodeVerse
Python
bench: one arm aggregate — the A/B summary reads the compare report's ArmStats
gaoypeng6ca1f4c27dfe
diff --git a/eval/bench/_ab_report.py b/eval/bench/_ab_report.pyindex ca06c355..025bb580 100644--- a/eval/bench/_ab_report.py+++ b/eval/bench/_ab_report.py@@ -20,7 +20,7 @@ from pathlib import Pathfrom pydantic import BaseModel, Field-from bench._compare_report import CellResult+from bench._compare_report import CellResult, arm_statsfrom bench._jsonl import latestfrom bench.stats import mean_ci, n_to_resolve, sign_test@@ -161,34 +161,6 @@ def verdict_of(pairs: list[PairOutcome]) -> Verdict:return v-class ArmSummary(BaseModel):- arm: str- n: int- n_scored: int- n_infra_failed: int- n_budget_exhausted: int- n_other_unscored: int- mean_score: float | None- median_score: float | None- cost_usd: float = Field(description="generation + fixed-judge spend over every cell that ran")- mean_minutes: float---def arm_summary(rows: list[CellResult], arm: str) -> ArmSummary:- rs = [r for r in rows if r.arm == arm]- ran = [r for r in rs if r.status != "infra_failed"]- scores = [r.score for r in ran if r.score is not None]- return ArmSummary(- arm=arm, n=len(rs), n_scored=len(scores), n_infra_failed=len(rs) - len(ran),- n_budget_exhausted=sum(r.status == "budget_exhausted" for r in ran),- n_other_unscored=sum(r.score is None and r.status != "budget_exhausted" for r in ran),- mean_score=round(statistics.fmean(scores), 4) if scores else None,- median_score=round(statistics.median(scores), 4) if scores else None,- cost_usd=round(sum(r.gen_cost_usd + r.judge_cost_usd for r in ran), 4),- mean_minutes=round(statistics.fmean(r.wall_s for r in ran) / 60, 2) if ran else 0.0,- )--def _fmt(x: float | None, signed: bool = False) -> str:if x is None:return "—"@@ -209,7 +181,8 @@ def render_summary(pairs: list[PairOutcome], rows: list[CellResult], *, title: smeasured delta IS the noise floor, and any decision word would be a lie."""v = verdict_of(pairs)- arms = [arm_summary(rows, a) for a in ARMS]+ by_arm = {s.arm: s for s in arm_stats(rows)} # the compare report's aggregate, outages excluded+ arms = [by_arm[a] for a in ARMS if a in by_arm]lines = [f"# A/{'A' if aa else 'B'}: {title}", ""]if aa:lines += ["> **A/A calibration — the arms are identical.** Every delta below is pure run-to-run",@@ -231,17 +204,18 @@ def render_summary(pairs: list[PairOutcome], rows: list[CellResult], *, title: sf"{'—' if v.sign_p is None else f'{v.sign_p:.3f}'} — the signal an 8-prompt battery can actually carry",*([f"\n**{v.caution}**"] if v.caution else []), "","## Arms", "",- "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | cost $ | mean min |",+ "| arm | n | scored | infra_failed | budget_exhausted | other unscored | mean | median | $/cell | mean min |","|---|---|---|---|---|---|---|---|---|---|",]for a in arms:- lines.append(f"| {a.arm} | {a.n} | {a.n_scored} | {a.n_infra_failed} | {a.n_budget_exhausted} | "- f"{a.n_other_unscored} | {_fmt(a.mean_score)} | {_fmt(a.median_score)} | {a.cost_usd:.2f} | {a.mean_minutes:.1f} |")+ lines.append(f"| {a.arm} | {a.n} | {a.n_scored} | {a.infra_failed} | {a.budget_exhausted} | "+ f"{a.n_evaluated - a.n_scored - a.budget_exhausted} | {_fmt(a.mean_score)} | {_fmt(a.median_score)} | "+ f"{a.mean_gen_usd + a.mean_judge_usd:.2f} | {a.mean_minutes:.1f} |")lines += ["", "## Per prompt", "", "| prompt | tier | control | variant | delta | note |", "|---|---|---|---|---|---|"]for p in pairs:note = "REGRESSION" if p.regression else ("" if p.paired else f"unpaired: {p.reason}")lines.append(f"| {p.prompt_id} | {p.tier} | {_cell(p.control)} | {_cell(p.variant)} | {_fmt(p.delta, True)} | {note} |")- dropped = sum(a.n_infra_failed for a in arms)+ dropped = sum(a.infra_failed for a in arms)lines += ["", f"n_infra_failed: {dropped}" + (" — re-run with `--redo-status infra_failed` before trusting the verdict"if dropped else ""), "","Rule: keep iff mean delta >= +0.02 and no regression; revert iff mean delta <= -0.02 or >= 2 regressions; "@@ -262,5 +236,5 @@ def write_report(out: Path, rows: list[CellResult], prompt_order: list[tuple[str__all__ = ["ARMS", "CONTROL", "KEEP_DELTA", "REGRESSION_DELTA", "REVERT_DELTA","REVERT_REGRESSIONS", "VARIANT",- "ArmSummary", "PairOutcome", "Verdict", "arm_summary", "pair_up", "render_summary", "verdict_of",+ "PairOutcome", "Verdict", "pair_up", "render_summary", "verdict_of","write_report"]diff --git a/eval/bench/_compare_report.py b/eval/bench/_compare_report.pyindex 4c455772..9f242873 100644--- a/eval/bench/_compare_report.py+++ b/eval/bench/_compare_report.py@@ -107,6 +107,7 @@ class ArmStats(BaseModel):kind: strn: int = Field(description="cells attempted")n_evaluated: int = Field(default=0, description="cells that actually ran (n minus provider outages)")+ n_scored: int = Field(default=0, description="cells that ran and carry a fixed-judge score")infra_failed: int = Field(default=0, description="cells dropped: the provider, not the model, failed")budget_exhausted: int = Field(default=0, description="cells that ran out of time/money with no artifact")mean_score: float | None@@ -149,7 +150,7 @@ def arm_stats(rows: list[CellResult]) -> list[ArmStats]:scores = [r.score for r in ev if r.score is not None]passed = [r.passed for r in ev if r.passed is not None]out.append(ArmStats(- arm=arm, kind=rs[0].kind, n=len(rs), n_evaluated=len(ev),+ arm=arm, kind=rs[0].kind, n=len(rs), n_evaluated=len(ev), n_scored=len(scores),infra_failed=len(rs) - len(ev),budget_exhausted=sum(1 for r in ev if r.status == "budget_exhausted"), mean_score=_mean(scores),median_score=round(statistics.median(scores), 4) if scores else None,diff --git a/eval/tests/test_ab_plan.py b/eval/tests/test_ab_plan.pyindex 4dcdef62..c0d84240 100644--- a/eval/tests/test_ab_plan.py+++ b/eval/tests/test_ab_plan.py@@ -17,12 +17,11 @@ from bench._ab_report import ( # noqa: E402KEEP_DELTA,VARIANT,PairOutcome,- arm_summary,pair_up,render_summary,verdict_of,)-from bench._compare_report import CellResult # noqa: E402+from bench._compare_report import CellResult, arm_stats # noqa: E402from bench._jsonl import latest, read_jsonl # noqa: E402from bench.ab_plan import ( # noqa: E402DEFAULT_MAX_IN_FLIGHT,@@ -110,9 +109,10 @@ def test_outage_cells_are_excluded_from_every_arm_rate():the loss must stay visible (docs/EVAL.md §7)."""clear = [_row(f"p{i}", CONTROL, 0.8) for i in range(4)]unlucky = [_row(f"p{i}", VARIANT, 0.8) for i in range(4)] + [_row("p9", VARIANT, None, "infra_failed", wall=3600, cost=0.0)]- c, v = arm_summary(clear + unlucky, CONTROL), arm_summary(clear + unlucky, VARIANT)- assert c.mean_score == v.mean_score and c.mean_minutes == v.mean_minutes and c.cost_usd == v.cost_usd- assert v.n_infra_failed == 1 and v.n == 5 and v.n_scored == 4+ by = {s.arm: s for s in arm_stats(clear + unlucky)}+ c, v = by[CONTROL], by[VARIANT]+ assert c.mean_score == v.mean_score and c.mean_minutes == v.mean_minutes and c.mean_gen_usd == v.mean_gen_usd+ assert v.infra_failed == 1 and v.n == 5 and v.n_scored == 4md = render_summary(pair_up(clear + unlucky), clear + unlucky, title="t", variant_env={"X": "1"},generator="g", judge="j", rounds=2)assert "n_infra_failed: 1" in md and "--redo-status infra_failed" in md and "variant env: `X=1`" in md