Skip to content

Official Codebase of 3DCodeVerse

Python

bench: --report-only rebuilt summary.md from the bare flags, relabelling a real A/B as an A/A in the one file that outlives the run

gaoypengdbf25d5c9295
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.py
index b8577438..eab3edf9 100644
--- a/harness/bench/ab_plan.py
+++ b/harness/bench/ab_plan.py
@@ -411,6 +411,17 @@ def _parser() -> argparse.ArgumentParser:
return ap
+def _stored_options(out: Path) -> AbOptions | None:
+ """The options this run was launched with, from its own ``ab.json``."""
+ p = out / "ab.json"
+ if not p.is_file():
+ return None
+ try:
+ return AbOptions.model_validate(json.loads(p.read_text())["options"])
+ except (OSError, KeyError, ValueError):
+ return None
+
+
def main(argv: Sequence[str] | None = None) -> int:
ns = _parser().parse_args(argv)
if ns.cmd == "cell":
@@ -429,6 +440,10 @@ def main(argv: Sequence[str] | None = None) -> int:
out = Path(ns.out)
if ns.report_only:
battery = Battery.load(ns.prompts)
+ # the RUN's own options, not this invocation's flags. `--report-only` is normally
+ # typed with no --variant-env, and reporting the arms as identical would relabel a
+ # real A/B as an A/A in the file everyone reads afterwards.
+ opts = _stored_options(out) or opts
v = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],
title=f"{battery.name} / {out.name}", variant_env={} if opts.aa else opts.variant_env,
generator=opts.generator, judge=opts.judge, rounds=opts.rounds, aa=opts.aa)
diff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.py
index 24abeb46..5145858a 100644
--- a/harness/tests/compare_bench/test_ab_plan.py
+++ b/harness/tests/compare_bench/test_ab_plan.py
@@ -426,3 +426,33 @@ def test_report_only_reads_the_same_rows_the_live_driver_wrote(tmp_path: Path):
assert arm_summary(loaded, VARIANT).mean_score == 0.80
assert arm_summary(loaded, VARIANT).n == 1 and arm_summary(loaded, VARIANT).n_infra_failed == 0
assert [r for r in loaded if r.status == "infra_failed"] == [], "a redone cell is not still lost"
+
+
+def test_report_only_uses_the_runs_own_options_not_this_invocations_flags(tmp_path):
+ """`--report-only` is typed without --variant-env, and the report is what people read.
+
+ Rebuilding it from the bare flags relabelled a real A/B as "variant env: (none)" — an
+ A/A — in the one file that outlives the run.
+ """
+ import json
+
+ from bench.ab_plan import AbOptions, _stored_options
+
+ out = tmp_path / "run"
+ out.mkdir()
+ opts = AbOptions(variant_env={"CV3D_SKILLS": "on"}, rounds=1, generator="api-agent:gemini:x")
+ (out / "ab.json").write_text(json.dumps({"options": json.loads(opts.model_dump_json())}))
+
+ got = _stored_options(out)
+ assert got is not None
+ assert got.variant_env == {"CV3D_SKILLS": "on"} and got.rounds == 1
+
+
+def test_report_only_survives_a_run_dir_with_no_or_broken_ab_json(tmp_path):
+ from bench.ab_plan import _stored_options
+
+ assert _stored_options(tmp_path) is None
+ (tmp_path / "ab.json").write_text("{not json")
+ assert _stored_options(tmp_path) is None
+ (tmp_path / "ab.json").write_text('{"no options key": 1}')
+ assert _stored_options(tmp_path) is None