Official Codebase of 3DCodeVerse
Python
bench: --report-only rebuilt summary.md from the bare flags, relabelling a real A/B as an A/A in the one file that outlives the run
gaoypengdbf25d5c9295
diff --git a/harness/bench/ab_plan.py b/harness/bench/ab_plan.pyindex b8577438..eab3edf9 100644--- a/harness/bench/ab_plan.py+++ b/harness/bench/ab_plan.py@@ -411,6 +411,17 @@ def _parser() -> argparse.ArgumentParser:return ap+def _stored_options(out: Path) -> AbOptions | None:+ """The options this run was launched with, from its own ``ab.json``."""+ p = out / "ab.json"+ if not p.is_file():+ return None+ try:+ return AbOptions.model_validate(json.loads(p.read_text())["options"])+ except (OSError, KeyError, ValueError):+ return None++def main(argv: Sequence[str] | None = None) -> int:ns = _parser().parse_args(argv)if ns.cmd == "cell":@@ -429,6 +440,10 @@ def main(argv: Sequence[str] | None = None) -> int:out = Path(ns.out)if ns.report_only:battery = Battery.load(ns.prompts)+ # the RUN's own options, not this invocation's flags. `--report-only` is normally+ # typed with no --variant-env, and reporting the arms as identical would relabel a+ # real A/B as an A/A in the file everyone reads afterwards.+ opts = _stored_options(out) or optsv = write_report(out, load_jsonl(out / "results.jsonl", CellResult), [(p.id, p.tier) for p in battery.prompts],title=f"{battery.name} / {out.name}", variant_env={} if opts.aa else opts.variant_env,generator=opts.generator, judge=opts.judge, rounds=opts.rounds, aa=opts.aa)diff --git a/harness/tests/compare_bench/test_ab_plan.py b/harness/tests/compare_bench/test_ab_plan.pyindex 24abeb46..5145858a 100644--- a/harness/tests/compare_bench/test_ab_plan.py+++ b/harness/tests/compare_bench/test_ab_plan.py@@ -426,3 +426,33 @@ def test_report_only_reads_the_same_rows_the_live_driver_wrote(tmp_path: Path):assert arm_summary(loaded, VARIANT).mean_score == 0.80assert arm_summary(loaded, VARIANT).n == 1 and arm_summary(loaded, VARIANT).n_infra_failed == 0assert [r for r in loaded if r.status == "infra_failed"] == [], "a redone cell is not still lost"+++def test_report_only_uses_the_runs_own_options_not_this_invocations_flags(tmp_path):+ """`--report-only` is typed without --variant-env, and the report is what people read.++ Rebuilding it from the bare flags relabelled a real A/B as "variant env: (none)" — an+ A/A — in the one file that outlives the run.+ """+ import json++ from bench.ab_plan import AbOptions, _stored_options++ out = tmp_path / "run"+ out.mkdir()+ opts = AbOptions(variant_env={"CV3D_SKILLS": "on"}, rounds=1, generator="api-agent:gemini:x")+ (out / "ab.json").write_text(json.dumps({"options": json.loads(opts.model_dump_json())}))++ got = _stored_options(out)+ assert got is not None+ assert got.variant_env == {"CV3D_SKILLS": "on"} and got.rounds == 1+++def test_report_only_survives_a_run_dir_with_no_or_broken_ab_json(tmp_path):+ from bench.ab_plan import _stored_options++ assert _stored_options(tmp_path) is None+ (tmp_path / "ab.json").write_text("{not json")+ assert _stored_options(tmp_path) is None+ (tmp_path / "ab.json").write_text('{"no options key": 1}')+ assert _stored_options(tmp_path) is None