Skip to content

Official Codebase of 3DCodeVerse

Python

compare_backends: --planner overrides the harness planner model per matrix

ziyao9d47fbea8a1b
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index 78313e3a..89152c94 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -506,6 +506,9 @@ def main(argv: Sequence[str] | None = None) -> int:
ap.add_argument("--max-usd", type=float, default=2.5)
ap.add_argument("--max-minutes", type=float, default=45.0, help="wall-clock ceiling for ONE harness run")
ap.add_argument("--loop-judge", default=None, help="harness in-loop judge (default: settings default)")
+ ap.add_argument("--planner", default=None,
+ help="harness planner model id (default: settings default) — e.g. gemini:gemini-3.1-pro-preview "
+ "to plan with a stronger model than the generator (compare_art_v4, 2026-08-28)")
ap.add_argument("--repair-attempts", type=int, default=2)
ap.add_argument("--gen-timeout", type=float, default=900.0)
ap.add_argument("--judge-samples", type=int, default=2,
@@ -526,7 +529,7 @@ def main(argv: Sequence[str] | None = None) -> int:
if ns.report_only:
build_compare_report(Path(ns.out))
return 0
- opts = CompareOptions(judge=ns.judge, loop_judge=ns.loop_judge, rounds=ns.rounds, max_usd=ns.max_usd,
+ opts = CompareOptions(judge=ns.judge, loop_judge=ns.loop_judge, planner=ns.planner, rounds=ns.rounds, max_usd=ns.max_usd,
max_minutes=ns.max_minutes, parallel=ns.parallel,
limit=ns.limit, ids=[i for i in ns.ids.split(",") if i],
tiers=[t for t in ns.tiers.split(",") if t], resume=not ns.no_resume,