Skip to content

Official Codebase of 3DCodeVerse

Python

bench: ab_plan hands its workers absolute paths — a relative --prompts made every cell exit 1

gaoypengf53cccc87dc7
diff --git a/eval/bench/ab_plan.py b/eval/bench/ab_plan.py
index a28cf7e5..654219ed 100644
--- a/eval/bench/ab_plan.py
+++ b/eval/bench/ab_plan.py
@@ -456,7 +456,9 @@ def run_ab(
on_result: Callable[[CellResult], None] | None = None,
) -> Verdict:
"""Run (or resume) the A/B, one prompt-pair at a time; write pairs.json + summary.md."""
- battery_path, out = Path(battery_path), Path(out_dir)
+ # absolute before any worker sees them: a worker runs with cwd=REPO, so the relative
+ # `--prompts bench/prompts/x.yaml` the docstring shows made every cell exit 1 (2026-09-23)
+ battery_path, out = Path(battery_path).resolve(), Path(out_dir).resolve()
battery = Battery.load(battery_path)
out.mkdir(parents=True, exist_ok=True)
(out / "ab.json").write_text(
diff --git a/eval/tests/test_ab_plan.py b/eval/tests/test_ab_plan.py
index dc6705ff..8192434d 100644
--- a/eval/tests/test_ab_plan.py
+++ b/eval/tests/test_ab_plan.py
@@ -191,6 +191,19 @@ class FakeCells:
return _row(item.id, arm, s, tier=item.tier, workspace=env.get("C3D_PLAN_BRIEF", ""))
+def test_a_relative_battery_path_reaches_the_workers_absolute(tmp_path: Path, monkeypatch):
+ """Workers run with cwd=REPO: the relative --prompts the usage line shows must not reach them as-is."""
+ seen: list[Path] = []
+
+ def record(battery_path, out, item, arm, opts):
+ seen.append(Path(battery_path))
+ return _row(item.id, arm, 0.5, tier=item.tier)
+
+ monkeypatch.chdir(Path(BATTERY).parent)
+ run_ab(Path(BATTERY).name, tmp_path, AbOptions(variant_env={"K": "v"}, limit=1), run_cell_fn=record)
+ assert seen and all(p.is_absolute() and p.is_file() for p in seen)
+
+
def test_driver_runs_pairs_in_prompt_order_and_writes_the_verdict(tmp_path: Path):
b = Battery.load(BATTERY)
ids = [p.id for p in b.prompts[:3]]