Skip to content

Official Codebase of 3DCodeVerse

Python

bench run: a --redo-status redo starts from a fresh workspace (the old one is archived as <id>.attemptN)

gaoypengc30888027f06
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.py
index e915b765..451a0530 100644
--- a/harness/bench/run_bench.py
+++ b/harness/bench/run_bench.py
@@ -199,6 +199,17 @@ def select_prompts(
RunFn = Callable[[Spec, Workspace, bool], RunRecord]
+
+def archive_attempt(root: Path) -> Path:
+ """Move a run workspace aside as ``<root>.attempt<N>`` (N = first free) and return the new path."""
+ n = 1
+ while (root.parent / f"{root.name}.attempt{n}").exists():
+ n += 1
+ dest = root.parent / f"{root.name}.attempt{n}"
+ root.rename(dest)
+ return dest
+
+
def default_run_track(spec: Spec, ws: Workspace, resume: bool) -> RunRecord:
"""Run the spec's track (the real thing; tests inject a fake ``run_fn``)."""
from codeverse.tracks import get_track
@@ -222,12 +233,20 @@ def run_battery(
results_jsonl = out / "results.jsonl"
done = _load_done(results_jsonl) if opts.resume else {}
# `--redo-status infra_failed` re-runs the cells the weather lost, once it clears
- for pid in [k for k, r in done.items() if r.status in set(opts.redo_status)]:
+ redo_ids = {k for k, r in done.items() if r.status in set(opts.redo_status)}
+ for pid in redo_ids:
del done[pid]
todo = [p for p in select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit) if p.id not in done]
def _one(item: BenchPrompt) -> BenchItemResult:
ws = Workspace(out / "runs" / item.id)
+ if item.id in redo_ids and ws.exists():
+ # A redo starts FRESH: resuming the old workspace keeps its spec (the old
+ # max_minutes) and its clock, so a `budget` row redone with --max-minutes 120
+ # was over budget before its first round (measured 2026-08-26: clock_q4 and
+ # lighthouse_1 came back `budget`, 0 rounds, 60.3 / 76.7 min "elapsed").
+ # The old tree is kept beside it as <id>.attempt<N>, the way ab_plan does.
+ archive_attempt(ws.root)
resume = ws.exists()
if not resume:
ws.create()
diff --git a/harness/tests/flywheel_cli/test_bench.py b/harness/tests/flywheel_cli/test_bench.py
index 41fd5c3c..4461d5b2 100644
--- a/harness/tests/flywheel_cli/test_bench.py
+++ b/harness/tests/flywheel_cli/test_bench.py
@@ -206,3 +206,34 @@ def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):
assert calls == [], "a plain resume still skips every recorded row"
run_battery(battery, out, BenchOptions(parallel=1, limit=1, redo_status=["infra_failed"]), run_fn=counting)
assert len(calls) == 1, "--redo-status infra_failed re-runs what the weather lost"
+
+
+def test_a_redo_starts_from_a_fresh_workspace(tmp_path):
+ """Measured 2026-08-26: a `budget` row redone with --max-minutes 120 resumed the old
+ workspace (old spec, old clock) and came back `budget` with 0 rounds. The old tree
+ is archived as <id>.attempt1 and the prompt runs fresh."""
+ from bench.run_bench import archive_attempt
+
+ out = tmp_path / "out"
+ b = Battery.load(Path("bench/prompts/static_objects_v1.yaml"))
+ pid = b.prompts[0].id
+ ws_root = out / "runs" / pid
+ (ws_root / "src").mkdir(parents=True)
+ (ws_root / "src" / "old.py").write_text("# stale")
+ out.mkdir(exist_ok=True)
+ (out / "results.jsonl").write_text(json.dumps({"id": pid, "tier": b.prompts[0].tier, "status": "budget", "score_final": None}) + "\n")
+ seen: list[bool] = []
+
+ def run(spec, ws, resume):
+ seen.append(resume)
+ _ws, rec = make_fake_run(ws.root.parent, ws.root.name, prompt=spec.prompt, language=spec.language, scores=(0.5, 0.7))
+ rec.spec = spec
+ ws.write_json(ws.spec_path, spec)
+ ws.write_json(ws.record_path, rec)
+ return rec
+
+ opts = BenchOptions(ids=[pid], redo_status=["budget"], parallel=1, rounds=1)
+ run_battery(Path("bench/prompts/static_objects_v1.yaml"), out, opts, run_fn=run)
+ assert seen == [False], "the redo ran fresh, not resumed"
+ assert (out / "runs" / f"{pid}.attempt1" / "src" / "old.py").exists(), "the old tree is archived beside the new one"
+ assert archive_attempt(out / "runs" / pid).name == f"{pid}.attempt2"