Official Codebase of 3DCodeVerse
Python
bench run: a --redo-status redo starts from a fresh workspace (the old one is archived as <id>.attemptN)
gaoypengc30888027f06
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.pyindex e915b765..451a0530 100644--- a/harness/bench/run_bench.py+++ b/harness/bench/run_bench.py@@ -199,6 +199,17 @@ def select_prompts(RunFn = Callable[[Spec, Workspace, bool], RunRecord]++def archive_attempt(root: Path) -> Path:+ """Move a run workspace aside as ``<root>.attempt<N>`` (N = first free) and return the new path."""+ n = 1+ while (root.parent / f"{root.name}.attempt{n}").exists():+ n += 1+ dest = root.parent / f"{root.name}.attempt{n}"+ root.rename(dest)+ return dest++def default_run_track(spec: Spec, ws: Workspace, resume: bool) -> RunRecord:"""Run the spec's track (the real thing; tests inject a fake ``run_fn``)."""from codeverse.tracks import get_track@@ -222,12 +233,20 @@ def run_battery(results_jsonl = out / "results.jsonl"done = _load_done(results_jsonl) if opts.resume else {}# `--redo-status infra_failed` re-runs the cells the weather lost, once it clears- for pid in [k for k, r in done.items() if r.status in set(opts.redo_status)]:+ redo_ids = {k for k, r in done.items() if r.status in set(opts.redo_status)}+ for pid in redo_ids:del done[pid]todo = [p for p in select_prompts(battery, ids=opts.ids, tiers=opts.tiers, limit=opts.limit) if p.id not in done]def _one(item: BenchPrompt) -> BenchItemResult:ws = Workspace(out / "runs" / item.id)+ if item.id in redo_ids and ws.exists():+ # A redo starts FRESH: resuming the old workspace keeps its spec (the old+ # max_minutes) and its clock, so a `budget` row redone with --max-minutes 120+ # was over budget before its first round (measured 2026-08-26: clock_q4 and+ # lighthouse_1 came back `budget`, 0 rounds, 60.3 / 76.7 min "elapsed").+ # The old tree is kept beside it as <id>.attempt<N>, the way ab_plan does.+ archive_attempt(ws.root)resume = ws.exists()if not resume:ws.create()diff --git a/harness/tests/flywheel_cli/test_bench.py b/harness/tests/flywheel_cli/test_bench.pyindex 41fd5c3c..4461d5b2 100644--- a/harness/tests/flywheel_cli/test_bench.py+++ b/harness/tests/flywheel_cli/test_bench.py@@ -206,3 +206,34 @@ def test_the_runner_classifies_the_outage_that_reaches_it(tmp_path: Path):assert calls == [], "a plain resume still skips every recorded row"run_battery(battery, out, BenchOptions(parallel=1, limit=1, redo_status=["infra_failed"]), run_fn=counting)assert len(calls) == 1, "--redo-status infra_failed re-runs what the weather lost"+++def test_a_redo_starts_from_a_fresh_workspace(tmp_path):+ """Measured 2026-08-26: a `budget` row redone with --max-minutes 120 resumed the old+ workspace (old spec, old clock) and came back `budget` with 0 rounds. The old tree+ is archived as <id>.attempt1 and the prompt runs fresh."""+ from bench.run_bench import archive_attempt++ out = tmp_path / "out"+ b = Battery.load(Path("bench/prompts/static_objects_v1.yaml"))+ pid = b.prompts[0].id+ ws_root = out / "runs" / pid+ (ws_root / "src").mkdir(parents=True)+ (ws_root / "src" / "old.py").write_text("# stale")+ out.mkdir(exist_ok=True)+ (out / "results.jsonl").write_text(json.dumps({"id": pid, "tier": b.prompts[0].tier, "status": "budget", "score_final": None}) + "\n")+ seen: list[bool] = []++ def run(spec, ws, resume):+ seen.append(resume)+ _ws, rec = make_fake_run(ws.root.parent, ws.root.name, prompt=spec.prompt, language=spec.language, scores=(0.5, 0.7))+ rec.spec = spec+ ws.write_json(ws.spec_path, spec)+ ws.write_json(ws.record_path, rec)+ return rec++ opts = BenchOptions(ids=[pid], redo_status=["budget"], parallel=1, rounds=1)+ run_battery(Path("bench/prompts/static_objects_v1.yaml"), out, opts, run_fn=run)+ assert seen == [False], "the redo ran fresh, not resumed"+ assert (out / "runs" / f"{pid}.attempt1" / "src" / "old.py").exists(), "the old tree is archived beside the new one"+ assert archive_attempt(out / "runs" / pid).name == f"{pid}.attempt2"