Skip to content

Official Codebase of 3DCodeVerse

Python

resume: no terminal-state guard, so `3dcv resume` corrupted finished runs

gaoypeng994c88783159
diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.py
index af0aed8f..5eff3d5e 100644
--- a/harness/codeverse/cli/main.py
+++ b/harness/codeverse/cli/main.py
@@ -185,6 +185,33 @@ def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None) -
raise typer.Exit(code=1)
+def _finished_reason(ws, raised: dict) -> str:
+ """Why this run must not be re-entered, or "" when resuming it is meaningful.
+
+ A run that reached a terminal state has nothing to resume, and re-entering it is
+ destructive, not idempotent: `3dcv resume` on a run that ended stop_reason='pass'
+ re-ran the plan stage as a real billed model call and rewrote run_state.status from
+ 'passed' back to 'planning', leaving a finished run stuck mid-pipeline while still
+ holding its best_score. A BUDGET stop is the documented exception — raising a cap is
+ how you continue one — so it only blocks when no cap was raised.
+ """
+ from codeverse.contracts.run import RunStatus
+ from codeverse.orchestrator.state import RunState, StateCorrupt
+
+ try:
+ state = RunState.load(ws)
+ except StateCorrupt:
+ return "" # let the track report it the way it always has
+ if state is None:
+ return ""
+ if state.status in (RunStatus.PASSED, RunStatus.PLATEAU) or (state.status is RunStatus.BUDGET and not raised):
+ detail = f"status={state.status.value}" + (f" stop_reason={state.stop_reason!r}" if state.stop_reason else "")
+ if state.status is RunStatus.BUDGET:
+ return f"{detail}: raise a cap to continue it (--max-usd / --max-minutes / --rounds)"
+ return f"{detail} best_score={state.best_score}"
+ return ""
+
+
@app.command()
def resume(
slug: str,
@@ -193,16 +220,23 @@ def resume(
max_usd: Annotated[float | None, typer.Option("--max-usd", help="raise the budget cap before resuming (rewrites spec.json)")] = None,
max_minutes: Annotated[float | None, typer.Option("--max-minutes", help="raise the time cap before resuming")] = None,
rounds: Annotated[int | None, typer.Option("--rounds", min=0, help="new max refine rounds (rewrites spec.json)")] = None,
+ force: Annotated[bool, typer.Option("--force", help="re-enter a run that already finished (it will be re-planned and re-scored)")] = False,
) -> None:
"""Resume an interrupted / partial run (or start a `--no-run` one).
``--max-usd`` / ``--max-minutes`` / ``--rounds`` rewrite the spec's budget
- first — the only way to continue a BUDGET-stopped run."""
+ first — the only way to continue a BUDGET-stopped run. A run that already
+ reached a terminal state is refused unless ``--force``: re-entering it spends
+ money and overwrites its final state."""
ws = C.open_workspace(slug, runs_dir)
spec = C.load_spec(ws)
if spec.options.profile: # the dial the run was created with (judge samples, montage px, turn cap)
get_settings().apply_profile(spec.options.profile, force=True)
raised = {k: v for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items() if v is not None}
+ if not force and (why := _finished_reason(ws, raised)):
+ raise C.CliError(f"run {ws.root.name} already finished ({why}); nothing to resume. "
+ f"`3dcv status {ws.root.name}` to look at it, or --force to re-enter it "
+ f"(that re-plans, re-scores and overwrites the final state).")
if raised:
from codeverse.events import EventLog
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 1d738871..edfd45e6 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -242,6 +242,53 @@ def test_resume_budget_flags_rewrite_spec_and_emit_event(tmp_path: Path, monkeyp
assert raised and raised[0]["max_usd"] == 4.5 and raised[0]["max_rounds"] == 3
+def test_resume_refuses_a_finished_run_and_never_re_enters_it(tmp_path: Path, monkeypatch):
+ """SMOKE1: `3dcv resume` called _run_track unconditionally, with no look at
+ run_state. On a run that had ended stop_reason='pass' it re-entered the pipeline,
+ re-ran the plan stage as a real billed model call and rewrote status from 'passed'
+ back to 'planning' — a finished run left stuck mid-pipeline, and money spent, from
+ one accidental or scripted resume."""
+ from codeverse.contracts.run import RunStatus
+ from codeverse.orchestrator.state import RunState
+ from codeverse.workspace import Workspace
+
+ runs = tmp_path / "runs"
+ assert runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run"]).exit_code == 0
+ ws = Workspace(next(d for d in runs.iterdir() if d.is_dir()))
+ state = RunState(status=RunStatus.PASSED, stop_reason="pass", best_score=0.7436)
+ state.save(ws)
+
+ entered = []
+
+ def fake_get_track(track, **options):
+ class _T:
+ def run(self, spec, ws_, *, resume=False):
+ entered.append(resume)
+ raise KeyboardInterrupt
+
+ return _T()
+
+ import codeverse.tracks as tracks_pkg
+
+ monkeypatch.setattr(tracks_pkg, "get_track", fake_get_track)
+ r = runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)])
+ assert r.exit_code == 1 and "already finished" in r.output
+ assert entered == [], "the pipeline must not be re-entered"
+ assert RunState.load(ws).status is RunStatus.PASSED, "the terminal state must survive"
+ # ... and the escape hatch still works, loudly
+ assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs), "--force"]).exit_code == 130
+ assert entered == [True]
+
+ # a budget stop is the documented exception: it resumes when a cap is raised
+ RunState(status=RunStatus.BUDGET, stop_reason="budget").save(ws)
+ assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)]).exit_code == 1
+ assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs), "--max-usd", "4.5"]).exit_code == 130
+
+ # an interrupted run is untouched by the guard
+ RunState(status=RunStatus.REFINING).save(ws)
+ assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)]).exit_code == 130
+
+
def test_status_shows_candidates_and_texturing(runs_dir: Path):
import json as _json