Official Codebase of 3DCodeVerse
Python
resume: a self-consistent state can no longer deliver the worse round
gaoypeng09c73cca4136
diff --git a/harness/codeverse/agents/cli_common.py b/harness/codeverse/agents/cli_common.pyindex 178b901a..aed50d55 100644--- a/harness/codeverse/agents/cli_common.py+++ b/harness/codeverse/agents/cli_common.py@@ -30,6 +30,7 @@ from codeverse.agents.transcript import Trajectoryfrom codeverse.agents.watchdog import CompletedProc, run_with_watchdogfrom codeverse.contracts.agent import AgentJob, AgentResult, FileChangefrom codeverse.contracts.common import Usage+from codeverse.proc import scrub_secretsfrom codeverse.workspace import Workspacelog = logging.getLogger(__name__)@@ -37,17 +38,12 @@ log = logging.getLogger(__name__)#: argv prompts above this many bytes are written to a file instead.MAX_ARGV_PROMPT_BYTES = 100_000-_SECRET_EXACT = {- "GEMINI_API_KEYS", "GEMINI_API_KEY", "GOOGLE_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",- "OPENAI_ORG_ID", "HF_TOKEN", "HUGGINGFACE_TOKEN", "GITHUB_TOKEN", "GH_TOKEN", "AWS_SECRET_ACCESS_KEY",- "AWS_SESSION_TOKEN", "SUPABASE_SERVICE_ROLE_KEY", "CODEX_API_KEY",-}-_SECRET_SUFFIXES = ("_API_KEY", "_SECRET", "_SECRET_KEY", "_TOKEN", "_AUTH_TOKEN", "_PASSWORD", "_PRIVATE_KEY")--def is_secret_env(name: str) -> bool:- """True for env vars that look like credentials (stripped from agent children)."""- return name in _SECRET_EXACT or name.endswith(_SECRET_SUFFIXES)+ """True for env vars that look like credentials (stripped from agent children).++ One owner for the patterns: :mod:`codeverse.proc` (a stdlib-only leaf every layer+ may import), which also scrubs the generated-code subprocesses."""+ return name not in scrub_secrets({name: ""})def hardened_env(ws: Workspace, job: AgentJob, *, keep: set[str] | None = None) -> dict[str, str]:diff --git a/harness/codeverse/agents/watchdog.py b/harness/codeverse/agents/watchdog.pyindex 6a42131d..131e0253 100644--- a/harness/codeverse/agents/watchdog.py+++ b/harness/codeverse/agents/watchdog.py@@ -17,10 +17,7 @@ tree (node → chrome, blender, mcp servers ...). The process lifecycle itselffrom __future__ import annotations-import contextlibimport os-import signal-import subprocessimport threadingimport timefrom collections.abc import Callable, Iterable, Mapping, Sequence@@ -83,24 +80,6 @@ def _latest_mtime(dirs: Iterable[Path]) -> float:return latest-def kill_process_group(proc: subprocess.Popen, grace_s: float = 5.0) -> None:- """SIGTERM the child's process group, then SIGKILL after ``grace_s``.-- Kept as the public utility for raw ``Popen`` holders; ``ManagedProcess.terminate``- implements the same contract for the managed path (``proc.py`` is stdlib-only and- cannot import this module)."""- if proc.poll() is not None:- return- try:- os.killpg(proc.pid, signal.SIGTERM)- except ProcessLookupError:- return- try:- proc.wait(timeout=grace_s)- except subprocess.TimeoutExpired:- with contextlib.suppress(ProcessLookupError):- os.killpg(proc.pid, signal.SIGKILL)- proc.wait(timeout=5)def run_with_watchdog(diff --git a/harness/codeverse/orchestrator/state.py b/harness/codeverse/orchestrator/state.pyindex d73f77fd..35516560 100644--- a/harness/codeverse/orchestrator/state.py+++ b/harness/codeverse/orchestrator/state.py@@ -38,6 +38,12 @@ class RunState(BaseModel):best_round: int | None = Nonebest_commit: str = ""best_score: float | None = None+ #: highest round index that has been THROUGH best selection (choose_best_round,+ #: including its paid pairwise comparison). Persisted so a resume can tell+ #: "the best was deliberately kept" from "this round was never considered" —+ #: without it, a state that merely LOOKS consistent with the journal kept a+ #: stale best and delivered the worse round (2026-08-27). -1 = nothing yet.+ best_considered_through: int = -1materialized_for: str = Field(default="", description="agent kind the workspace was materialised for")stop_reason: str = ""error: str = ""diff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.pyindex dce76957..ffc014e7 100644--- a/harness/codeverse/proc.py+++ b/harness/codeverse/proc.py@@ -325,12 +325,6 @@ class ManagedProcess:self.proc.wait(timeout=DRAIN_TIMEOUT_S) # reap — a zombie survives killpg-def kill_group(proc: subprocess.Popen[str]) -> None:- """SIGKILL ``proc``'s whole process group (fall back to just the child)."""- try:- os.killpg(proc.pid, signal.SIGKILL)- except (ProcessLookupError, PermissionError):- proc.kill()# ------------------------------------------------------------------ env scrubbingdiff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.pyindex 796cd2ad..936168e8 100644--- a/harness/codeverse/tracks/lifecycle.py+++ b/harness/codeverse/tracks/lifecycle.py@@ -95,6 +95,28 @@ def detail_round_budget(policy: RoundPolicy, supported: bool, *, explicit: bool)return policy.detail_rounds or DEFAULT_DETAIL_ROUNDS+def _reconcile_billed_from_ledger(budget: BudgetGuard, ws: Workspace, events: EventLog) -> None:+ """Adopt the run ledger's total when it exceeds the restored snapshot.++ The snapshot is saved at boundaries; the ledger is appended per CALL, so a crash+ between a round's spend and its save loses that money from the snapshot but never+ from the ledger. Taking the larger of the two makes resume charge for everything+ the provider actually billed — silently under-counting is how a resumed run walks+ past its ceiling. Attempt rows stay excluded (the winner is already on the logical+ row); a missing or unreadable ledger simply leaves the snapshot alone."""+ try:+ from codeverse.cost.ledger import load_ledger++ billed = sum(float(r.cost_usd or 0.0) for r in load_ledger(ws.root))+ except Exception as e: # noqa: BLE001 — accounting repair must never block a resume+ log.debug("ledger reconcile skipped: %s", e)+ return+ if billed > budget.billed_usd + 1e-9:+ events.emit("resume.billed_reconciled", snapshot_usd=round(budget.billed_usd, 6),+ ledger_usd=round(billed, 6))+ budget.billed_usd = billed++def run_ledger_path(ws: Workspace) -> Any:"""Where this run's priced per-call rows go.@@ -431,12 +453,16 @@ class BaseTrack:state.current_round = len(journal)best = state.best_roundbest_invalid = best is not None and (best >= len(journal) or journal[best].commit != state.best_commit)+ # a round the journal has but best selection never saw: the state can look+ # perfectly consistent (completed_rounds and commits agree) and still name a+ # stale best, which is then restored and DELIVERED over the newer paid round+ unranked = bool(journal) and state.best_considered_through < journal[-1].indexbest_changed = Falseif not journal:if best is not None:state.best_round, state.best_commit, state.best_score = None, "", Nonebest_changed = True- elif stale or best_invalid or best is None:+ elif stale or best_invalid or unranked or best is None:pick = BestSelector().pick(journal)if pick is None:best_changed = best is not None@@ -444,10 +470,12 @@ class BaseTrack:elif pick != best or journal[pick].commit != state.best_commit:state.update_best(pick, journal[pick].commit, journal[pick].score)best_changed = True+ if journal:+ state.best_considered_through = journal[-1].index # every journal round is now rankedstate.save(ws)events.emit("resume.reconciled", rounds=len(journal), dropped=dropped, state_was_stale=stale,- best_round=state.best_round, best_score=state.best_score, best_changed=best_changed,- spec_fingerprint=fp)+ unranked_rounds=unranked, best_round=state.best_round, best_score=state.best_score,+ best_changed=best_changed, spec_fingerprint=fp)return journal@@ -483,6 +511,7 @@ class BaseTrack:spent = state.extra.get("spent_usage") # legacy run dirs (pre-snapshot)if spent:budget.spent = Usage.model_validate(spent)+ _reconcile_billed_from_ledger(budget, ws, events)n_cand = self._resolve_candidates(spec, state, settings)policy = ((self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds)).with_candidates(n_cand).with_judge(spec.backends.judge))@@ -658,11 +687,16 @@ class BaseTrack:ctx.events.emit("stop", reason="no_change", rounds=len(rounds), best=ctx.state.best_round)return "plateau"rounds.append(rec)- # durable BEFORE choose_best_round's possible PAID pairwise call: a crash in- # that window left state.json without a round the journal already had, and- # an immediate stop on resume then restored/delivered a stale best.- ctx.state.mark_round_done(index, rec.commit, ctx.ws)+ # ONE save, AFTER the best is chosen — never a durable "round done" with a+ # not-yet-updated best. Saving the round first (an earlier attempt at+ # shrinking the paid-pairwise crash window) made a crash in that window look+ # SELF-CONSISTENT to reconcile_resume — journal and completed_rounds agreed,+ # so it kept the stale best and delivered the worse round (r0=0.5 kept over+ # r1=0.7). Crashing before this save leaves state behind the journal, which+ # reconcile detects (stale) and repairs by re-ranking; that is the safe side.+ ctx.state.mark_round_done(index, rec.commit)best = choose_best_round(ctx, rounds, selector, index)+ ctx.state.best_considered_through = index # this round HAS been rankedif best is not None and ctx.state.update_best(best, rounds[best].commit, rounds[best].score):ctx.events.emit("best.updated", round=best, score=rounds[best].score)self._save_budget(ctx)diff --git a/harness/tests/core/test_proc.py b/harness/tests/core/test_proc.pyindex 1b4f888a..efd2b395 100644--- a/harness/tests/core/test_proc.py+++ b/harness/tests/core/test_proc.py@@ -19,7 +19,6 @@ from codeverse.proc import (ProcResult,append_jsonl_line,iter_jsonl_lines,- kill_group,read_json_or_none,read_jsonl_lenient,run_subprocess,@@ -102,13 +101,6 @@ def test_preexec_fn_runs_in_the_child(tmp_path: Path):assert r.stdout.strip() == str(soft)-def test_kill_group_is_public_and_tolerates_dead_proc(tmp_path: Path):- import subprocess-- proc = subprocess.Popen(["sleep", "0.05"], start_new_session=True)- proc.wait()- kill_group(proc) # must not raise on an already-reaped child-def test_tail_caps_lines_and_chars():text = "\n".join(f"line{i}" for i in range(100))@@ -293,22 +285,6 @@ def test_scrub_secrets_keeps_everything_generated_code_needs():assert scrub_secrets(dict(env)) == env-def test_scrub_secrets_stays_in_sync_with_agents_cli_common():- """proc.scrub_secrets is a COPY of agents/cli_common.is_secret_env (proc.py must- stay stdlib-only) — this pins the two pattern sets together."""- from codeverse.agents.cli_common import is_secret_env- from codeverse.proc import _SECRET_EXACT, _SECRET_SUFFIXES-- probes = (sorted(_SECRET_EXACT) + [f"X{s}" for s in _SECRET_SUFFIXES]- + ["PATH", "HOME", "NODE_PATH", "TOKENIZERS_PARALLELISM"])- for name in probes:- assert is_secret_env(name) == (name not in scrub_secrets({name: "v"})), name---# --------------------------------------------------------------------------- ManagedProcess lifecycle-def _raise_ki(signum: int, frame: object) -> None:- raise KeyboardInterrupt-def _assert_group_gone(pgid: int, timeout_s: float = 10.0) -> None:deadline = time.monotonic() + timeout_s@@ -383,6 +359,11 @@ def test_huge_output_is_bounded_head_and_tail(tmp_path: Path):assert "x" * 1000 in r.stdout[:cut] and "x" * 1000 in r.stdout[cut:]+# --------------------------------------------------------------------------- ManagedProcess lifecycle+def _raise_ki(signum: int, frame: object) -> None:+ raise KeyboardInterrupt++def test_managed_process_reaps_on_exit_no_zombie(tmp_path: Path):with ManagedProcess([sys.executable, "-c", "print('hi')"], cwd=tmp_path) as mp:mp.wait(timeout=30)diff --git a/harness/tests/orchestrator_tracks/test_resume_reconcile.py b/harness/tests/orchestrator_tracks/test_resume_reconcile.pyindex fda68222..b43d4fd9 100644--- a/harness/tests/orchestrator_tracks/test_resume_reconcile.py+++ b/harness/tests/orchestrator_tracks/test_resume_reconcile.py@@ -190,3 +190,58 @@ def test_notes_survive_a_judge_crash(tmp_path, chair_plan, settings):assert "judge failed: RuntimeError: vlm 500" in r1.notessaved = json.loads((ws.root / "rounds" / "r01.json").read_text())assert "judge failed: RuntimeError: vlm 500" in saved["notes"]+++def test_a_self_consistent_state_with_an_unranked_round_still_re_ranks(tmp_path, chair_plan, settings):+ """The OTHER crash shape, and the sharper one (reproduced by review, 2026-08-27).++ While the round was saved BEFORE best selection, a crash in between left a state+ that looked perfectly consistent — completed_rounds and round_commits matched the+ journal exactly — while best_round still pointed at the older, worse round. Nothing+ was 'stale', so reconcile kept it and the run DELIVERED r0=0.5 over the paid r1=0.7.+ The writer now saves once, after ranking; `best_considered_through` makes the repair+ independent of write ordering, so even such a state (an older run dir, a partial+ write) is re-ranked rather than believed."""+ spec = make_spec(language=Language.BLENDER, max_rounds=1)+ ws = Workspace(tmp_path / "runs" / "r")+ planner = _planner(chair_plan.model_dump(mode="json"))+ pol = RoundPolicy(max_rounds=1, target=0.9)+ rec1 = _track(planner, settings, policy=pol).run(spec, ws)+ assert rec1.best_round == 1 and rec1.rounds[1].score == pytest.approx(0.7)++ state = json.loads(ws.state_path.read_text()) # journal-consistent, best stale+ assert state["completed_rounds"] == [0, 1]+ state["best_round"], state["best_commit"], state["best_score"] = 0, rec1.rounds[0].commit, 0.5+ state["best_considered_through"] = 0 # r1 was written but never ranked+ ws.state_path.write_text(json.dumps(state))+ ws.restore(rec1.rounds[0].commit)+ ws.commit("stale delivery")++ rec2 = _track(planner, settings, policy=pol).run(spec, ws, resume=True)+ assert rec2.best_round == 1 and rec2.final_score == pytest.approx(0.7)+ ev = [e for e in EventLog(ws.events_path).read() if e["event"] == "resume.reconciled"][-1]+ assert ev["state_was_stale"] is False and ev["unranked_rounds"] is True and ev["best_changed"] is True+++def test_resume_charges_for_spend_the_snapshot_missed(tmp_path, chair_plan, settings):+ """The ledger is appended per CALL; the snapshot is saved at boundaries. A crash+ between a round's spend and its save used to hand the resumed run that money back —+ silently under-counting is how a resumed run walks past its ceiling."""+ from codeverse.cost.ledger import open_run_ledger+ from codeverse.cost.types import CallCost+ from codeverse.orchestrator.budget import BudgetGuard, BudgetSnapshot+ from codeverse.tracks.lifecycle import _reconcile_billed_from_ledger++ ws = Workspace(tmp_path / "runs" / "r")+ ws.create()+ led = open_run_ledger(ws.root)+ for cost in (0.30, 0.12): # what the provider actually billed+ led.append(CallCost(run=ws.root.name, model="gemini:flash", label="planner", cost_usd=cost))+ guard = BudgetGuard(make_spec().budget, run=ws.root.name)+ guard.restore(BudgetSnapshot(spent=guard.spent, billed_usd=0.10, # the boundary save missed 0.32+ calls=1, by_stage={}, by_round={}, active_s=0.0))+ _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))+ assert guard.billed_usd == pytest.approx(0.42), "resume must charge for every billed call"+ guard.billed_usd = 5.0 # a snapshot AHEAD of the ledger wins+ _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))+ assert guard.billed_usd == pytest.approx(5.0), "reconcile never lowers what was already billed"