Skip to content

Official Codebase of 3DCodeVerse

Python

resume: a self-consistent state can no longer deliver the worse round

gaoypeng09c73cca4136
diff --git a/harness/codeverse/agents/cli_common.py b/harness/codeverse/agents/cli_common.py
index 178b901a..aed50d55 100644
--- a/harness/codeverse/agents/cli_common.py
+++ b/harness/codeverse/agents/cli_common.py
@@ -30,6 +30,7 @@ from codeverse.agents.transcript import Trajectory
from codeverse.agents.watchdog import CompletedProc, run_with_watchdog
from codeverse.contracts.agent import AgentJob, AgentResult, FileChange
from codeverse.contracts.common import Usage
+from codeverse.proc import scrub_secrets
from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
@@ -37,17 +38,12 @@ log = logging.getLogger(__name__)
#: argv prompts above this many bytes are written to a file instead.
MAX_ARGV_PROMPT_BYTES = 100_000
-_SECRET_EXACT = {
- "GEMINI_API_KEYS", "GEMINI_API_KEY", "GOOGLE_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",
- "OPENAI_ORG_ID", "HF_TOKEN", "HUGGINGFACE_TOKEN", "GITHUB_TOKEN", "GH_TOKEN", "AWS_SECRET_ACCESS_KEY",
- "AWS_SESSION_TOKEN", "SUPABASE_SERVICE_ROLE_KEY", "CODEX_API_KEY",
-}
-_SECRET_SUFFIXES = ("_API_KEY", "_SECRET", "_SECRET_KEY", "_TOKEN", "_AUTH_TOKEN", "_PASSWORD", "_PRIVATE_KEY")
-
-
def is_secret_env(name: str) -> bool:
- """True for env vars that look like credentials (stripped from agent children)."""
- return name in _SECRET_EXACT or name.endswith(_SECRET_SUFFIXES)
+ """True for env vars that look like credentials (stripped from agent children).
+
+ One owner for the patterns: :mod:`codeverse.proc` (a stdlib-only leaf every layer
+ may import), which also scrubs the generated-code subprocesses."""
+ return name not in scrub_secrets({name: ""})
def hardened_env(ws: Workspace, job: AgentJob, *, keep: set[str] | None = None) -> dict[str, str]:
diff --git a/harness/codeverse/agents/watchdog.py b/harness/codeverse/agents/watchdog.py
index 6a42131d..131e0253 100644
--- a/harness/codeverse/agents/watchdog.py
+++ b/harness/codeverse/agents/watchdog.py
@@ -17,10 +17,7 @@ tree (node → chrome, blender, mcp servers ...). The process lifecycle itself
from __future__ import annotations
-import contextlib
import os
-import signal
-import subprocess
import threading
import time
from collections.abc import Callable, Iterable, Mapping, Sequence
@@ -83,24 +80,6 @@ def _latest_mtime(dirs: Iterable[Path]) -> float:
return latest
-def kill_process_group(proc: subprocess.Popen, grace_s: float = 5.0) -> None:
- """SIGTERM the child's process group, then SIGKILL after ``grace_s``.
-
- Kept as the public utility for raw ``Popen`` holders; ``ManagedProcess.terminate``
- implements the same contract for the managed path (``proc.py`` is stdlib-only and
- cannot import this module)."""
- if proc.poll() is not None:
- return
- try:
- os.killpg(proc.pid, signal.SIGTERM)
- except ProcessLookupError:
- return
- try:
- proc.wait(timeout=grace_s)
- except subprocess.TimeoutExpired:
- with contextlib.suppress(ProcessLookupError):
- os.killpg(proc.pid, signal.SIGKILL)
- proc.wait(timeout=5)
def run_with_watchdog(
diff --git a/harness/codeverse/orchestrator/state.py b/harness/codeverse/orchestrator/state.py
index d73f77fd..35516560 100644
--- a/harness/codeverse/orchestrator/state.py
+++ b/harness/codeverse/orchestrator/state.py
@@ -38,6 +38,12 @@ class RunState(BaseModel):
best_round: int | None = None
best_commit: str = ""
best_score: float | None = None
+ #: highest round index that has been THROUGH best selection (choose_best_round,
+ #: including its paid pairwise comparison). Persisted so a resume can tell
+ #: "the best was deliberately kept" from "this round was never considered" —
+ #: without it, a state that merely LOOKS consistent with the journal kept a
+ #: stale best and delivered the worse round (2026-08-27). -1 = nothing yet.
+ best_considered_through: int = -1
materialized_for: str = Field(default="", description="agent kind the workspace was materialised for")
stop_reason: str = ""
error: str = ""
diff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.py
index dce76957..ffc014e7 100644
--- a/harness/codeverse/proc.py
+++ b/harness/codeverse/proc.py
@@ -325,12 +325,6 @@ class ManagedProcess:
self.proc.wait(timeout=DRAIN_TIMEOUT_S) # reap — a zombie survives killpg
-def kill_group(proc: subprocess.Popen[str]) -> None:
- """SIGKILL ``proc``'s whole process group (fall back to just the child)."""
- try:
- os.killpg(proc.pid, signal.SIGKILL)
- except (ProcessLookupError, PermissionError):
- proc.kill()
# ------------------------------------------------------------------ env scrubbing
diff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.py
index 796cd2ad..936168e8 100644
--- a/harness/codeverse/tracks/lifecycle.py
+++ b/harness/codeverse/tracks/lifecycle.py
@@ -95,6 +95,28 @@ def detail_round_budget(policy: RoundPolicy, supported: bool, *, explicit: bool)
return policy.detail_rounds or DEFAULT_DETAIL_ROUNDS
+def _reconcile_billed_from_ledger(budget: BudgetGuard, ws: Workspace, events: EventLog) -> None:
+ """Adopt the run ledger's total when it exceeds the restored snapshot.
+
+ The snapshot is saved at boundaries; the ledger is appended per CALL, so a crash
+ between a round's spend and its save loses that money from the snapshot but never
+ from the ledger. Taking the larger of the two makes resume charge for everything
+ the provider actually billed — silently under-counting is how a resumed run walks
+ past its ceiling. Attempt rows stay excluded (the winner is already on the logical
+ row); a missing or unreadable ledger simply leaves the snapshot alone."""
+ try:
+ from codeverse.cost.ledger import load_ledger
+
+ billed = sum(float(r.cost_usd or 0.0) for r in load_ledger(ws.root))
+ except Exception as e: # noqa: BLE001 — accounting repair must never block a resume
+ log.debug("ledger reconcile skipped: %s", e)
+ return
+ if billed > budget.billed_usd + 1e-9:
+ events.emit("resume.billed_reconciled", snapshot_usd=round(budget.billed_usd, 6),
+ ledger_usd=round(billed, 6))
+ budget.billed_usd = billed
+
+
def run_ledger_path(ws: Workspace) -> Any:
"""Where this run's priced per-call rows go.
@@ -431,12 +453,16 @@ class BaseTrack:
state.current_round = len(journal)
best = state.best_round
best_invalid = best is not None and (best >= len(journal) or journal[best].commit != state.best_commit)
+ # a round the journal has but best selection never saw: the state can look
+ # perfectly consistent (completed_rounds and commits agree) and still name a
+ # stale best, which is then restored and DELIVERED over the newer paid round
+ unranked = bool(journal) and state.best_considered_through < journal[-1].index
best_changed = False
if not journal:
if best is not None:
state.best_round, state.best_commit, state.best_score = None, "", None
best_changed = True
- elif stale or best_invalid or best is None:
+ elif stale or best_invalid or unranked or best is None:
pick = BestSelector().pick(journal)
if pick is None:
best_changed = best is not None
@@ -444,10 +470,12 @@ class BaseTrack:
elif pick != best or journal[pick].commit != state.best_commit:
state.update_best(pick, journal[pick].commit, journal[pick].score)
best_changed = True
+ if journal:
+ state.best_considered_through = journal[-1].index # every journal round is now ranked
state.save(ws)
events.emit("resume.reconciled", rounds=len(journal), dropped=dropped, state_was_stale=stale,
- best_round=state.best_round, best_score=state.best_score, best_changed=best_changed,
- spec_fingerprint=fp)
+ unranked_rounds=unranked, best_round=state.best_round, best_score=state.best_score,
+ best_changed=best_changed, spec_fingerprint=fp)
return journal
@@ -483,6 +511,7 @@ class BaseTrack:
spent = state.extra.get("spent_usage") # legacy run dirs (pre-snapshot)
if spent:
budget.spent = Usage.model_validate(spent)
+ _reconcile_billed_from_ledger(budget, ws, events)
n_cand = self._resolve_candidates(spec, state, settings)
policy = ((self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds))
.with_candidates(n_cand).with_judge(spec.backends.judge))
@@ -658,11 +687,16 @@ class BaseTrack:
ctx.events.emit("stop", reason="no_change", rounds=len(rounds), best=ctx.state.best_round)
return "plateau"
rounds.append(rec)
- # durable BEFORE choose_best_round's possible PAID pairwise call: a crash in
- # that window left state.json without a round the journal already had, and
- # an immediate stop on resume then restored/delivered a stale best.
- ctx.state.mark_round_done(index, rec.commit, ctx.ws)
+ # ONE save, AFTER the best is chosen — never a durable "round done" with a
+ # not-yet-updated best. Saving the round first (an earlier attempt at
+ # shrinking the paid-pairwise crash window) made a crash in that window look
+ # SELF-CONSISTENT to reconcile_resume — journal and completed_rounds agreed,
+ # so it kept the stale best and delivered the worse round (r0=0.5 kept over
+ # r1=0.7). Crashing before this save leaves state behind the journal, which
+ # reconcile detects (stale) and repairs by re-ranking; that is the safe side.
+ ctx.state.mark_round_done(index, rec.commit)
best = choose_best_round(ctx, rounds, selector, index)
+ ctx.state.best_considered_through = index # this round HAS been ranked
if best is not None and ctx.state.update_best(best, rounds[best].commit, rounds[best].score):
ctx.events.emit("best.updated", round=best, score=rounds[best].score)
self._save_budget(ctx)
diff --git a/harness/tests/core/test_proc.py b/harness/tests/core/test_proc.py
index 1b4f888a..efd2b395 100644
--- a/harness/tests/core/test_proc.py
+++ b/harness/tests/core/test_proc.py
@@ -19,7 +19,6 @@ from codeverse.proc import (
ProcResult,
append_jsonl_line,
iter_jsonl_lines,
- kill_group,
read_json_or_none,
read_jsonl_lenient,
run_subprocess,
@@ -102,13 +101,6 @@ def test_preexec_fn_runs_in_the_child(tmp_path: Path):
assert r.stdout.strip() == str(soft)
-def test_kill_group_is_public_and_tolerates_dead_proc(tmp_path: Path):
- import subprocess
-
- proc = subprocess.Popen(["sleep", "0.05"], start_new_session=True)
- proc.wait()
- kill_group(proc) # must not raise on an already-reaped child
-
def test_tail_caps_lines_and_chars():
text = "\n".join(f"line{i}" for i in range(100))
@@ -293,22 +285,6 @@ def test_scrub_secrets_keeps_everything_generated_code_needs():
assert scrub_secrets(dict(env)) == env
-def test_scrub_secrets_stays_in_sync_with_agents_cli_common():
- """proc.scrub_secrets is a COPY of agents/cli_common.is_secret_env (proc.py must
- stay stdlib-only) — this pins the two pattern sets together."""
- from codeverse.agents.cli_common import is_secret_env
- from codeverse.proc import _SECRET_EXACT, _SECRET_SUFFIXES
-
- probes = (sorted(_SECRET_EXACT) + [f"X{s}" for s in _SECRET_SUFFIXES]
- + ["PATH", "HOME", "NODE_PATH", "TOKENIZERS_PARALLELISM"])
- for name in probes:
- assert is_secret_env(name) == (name not in scrub_secrets({name: "v"})), name
-
-
-# --------------------------------------------------------------------------- ManagedProcess lifecycle
-def _raise_ki(signum: int, frame: object) -> None:
- raise KeyboardInterrupt
-
def _assert_group_gone(pgid: int, timeout_s: float = 10.0) -> None:
deadline = time.monotonic() + timeout_s
@@ -383,6 +359,11 @@ def test_huge_output_is_bounded_head_and_tail(tmp_path: Path):
assert "x" * 1000 in r.stdout[:cut] and "x" * 1000 in r.stdout[cut:]
+# --------------------------------------------------------------------------- ManagedProcess lifecycle
+def _raise_ki(signum: int, frame: object) -> None:
+ raise KeyboardInterrupt
+
+
def test_managed_process_reaps_on_exit_no_zombie(tmp_path: Path):
with ManagedProcess([sys.executable, "-c", "print('hi')"], cwd=tmp_path) as mp:
mp.wait(timeout=30)
diff --git a/harness/tests/orchestrator_tracks/test_resume_reconcile.py b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
index fda68222..b43d4fd9 100644
--- a/harness/tests/orchestrator_tracks/test_resume_reconcile.py
+++ b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
@@ -190,3 +190,58 @@ def test_notes_survive_a_judge_crash(tmp_path, chair_plan, settings):
assert "judge failed: RuntimeError: vlm 500" in r1.notes
saved = json.loads((ws.root / "rounds" / "r01.json").read_text())
assert "judge failed: RuntimeError: vlm 500" in saved["notes"]
+
+
+def test_a_self_consistent_state_with_an_unranked_round_still_re_ranks(tmp_path, chair_plan, settings):
+ """The OTHER crash shape, and the sharper one (reproduced by review, 2026-08-27).
+
+ While the round was saved BEFORE best selection, a crash in between left a state
+ that looked perfectly consistent — completed_rounds and round_commits matched the
+ journal exactly — while best_round still pointed at the older, worse round. Nothing
+ was 'stale', so reconcile kept it and the run DELIVERED r0=0.5 over the paid r1=0.7.
+ The writer now saves once, after ranking; `best_considered_through` makes the repair
+ independent of write ordering, so even such a state (an older run dir, a partial
+ write) is re-ranked rather than believed."""
+ spec = make_spec(language=Language.BLENDER, max_rounds=1)
+ ws = Workspace(tmp_path / "runs" / "r")
+ planner = _planner(chair_plan.model_dump(mode="json"))
+ pol = RoundPolicy(max_rounds=1, target=0.9)
+ rec1 = _track(planner, settings, policy=pol).run(spec, ws)
+ assert rec1.best_round == 1 and rec1.rounds[1].score == pytest.approx(0.7)
+
+ state = json.loads(ws.state_path.read_text()) # journal-consistent, best stale
+ assert state["completed_rounds"] == [0, 1]
+ state["best_round"], state["best_commit"], state["best_score"] = 0, rec1.rounds[0].commit, 0.5
+ state["best_considered_through"] = 0 # r1 was written but never ranked
+ ws.state_path.write_text(json.dumps(state))
+ ws.restore(rec1.rounds[0].commit)
+ ws.commit("stale delivery")
+
+ rec2 = _track(planner, settings, policy=pol).run(spec, ws, resume=True)
+ assert rec2.best_round == 1 and rec2.final_score == pytest.approx(0.7)
+ ev = [e for e in EventLog(ws.events_path).read() if e["event"] == "resume.reconciled"][-1]
+ assert ev["state_was_stale"] is False and ev["unranked_rounds"] is True and ev["best_changed"] is True
+
+
+def test_resume_charges_for_spend_the_snapshot_missed(tmp_path, chair_plan, settings):
+ """The ledger is appended per CALL; the snapshot is saved at boundaries. A crash
+ between a round's spend and its save used to hand the resumed run that money back —
+ silently under-counting is how a resumed run walks past its ceiling."""
+ from codeverse.cost.ledger import open_run_ledger
+ from codeverse.cost.types import CallCost
+ from codeverse.orchestrator.budget import BudgetGuard, BudgetSnapshot
+ from codeverse.tracks.lifecycle import _reconcile_billed_from_ledger
+
+ ws = Workspace(tmp_path / "runs" / "r")
+ ws.create()
+ led = open_run_ledger(ws.root)
+ for cost in (0.30, 0.12): # what the provider actually billed
+ led.append(CallCost(run=ws.root.name, model="gemini:flash", label="planner", cost_usd=cost))
+ guard = BudgetGuard(make_spec().budget, run=ws.root.name)
+ guard.restore(BudgetSnapshot(spent=guard.spent, billed_usd=0.10, # the boundary save missed 0.32
+ calls=1, by_stage={}, by_round={}, active_s=0.0))
+ _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))
+ assert guard.billed_usd == pytest.approx(0.42), "resume must charge for every billed call"
+ guard.billed_usd = 5.0 # a snapshot AHEAD of the ledger wins
+ _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))
+ assert guard.billed_usd == pytest.approx(5.0), "reconcile never lowers what was already billed"