Skip to content

Official Codebase of 3DCodeVerse

Python

StageRunner: an unreadable cached stage result bricked `3dcv resume` forever

gaoypengc717d7639649
diff --git a/harness/codeverse/orchestrator/runner.py b/harness/codeverse/orchestrator/runner.py
index abbc8e99..aad24e4b 100644
--- a/harness/codeverse/orchestrator/runner.py
+++ b/harness/codeverse/orchestrator/runner.py
@@ -6,12 +6,18 @@ file (``<ws>/stages/<name>.json``) exists, the cached result is returned
without running ``fn`` (content-addressed skip — no mtime heuristics).
Results may be pydantic models, lists of models, plain JSON values, or
``Path``s; pass ``model=`` to re-validate a cached JSON into its type.
+
+The hash covers the *inputs* only, never the result model's schema, so a cached
+file that no longer reads back — a contract that gained a field, a clobbered
+write — is treated as a cache MISS and re-run rather than raising out of the
+resume path.
"""
from __future__ import annotations
import hashlib
import json
+import logging
import re
import time
from collections.abc import Callable
@@ -24,6 +30,8 @@ from codeverse.events import EventLog
from codeverse.orchestrator.state import RunState, StageState
from codeverse.workspace import Workspace
+log = logging.getLogger(__name__)
+
T = TypeVar("T")
_SAFE = re.compile(r"[^0-9A-Za-z._-]+")
@@ -80,10 +88,23 @@ class StageRunner:
path = self.result_path(name)
prior = self.state.stages.get(name)
if not force and prior is not None and prior.inputs_hash == h and path.is_file():
- data = json.loads(path.read_text())
- result = _revive(data, model, list_of)
- self.events.emit("stage.cached", stage=name, inputs_hash=h, path=str(path))
- return result # type: ignore[return-value]
+ # A cached result that cannot be read back is a cache MISS, not a dead run.
+ # ``inputs_hash`` covers the INPUTS only, never the result model's schema, so a
+ # contract that gained a field invalidates nothing — and a clobbered file
+ # invalidates nothing either. Both used to escape as ValidationError /
+ # JSONDecodeError through BaseTrack.run, which marks the run FAILED and
+ # re-raises, so every later `3dcv resume <slug>` died the same way with no way
+ # out (StageRunner.invalidate has no caller and no flag). Re-run instead and
+ # overwrite the file — the same tolerance load_ledger and _read_jsonl apply.
+ try:
+ result = _revive(json.loads(path.read_text()), model, list_of)
+ except (OSError, ValueError) as e: # ValidationError is a ValueError
+ self.events.emit("stage.cache_invalid", stage=name, inputs_hash=h, path=str(path),
+ error=f"{type(e).__name__}: {e}")
+ log.warning("stage %s: cached result at %s is unusable (%s); re-running", name, path, e)
+ else:
+ self.events.emit("stage.cached", stage=name, inputs_hash=h, path=str(path))
+ return result # type: ignore[return-value]
self.events.emit("stage.start", stage=name, inputs_hash=h)
t0 = time.time()
diff --git a/harness/tests/orchestrator_tracks/test_orchestrator.py b/harness/tests/orchestrator_tracks/test_orchestrator.py
index 2dd4ffca..8fa05de5 100644
--- a/harness/tests/orchestrator_tracks/test_orchestrator.py
+++ b/harness/tests/orchestrator_tracks/test_orchestrator.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import time
import pytest
+from pydantic import BaseModel
from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Severity
from codeverse.contracts.common import Budget, Usage
@@ -96,6 +97,41 @@ def test_stage_runner_caches_by_input_hash(tmp_ws):
assert runner.result_path("a:b/c").name == "a_b_c.json"
+def test_an_unreadable_cached_stage_is_a_miss_not_a_dead_run(tmp_ws):
+ """RS-2: inputs_hash covers the INPUTS, never the result model's schema. A cached
+ result that no longer validates (the contract gained a field) or no longer parses
+ (a clobbered file) used to escape as ValidationError / JSONDecodeError, which
+ BaseTrack.run turns into a FAILED run — so every later `3dcv resume` died the same
+ way. Both must re-run the stage and overwrite the file."""
+
+ class PlanV1(BaseModel):
+ name: str
+
+ class PlanV2(BaseModel):
+ name: str
+ units: str # added after the cache was written
+
+ events = EventLog(tmp_ws.events_path)
+ runner = StageRunner(tmp_ws, events)
+ runner.stage("plan", lambda: PlanV1(name="stool"), inputs={"prompt": "a stool"}, model=PlanV1)
+
+ calls = []
+
+ def v2():
+ calls.append(1)
+ return PlanV2(name="stool", units="m")
+
+ r2 = StageRunner(tmp_ws, events, RunState.load(tmp_ws))
+ assert r2.stage("plan", v2, inputs={"prompt": "a stool"}, model=PlanV2).units == "m"
+ assert len(calls) == 1, "a cached result that does not validate must re-run the stage"
+
+ r2.result_path("plan").write_text('{"stage": "plan", "inputs_hash": ') # clobbered mid-write
+ r3 = StageRunner(tmp_ws, events, RunState.load(tmp_ws))
+ assert r3.stage("plan", v2, inputs={"prompt": "a stool"}, model=PlanV2).units == "m"
+ assert len(calls) == 2
+ assert [e["event"] for e in events.read()].count("stage.cache_invalid") == 2
+
+
def test_stage_runner_revives_models_and_raises(tmp_ws):
runner = StageRunner(tmp_ws, EventLog(tmp_ws.events_path))
r = runner.stage("rounds", lambda: [RoundRecord(index=0, kind="baseline")], inputs="x", list_of=RoundRecord)