Skip to content

Official Codebase of 3DCodeVerse

Python

resume: the round journal is the truth; judge money and notes stop vanishing

gaoypeng89b2864a67d2
diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.py
index 31cfbe69..aacefc04 100644
--- a/harness/codeverse/cli/main.py
+++ b/harness/codeverse/cli/main.py
@@ -321,7 +321,7 @@ def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:
return grounded
-def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None) -> None:
+def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None, force: bool = False) -> None:
from codeverse.cost.instrument import run_ledger
from codeverse.runlock import RunLocked, run_lock
@@ -335,7 +335,7 @@ def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None) -
run_lock(ws.root, what=f"3dcv {'resume' if resume else 'make'} {ws.root.name}"),
run_ledger(ws.root, run=ws.root.name),
):
- record = get_track(spec.track, **options).run(spec, ws, resume=resume)
+ record = get_track(spec.track, **options).run(spec, ws, resume=resume, force=force)
except RunLocked as e:
raise C.CliError(str(e), code=2) from None
except KeyboardInterrupt:
@@ -343,6 +343,12 @@ def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None) -
f"interrupted; resume with `3dcv resume {ws.root.name}`", code=130
) from None
except Exception as e: # the track failed outside its own error handling
+ from codeverse.tracks.lifecycle import SpecChanged
+
+ if isinstance(e, SpecChanged):
+ # a refusal with instructions, not a crash: no traceback, and the run
+ # itself was never entered (reconcile_resume raises before any stage)
+ raise C.CliError(str(e), code=2) from None
err_console.print_exception(max_frames=8)
raise C.CliError(
f"run failed: {type(e).__name__}: {e} (workspace {ws.root}; see events.jsonl)"
@@ -413,7 +419,9 @@ def resume(
bool,
typer.Option(
"--force",
- help="re-enter a run that already finished (it will be re-planned and re-scored)",
+ help="re-enter a run that already finished (it will be re-planned and re-scored), "
+ "and allow resuming after a spec edit — the old rounds and record are archived "
+ "under rounds/pre_force/ and the run re-plans from the edited spec",
),
] = False,
) -> None:
@@ -449,7 +457,7 @@ def resume(
spec = spec.model_copy(update={"budget": spec.budget.model_copy(update=raised)})
ws.write_json(ws.spec_path, spec)
EventLog(ws.events_path).emit("budget.raised", **raised)
- _run_track(spec, ws, resume=True, candidates=candidates)
+ _run_track(spec, ws, resume=True, candidates=candidates, force=force)
# --------------------------------------------------------------------------- status / render / judge
diff --git a/harness/codeverse/models/gemini_convert.py b/harness/codeverse/models/gemini_convert.py
index 707ba804..dd547234 100644
--- a/harness/codeverse/models/gemini_convert.py
+++ b/harness/codeverse/models/gemini_convert.py
@@ -42,11 +42,27 @@ SIGNATURES: BoundedCache[bytes] = BoundedCache(4096)
_call_counter = [0]
_call_lock = threading.Lock()
+#: prefix of the ids WE mint when the provider sent none (``extract_candidate``).
+#: A synthetic id exists only to pair ToolCallPart/ToolResultPart locally and is
+#: NEVER echoed back to the provider; a GENUINE provider id is echoed verbatim on
+#: both ``FunctionCall`` and ``FunctionResponse``.
+SYNTHETIC_CALL_ID_PREFIX = "synth-call-"
+
def _next_call_id() -> str:
with _call_lock:
_call_counter[0] += 1
- return f"call_{_call_counter[0]:06d}"
+ return f"{SYNTHETIC_CALL_ID_PREFIX}{_call_counter[0]:06d}"
+
+
+def is_synthetic_call_id(call_id: str | None) -> bool:
+ """True for ids minted here rather than issued by the provider."""
+ return bool(call_id) and str(call_id).startswith(SYNTHETIC_CALL_ID_PREFIX)
+
+
+def _echo_id(call_id: str | None) -> str | None:
+ """The id to send back to Gemini: the provider's own id verbatim, never a synthetic one."""
+ return None if not call_id or is_synthetic_call_id(call_id) else str(call_id)
# ------------------------------------------------------------------ contents
@@ -68,7 +84,9 @@ def to_contents(messages: list[ChatMessage]) -> list[types.Content]:
parts.append(types.Part.from_bytes(data=raw, mime_type=mime))
elif isinstance(p, ToolCallPart):
part = types.Part(
- function_call=types.FunctionCall(name=p.name, args=dict(p.arguments))
+ function_call=types.FunctionCall(
+ name=p.name, args=dict(p.arguments), id=_echo_id(p.id)
+ )
)
sig = SIGNATURES.get(p.id)
if sig:
@@ -102,7 +120,9 @@ def _function_response_part(p: ToolResultPart) -> types.Part:
payload = {"result": p.content}
if p.is_error:
payload = {"error": payload.get("result", payload)} if "error" not in payload else payload
- return types.Part(function_response=types.FunctionResponse(name=p.name, response=payload))
+ return types.Part(
+ function_response=types.FunctionResponse(name=p.name, response=payload, id=_echo_id(p.call_id))
+ )
# -------------------------------------------------------------------- config
@@ -203,6 +223,8 @@ def extract_candidate(resp: types.GenerateContentResponse) -> tuple[str, list[To
for part in cand.content.parts if cand.content and cand.content.parts else []:
if part.function_call is not None:
fc = part.function_call
+ # a genuine provider id is kept verbatim (and echoed back later); a synthetic
+ # one is minted, marked by its prefix, only so the parts pair locally
call_id = fc.id or _next_call_id()
SIGNATURES.put(call_id, part.thought_signature)
calls.append(
diff --git a/harness/codeverse/orchestrator/rounds.py b/harness/codeverse/orchestrator/rounds.py
index e1a96cf1..bf05dd18 100644
--- a/harness/codeverse/orchestrator/rounds.py
+++ b/harness/codeverse/orchestrator/rounds.py
@@ -153,6 +153,20 @@ class RoundPolicy:
return self.marginal_sigma * self.sigma
+def meaningful_regression(score: float | None, before: float | None, policy: RoundPolicy) -> bool:
+ """Did ``score`` fall below ``before`` by MORE than this judge's own noise?
+
+ THE regression predicate: ``StopPolicy._regression`` (the stop gate) and
+ ``StopPolicy._regressions`` (the counter behind "a second regression stops
+ the run") both call it, so they cannot drift apart. The scale is the
+ measured per-judge sigma table via ``policy.regression_delta`` — deliberately
+ NOT the 0.202 A/A replay floor, which measures cross-run planner variance,
+ not judge noise."""
+ if score is None or before is None:
+ return False
+ return (before - score) > policy.regression_delta
+
+
@dataclass(frozen=True)
class StopDecision:
"""``StopPolicy``'s answer: stop (and why), or continue (and in what shape)."""
@@ -235,11 +249,9 @@ class StopPolicy:
is the question that outranks every other stop."""
last = history[-1]
score, best_before = last.score, best_score(history[:-1])
- if score is None or best_before is None:
+ if not meaningful_regression(score, best_before, self.policy):
return None
drop = best_before - score
- if drop <= self.policy.regression_delta:
- return None
detail = (f"r{last.index:02d} scored {score:.3f} vs best {best_before:.3f} "
f"({-drop:+.3f}, judge σ {self.policy.sigma:.3f})")
if not self.policy.regression_allow_switch:
@@ -269,15 +281,14 @@ class StopPolicy:
f"{self.policy.target:.2f}")
return None
- @staticmethod
- def _regressions(history: Sequence[RoundRecord]) -> int:
- """How many scored rounds ended below the best of everything before them."""
- n = 0
- for i in range(1, len(history)):
- s, before = history[i].score, best_score(history[:i])
- if s is not None and before is not None and s < before:
- n += 1
- return n
+ def _regressions(self, history: Sequence[RoundRecord]) -> int:
+ """How many scored rounds ended MEANINGFULLY below the best of everything
+ before them. Same predicate as the gate (:func:`meaningful_regression`),
+ so a sub-noise dip can never help burn the run's single strategy switch
+ (a -0.005 blip + one real regression used to count 2 = "regression" stop
+ with the switch never offered)."""
+ return sum(1 for i in range(1, len(history))
+ if meaningful_regression(history[i].score, best_score(history[:i]), self.policy))
def _plateaued(self, history: Sequence[RoundRecord]) -> bool:
"""True when the last ``plateau_window`` scored rounds did not raise the best
@@ -388,5 +399,5 @@ __all__ = [
"REWRITE_KIND", "RefineTask", "RoundPolicy", "StopDecision", "StopPolicy", "StopReason",
"Strategy", "TaskGroup", "best_score", "build_refine_instructions", "compact_instructions",
"detail_blocked", "gate_error_count", "judge_sigma", "kind_for_strategy", "last_gain",
- "plan_parallel_groups", "plan_refine_groups",
+ "meaningful_regression", "plan_parallel_groups", "plan_refine_groups",
]
diff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.py
index 92e8376f..796cd2ad 100644
--- a/harness/codeverse/tracks/lifecycle.py
+++ b/harness/codeverse/tracks/lifecycle.py
@@ -40,7 +40,7 @@ from codeverse.orchestrator.rounds import (
kind_for_strategy,
plan_refine_groups,
)
-from codeverse.orchestrator.runner import StageRunner
+from codeverse.orchestrator.runner import StageRunner, hash_inputs
from codeverse.orchestrator.state import RunState
from codeverse.prompts import render
from codeverse.tracks.candidates import choose_best_round, run_best_of_n
@@ -61,7 +61,7 @@ from codeverse.tracks.steps import (
RoundFailed,
RoundPipeline,
failed_acceptance,
- load_round_records,
+ load_round_journal,
rejudge_round,
run_round,
sum_usage,
@@ -136,6 +136,23 @@ def plan_stage_inputs(spec: Spec) -> dict[str, Any]:
}
+def spec_fingerprint(spec: Spec) -> str:
+ """Identity of the spec THE PLAN depends on (hash of :func:`plan_stage_inputs`).
+
+ Stamped into ``state.extra["spec_fingerprint"]`` on first run and verified by
+ ``reconcile_resume``. Budget raises — the sanctioned way to continue a
+ BUDGET-stopped run — are outside ``plan_stage_inputs`` and never change it."""
+ return hash_inputs(plan_stage_inputs(spec))
+
+
+class SpecChanged(RuntimeError):
+ """spec.json's plan inputs changed under an existing run.
+
+ Resuming would pair the recorded rounds with a new spec/plan in record.json —
+ fake prompt→code provenance. Fork a new run, or resume with ``--force`` to
+ archive the old rounds under ``rounds/pre_force/`` and re-plan."""
+
+
class BaseTrack:
"""Shared lifecycle. Subclasses set ``track``, ``rubric``, ``plan_model``
and parameterise the ONE planner loop / refine scaffold via the hook
@@ -303,17 +320,21 @@ class BaseTrack:
return run_planner(spec, spec.backends.planner, self.plan_model, ws, model=self._planner_model,
events=events, runtime=runtime, **self._plan_kwargs(spec))
- def run(self, spec: Spec, ws: Workspace, *, resume: bool = False) -> RunRecord:
+ def run(self, spec: Spec, ws: Workspace, *, resume: bool = False, force: bool = False) -> RunRecord:
ws.create()
if not ws.spec_path.is_file() or not resume:
ws.write_json(ws.spec_path, spec)
events = EventLog(ws.events_path)
state = RunState.load_or_new(ws, resume=resume)
+ # BEFORE build_context (which restores the budget snapshot from state.extra):
+ # make the state agree with the durable round journal, verify spec identity,
+ # and take the reconciled rounds as the loop's initial history — so the
+ # FAILED/BUDGET handlers below serialize the real rounds, never [].
+ rounds: list[RoundRecord] = self.reconcile_resume(spec, ws, events, state, resume=resume, force=force)
ctx = self.build_context(spec, ws, events, state)
runner = StageRunner(ws, events, state)
events.emit("run.start", track=self.track.value, language=spec.language.value, resume=resume,
agent=ctx.agent_id, planner=spec.backends.planner, judge=spec.backends.judge)
- rounds: list[RoundRecord] = []
stop: StopReason | str = "failed"
error = ""
try:
@@ -321,7 +342,6 @@ class BaseTrack:
model=self.plan_model)
self.after_plan(ctx)
self.prepare(ctx, runner)
- rounds = load_round_records(ctx) if resume else []
stop = self._round_loop(ctx, rounds)
except BudgetExceeded as e:
events.emit("budget.exceeded", reason=e.reason, spent_usd=round(e.spent_usd, 4))
@@ -340,6 +360,114 @@ class BaseTrack:
rec = self.finalise(ctx, rounds, status, stop_reason=str(stop), error=error)
return rec
+ # ------------------------------------------------------------------ resume reconciliation
+ def reconcile_resume(self, spec: Spec, ws: Workspace, events: EventLog, state: RunState,
+ *, resume: bool, force: bool) -> list[RoundRecord]:
+ """Make ``run_state.json`` agree with the on-disk round journal before anything runs.
+
+ The journal (``rounds/rNN.json`` + its git commit, written at the end of every
+ round) is the durable record; the state file is a cache of it that a crash can
+ leave stale — round persisted, ``mark_round_done``/best never saved. On resume:
+
+ 1. **Spec identity.** ``state.extra["spec_fingerprint"]`` (hash of
+ :func:`plan_stage_inputs`, stamped on first run) must match the spec we are
+ resuming with; budget raises are outside the fingerprint by construction.
+ On a mismatch a plain resume raises :class:`SpecChanged`; ``--force``
+ emits ``resume.spec_changed`` and archives the old journal + ``record.json``
+ to ``rounds/pre_force/`` — old rounds are never paired with a new spec/plan.
+ 2. **Journal integrity.** Trailing rounds whose commit git does not have are
+ dropped (half-written journal), with a ``resume.dropped_rounds`` event.
+ 3. **Rebuild.** ``completed_rounds``/``round_commits`` come from the journal;
+ when the state disagrees with it (rounds it never promoted, a best pointing
+ at the wrong commit) the best is recomputed with ``BestSelector`` — the
+ same ranking ``choose_best_round`` starts from, with no paid pairwise call.
+ A state that already agrees keeps its best untouched (it may embody a paid
+ pairwise keep-the-incumbent decision the pure ranking lacks).
+
+ Returns the loop's initial round history. A fresh (non-resume) run only
+ stamps the fingerprint and starts empty."""
+ fp = spec_fingerprint(spec)
+ stored = str(state.extra.get("spec_fingerprint") or "")
+ if not resume:
+ state.extra["spec_fingerprint"] = fp
+ return []
+ journal = load_round_journal(ws)
+ if stored and stored != fp:
+ if not force:
+ raise SpecChanged(
+ f"spec.json for run {ws.root.name} no longer matches the spec its "
+ f"{len(journal)} recorded round(s) were built from (plan-input fingerprint "
+ f"{stored} != {fp}: prompt/track/language/constraints/references/planner/seed). "
+ f"Resuming would record those rounds against the new spec. Fork a new run "
+ f"(`3dcv make`), or resume with --force to archive the old rounds under "
+ f"rounds/pre_force/ and re-plan. Raising budget caps alone never trips this."
+ )
+ events.emit("resume.spec_changed", old_fingerprint=stored, new_fingerprint=fp,
+ archived_rounds=len(journal))
+ archived = self._archive_pre_force(ws)
+ events.emit("resume.archived", dest=str(archived))
+ journal = []
+ state.stages.pop("plan", None) # the plan must be rebuilt from the edited spec
+ state.completed_rounds, state.round_commits, state.current_round = [], {}, 0
+ state.best_round, state.best_commit, state.best_score = None, "", None
+ state.extra["spec_fingerprint"] = fp
+ kept: list[RoundRecord] = []
+ dropped: list[int] = []
+ for rec in journal:
+ if dropped or not rec.commit or not ws.has_commit(rec.commit):
+ dropped.append(rec.index) # half-written tail: rNN.json without its commit
+ else:
+ kept.append(rec)
+ if dropped:
+ events.emit("resume.dropped_rounds", rounds=dropped,
+ reason="round journal names commits git does not have")
+ journal = kept
+ journal_commits = {r.index: r.commit for r in journal}
+ stale = (state.completed_rounds != [r.index for r in journal]
+ or {int(k): v for k, v in state.round_commits.items()} != journal_commits)
+ if stale:
+ state.completed_rounds = [r.index for r in journal]
+ state.round_commits = dict(journal_commits)
+ state.current_round = len(journal)
+ best = state.best_round
+ best_invalid = best is not None and (best >= len(journal) or journal[best].commit != state.best_commit)
+ best_changed = False
+ if not journal:
+ if best is not None:
+ state.best_round, state.best_commit, state.best_score = None, "", None
+ best_changed = True
+ elif stale or best_invalid or best is None:
+ pick = BestSelector().pick(journal)
+ if pick is None:
+ best_changed = best is not None
+ state.best_round, state.best_commit, state.best_score = None, "", None
+ elif pick != best or journal[pick].commit != state.best_commit:
+ state.update_best(pick, journal[pick].commit, journal[pick].score)
+ best_changed = True
+ state.save(ws)
+ events.emit("resume.reconciled", rounds=len(journal), dropped=dropped, state_was_stale=stale,
+ best_round=state.best_round, best_score=state.best_score, best_changed=best_changed,
+ spec_fingerprint=fp)
+ return journal
+
+
+ @staticmethod
+ def _archive_pre_force(ws: Workspace) -> Path:
+ """Move the old journal + record.json under ``rounds/pre_force/`` (never delete
+ evidence; a second --force lands in ``pre_force_2/`` and so on)."""
+ rounds_dir = ws.root / "rounds"
+ dest, n = rounds_dir / "pre_force", 1
+ while dest.exists():
+ n += 1
+ dest = rounds_dir / f"pre_force_{n}"
+ dest.mkdir(parents=True, exist_ok=True)
+ for p in sorted(rounds_dir.iterdir()):
+ if p.is_file():
+ p.rename(dest / p.name)
+ if ws.record_path.is_file():
+ ws.record_path.rename(dest / ws.record_path.name)
+ return dest
+
# ------------------------------------------------------------------ context
def build_context(self, spec: Spec, ws: Workspace, events: EventLog, state: RunState) -> RunContext:
settings = self._settings or get_settings()
@@ -530,7 +658,10 @@ class BaseTrack:
ctx.events.emit("stop", reason="no_change", rounds=len(rounds), best=ctx.state.best_round)
return "plateau"
rounds.append(rec)
- ctx.state.mark_round_done(index, rec.commit)
+ # durable BEFORE choose_best_round's possible PAID pairwise call: a crash in
+ # that window left state.json without a round the journal already had, and
+ # an immediate stop on resume then restored/delivered a stale best.
+ ctx.state.mark_round_done(index, rec.commit, ctx.ws)
best = choose_best_round(ctx, rounds, selector, index)
if best is not None and ctx.state.update_best(best, rounds[best].commit, rounds[best].score):
ctx.events.emit("best.updated", round=best, score=rounds[best].score)
@@ -566,7 +697,7 @@ class BaseTrack:
ctx.events.emit("budget.salvage_failed", error=f"{type(e).__name__}: {e}")
return
rounds.append(rec)
- ctx.state.mark_round_done(0, rec.commit)
+ ctx.state.mark_round_done(0, rec.commit, ctx.ws)
if ctx.state.update_best(0, rec.commit, rec.score):
ctx.events.emit("best.updated", round=0, score=rec.score)
self._save_budget(ctx)
@@ -703,16 +834,44 @@ class BaseTrack:
# rounds the budget (or a crash) cut after the money was spent: they are not
# in ``rounds``, so name them here and keep their dollars in total_usage.
extra["aborted_rounds"] = ctx.extra["aborted_rounds"]
+ # A resume rewrite must not drop what earlier sessions / other packages put on
+ # the record post-hoc (flywheel captions, texturing/run.record_texturing, prior
+ # sessions' aborted_rounds, prompt hashes of stages not re-executed): seed from
+ # the prior record — unknown keys preserved, this session's values win.
+ prior_extra, prior_hashes = self._prior_record_fields(ctx.ws)
+ prior_aborted = prior_extra.get("aborted_rounds")
+ for k, v in prior_extra.items():
+ if k not in extra or extra[k] is None:
+ extra[k] = v
+ if isinstance(prior_aborted, list) and ctx.extra.get("aborted_rounds"):
+ extra["aborted_rounds"] = prior_aborted + [a for a in ctx.extra["aborted_rounds"]
+ if a not in prior_aborted]
return RunRecord(
spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, best_round=best,
baseline_score=baseline, final_score=final, total_usage=total,
environment={"python": platform.python_version(), "host": platform.node(), "track": self.track.value,
"language": ctx.language.value, "generator": ctx.agent_id},
- prompt_hashes=dict(ctx.prompt_hashes), started_at=ctx.state.started_at,
+ prompt_hashes={**prior_hashes, **dict(ctx.prompt_hashes)}, started_at=ctx.state.started_at,
finished_at=datetime.now(UTC), error=error,
extra=extra,
)
+ @staticmethod
+ def _prior_record_fields(ws: Workspace) -> tuple[dict[str, Any], dict[str, str]]:
+ """``(extra, prompt_hashes)`` of the record.json a previous session wrote
+ (empty when there is none / it cannot be read — never run-fatal)."""
+ if not ws.record_path.is_file():
+ return {}, {}
+ try:
+ prior = ws.read_json(ws.record_path)
+ except Exception as e: # noqa: BLE001 — a corrupt old record must not block finalise
+ log.warning("prior record.json unreadable; post-hoc extra keys not carried over: %s", e)
+ return {}, {}
+ extra = prior.get("extra")
+ hashes = prior.get("prompt_hashes")
+ return (dict(extra) if isinstance(extra, dict) else {},
+ {str(k): str(v) for k, v in hashes.items()} if isinstance(hashes, dict) else {})
+
def rewrite_task(last: RoundRecord, best: float | None) -> RefineTask:
"""The instruction that turns a repeat into a change of approach."""
diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.py
index c2a8bb46..027cfe05 100644
--- a/harness/codeverse/tracks/steps.py
+++ b/harness/codeverse/tracks/steps.py
@@ -67,12 +67,20 @@ def round_record_path(ctx: RunContext, index: int) -> Path:
def load_round_records(ctx: RunContext) -> list[RoundRecord]:
"""Rounds persisted so far (sorted by index, contiguous prefix only)."""
- d = ctx.ws.root / "rounds"
+ return load_round_journal(ctx.ws)
+
+
+def load_round_journal(ws: Workspace) -> list[RoundRecord]:
+ """The on-disk round journal: ``rounds/r*.json`` sorted, contiguous prefix only.
+
+ Workspace-level (no ``RunContext``) because ``lifecycle.reconcile_resume``
+ reads it before the context exists; :func:`load_round_records` delegates here."""
+ d = ws.root / "rounds"
if not d.is_dir():
return []
out: list[RoundRecord] = []
for i, p in enumerate(sorted(d.glob("r*.json"))):
- rec = RoundRecord.model_validate(ctx.ws.read_json(p))
+ rec = RoundRecord.model_validate(ws.read_json(p))
if rec.index != i:
break
out.append(rec)
@@ -254,10 +262,14 @@ def _run_round(
notes.append(f"judge skipped ({skip})")
ctx.events.emit("judge.skipped", round=index, reason=skip)
else:
- rec.judgment = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous)
+ rec.judgment, judge_usage = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous, notes)
+ if rec.judgment is not None or judge_usage.cost_usd:
+ # a degraded/crashed verdict was still PAID: fold its usage into the
+ # round's usage and cost["judge"] so rec.usage and the cost.round event
+ # see the money even when judged=False.
+ usage = usage + judge_usage
+ cost["judge"] = round(judge_usage.cost_usd, 6)
if rec.judgment is not None:
- usage = usage + rec.judgment.usage
- cost["judge"] = round(rec.judgment.usage.cost_usd, 6)
# add, never charge: the verdict exists and is paid for — raising here
# would drop a fully judged round before it is committed/recorded
# (the loop stops at its next budget_ok check instead, AFTER best promotion).
@@ -328,10 +340,13 @@ def emit_round_cost(
build_ok: bool,
judged: bool,
aborted: str = "",
+ corrected: bool = False,
) -> None:
"""One ``cost.round`` event per round: ``{stage → $}``, judge $, agent turns and
whether the money bought anything. The ledger/audit reads this instead of
- reconstructing the round from trajectories and events."""
+ reconstructing the round from trajectories and events. ``corrected=True``
+ marks a re-emission after ``rejudge_round`` recovered a verdict — the LAST
+ event per round is the truth."""
wasted, why = _waste_flag(score=score, previous_best=previous_best, build_ok=build_ok,
judged=judged, aborted=aborted)
ctx.events.emit(
@@ -342,6 +357,7 @@ def emit_round_cost(
agent_turns=turns,
input_tokens=usage.input_tokens, output_tokens=usage.output_tokens, cached_tokens=usage.cached_tokens,
score=score, previous_best=previous_best, wasted=wasted, waste_reason=why, aborted=aborted,
+ corrected=corrected,
run_usd=round(ctx.budget.spent.cost_usd, 6),
)
@@ -388,7 +404,15 @@ def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage
def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildResult, gates: list[GateReport],
- rec: RoundRecord, previous: Judgment | None) -> Judgment | None:
+ rec: RoundRecord, previous: Judgment | None, notes: list[str]) -> tuple[Judgment | None, Usage]:
+ """Judge one round → ``(judgment, paid_usage)``.
+
+ ``judgment`` is ``None`` when the verdict is unusable (the judge crashed or
+ returned a degraded non-score); ``paid_usage`` is what the attempt cost
+ EITHER WAY, so a failed verdict's dollars still reach the round record and
+ its ``cost.round`` event. Failure notes are appended to the CALLER's
+ ``notes`` list — ``_run_round`` joins that list into ``rec.notes`` at the
+ end of the round, so writing to ``rec.notes`` here was silently lost."""
from codeverse.judges.base import JudgeInput
renders = rec.renders
@@ -418,9 +442,16 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildRes
judgment: Judgment = ctx.judge.judge(inp)
except Exception as e: # noqa: BLE001 — a judge outage must not destroy the run's code/commits
log.exception("judge failed in round %d", index)
- ctx.events.emit("judge.failed", round=index, error=f"{type(e).__name__}: {e}")
- rec.notes = (rec.notes + "; " if rec.notes else "") + f"judge failed: {type(e).__name__}: {e}"
- return None
+ paid = getattr(e, "usage", None)
+ paid = paid if isinstance(paid, Usage) else Usage()
+ if paid.cost_usd:
+ # the provider billed the failed attempt (ModelError.usage): book it
+ ctx.budget.add(paid, stage="judge", role="judge", round_index=index,
+ label="judge", outcome="failed")
+ ctx.events.emit("judge.failed", round=index, error=f"{type(e).__name__}: {e}",
+ cost_usd=round(paid.cost_usd, 4))
+ notes.append(f"judge failed: {type(e).__name__}: {e}")
+ return None, paid
ctx.ws.write_json(ctx.ws.judge_path(index), judgment)
if _is_degraded(judgment):
# a glitch, never a score: keep the raw verdict on disk, pay for it, but do not
@@ -429,12 +460,12 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildRes
label=judgment.rubric or "judge", outcome="degraded")
ctx.events.emit("judge.degraded", round=index, error=judgment.summary[:300],
cost_usd=round(judgment.usage.cost_usd, 4))
- rec.notes = (rec.notes + "; " if rec.notes else "") + f"judge degraded: {judgment.summary[:200]}"
- return None
+ notes.append(f"judge degraded: {judgment.summary[:200]}")
+ return None, judgment.usage
ctx.events.emit("judge.done", round=index, overall=round(judgment.overall, 3), passed=judgment.passed,
n_issues=len(judgment.issues), n_plan=len(judgment.improvement_plan),
duration_s=round(time.time() - t0, 1), cost_usd=round(judgment.usage.cost_usd, 4))
- return judgment
+ return judgment, judgment.usage
def _is_degraded(judgment: Judgment) -> bool:
@@ -461,14 +492,26 @@ def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, pr
if judge_blocked_by_gates(ctx, rec.gates):
return False
ctx.events.emit("judge.retry", round=rec.index)
- judgment = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous)
+ notes: list[str] = [rec.notes] if rec.notes else []
+ judgment, judge_usage = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)
if judgment is None:
+ # the retry failed AGAIN — but it was still paid for: keep the money and the
+ # failure note on the persisted record so totals and resume can see them.
+ rec.usage = rec.usage + judge_usage
+ rec.notes = "; ".join(notes)
+ ctx.ws.write_json(round_record_path(ctx, rec.index), rec)
return False
rec.judgment = judgment
rec.usage = rec.usage + judgment.usage
+ rec.notes = "; ".join(notes)
ctx.budget.add(judgment.usage, stage="judge", role="judge", round_index=rec.index,
label=judgment.rubric or "judge", outcome="rejudge")
ctx.ws.write_json(round_record_path(ctx, rec.index), rec)
+ # the round's original cost.round went out as wasted=unjudged; emit the corrected
+ # one so the audit stream stops counting a now-scored round as wasted.
+ emit_round_cost(ctx, index=rec.index, kind=rec.kind, cost={"judge": round(judgment.usage.cost_usd, 6)},
+ usage=rec.usage, turns=0, score=rec.score, previous_best=None,
+ build_ok=True, judged=True, corrected=True)
return True
diff --git a/harness/codeverse/workspace.py b/harness/codeverse/workspace.py
index 27bbf366..8cad8bed 100644
--- a/harness/codeverse/workspace.py
+++ b/harness/codeverse/workspace.py
@@ -332,6 +332,13 @@ class Workspace:
def head(self) -> str:
return self._git("rev-parse", "HEAD").stdout.strip()
+ def has_commit(self, commit: str) -> bool:
+ """Does this repo actually have ``commit``? A round journal can name a commit
+ the crash never wrote (``tracks/lifecycle.reconcile_resume`` drops such rounds)."""
+ if not commit:
+ return False
+ return self._git("cat-file", "-e", f"{commit}^{{commit}}", check=False).returncode == 0
+
def changed_files(self, since: str | None = None) -> list[FileChange]:
"""Files changed vs ``since`` (a commit) or vs HEAD (uncommitted work)."""
self._git("add", "-A", "-N") # register untracked so they show up in diff
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index aa4f5037..4fc7cfdb 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -243,7 +243,7 @@ def test_resume_budget_flags_rewrite_spec_and_emit_event(tmp_path: Path, monkeyp
def fake_get_track(track, **options):
class _T:
- def run(self, spec, ws_, *, resume=False):
+ def run(self, spec, ws_, *, resume=False, force=False):
seen["spec"] = spec
raise KeyboardInterrupt # stop before any real work
@@ -282,7 +282,7 @@ def test_resume_refuses_a_finished_run_and_never_re_enters_it(tmp_path: Path, mo
def fake_get_track(track, **options):
class _T:
- def run(self, spec, ws_, *, resume=False):
+ def run(self, spec, ws_, *, resume=False, force=False):
entered.append(resume)
raise KeyboardInterrupt
@@ -309,6 +309,57 @@ def test_resume_refuses_a_finished_run_and_never_re_enters_it(tmp_path: Path, mo
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)]).exit_code == 130
+def test_resume_threads_force_through_to_the_track(tmp_path: Path, monkeypatch):
+ """reconcile_resume (tracks/lifecycle.py) needs the CLI's --force to decide between
+ refusing a spec edit (SpecChanged) and archiving the old rounds under
+ rounds/pre_force/ — so `3dcv resume --force` must reach BaseTrack.run(force=...)."""
+ runs = tmp_path / "runs"
+ assert runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run"]).exit_code == 0
+ ws = next(d for d in runs.iterdir() if d.is_dir())
+ seen = {}
+
+ def fake_get_track(track, **options):
+ class _T:
+ def run(self, spec, ws_, *, resume=False, force=False):
+ seen["resume"], seen["force"] = resume, force
+ raise KeyboardInterrupt
+
+ return _T()
+
+ import codeverse.tracks as tracks_pkg
+
+ monkeypatch.setattr(tracks_pkg, "get_track", fake_get_track)
+ assert runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs)]).exit_code == 130
+ assert seen == {"resume": True, "force": False}
+ assert runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs), "--force"]).exit_code == 130
+ assert seen == {"resume": True, "force": True}
+
+
+def test_a_spec_change_refusal_is_a_clean_cli_error(tmp_path: Path, monkeypatch):
+ """SpecChanged is a refusal with instructions (fork, or --force to archive), not a
+ crash: no traceback, and the CLI's typed configuration-error exit code."""
+ from codeverse.tracks.lifecycle import SpecChanged
+
+ runs = tmp_path / "runs"
+ assert runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run"]).exit_code == 0
+ ws = next(d for d in runs.iterdir() if d.is_dir())
+
+ def fake_get_track(track, **options):
+ class _T:
+ def run(self, spec, ws_, *, resume=False, force=False):
+ raise SpecChanged("spec.json changed under this run; fork a new run or resume with --force")
+
+ return _T()
+
+ import codeverse.tracks as tracks_pkg
+
+ monkeypatch.setattr(tracks_pkg, "get_track", fake_get_track)
+ r = runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs)])
+ out = r.output + str(getattr(r, "stderr", "") or "")
+ assert r.exit_code == 2 and "--force" in out
+ assert "Traceback" not in r.output
+
+
def test_status_shows_candidates_and_texturing(runs_dir: Path):
import json as _json
diff --git a/harness/tests/models/test_gemini.py b/harness/tests/models/test_gemini.py
index 1b828cd0..a616265e 100644
--- a/harness/tests/models/test_gemini.py
+++ b/harness/tests/models/test_gemini.py
@@ -529,3 +529,113 @@ def test_a_charged_but_invalid_reply_carries_its_usage_on_the_error():
m.generate(ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"}))
assert ei.value.usage.input_tokens == 100 and ei.value.usage.output_tokens == 20
assert ei.value.usage.cost_usd > 0
+
+
+def _resp_with_calls(parts: list[types.Part]) -> types.GenerateContentResponse:
+ return types.GenerateContentResponse(
+ candidates=[
+ types.Candidate(content=types.Content(role="model", parts=parts), finish_reason="STOP")
+ ],
+ usage_metadata=types.GenerateContentResponseUsageMetadata(
+ prompt_token_count=10, candidates_token_count=5
+ ),
+ )
+
+
+def test_provider_call_id_round_trips_verbatim():
+ """A GENUINE provider function_call.id must be echoed unchanged on BOTH the
+ function_call replay and the function_response (both used to drop it)."""
+ from codeverse.models.gemini_convert import is_synthetic_call_id
+
+ tool = ToolSpec(
+ name="measure",
+ description="m",
+ parameters={"type": "object", "properties": {"part": {"type": "string"}}},
+ )
+ part = types.Part(
+ function_call=types.FunctionCall(name="measure", args={"part": "all"}, id="prov-42")
+ )
+ part.thought_signature = b"sig-prov"
+ m, log, _ = make_model([_resp_with_calls([part]), text_response("45 cm")])
+ req = ChatRequest(messages=[ChatMessage.user("measure it")], tools=[tool])
+ r = m.generate(req)
+ call = r.tool_calls[0]
+ assert call.id == "prov-42" and not is_synthetic_call_id(call.id)
+ assert SIGNATURES.get("prov-42") == b"sig-prov"
+ msgs = [
+ *req.messages,
+ ChatMessage(role="assistant", parts=[call]),
+ ChatMessage(
+ role="tool",
+ parts=[
+ ToolResultPart(call_id=call.id, name="measure", content=json.dumps({"h": 0.45}))
+ ],
+ ),
+ ]
+ m.generate(ChatRequest(messages=msgs, tools=[tool]))
+ contents = log[1]["contents"]
+ assert contents[1].parts[0].function_call.id == "prov-42"
+ assert contents[2].parts[0].function_response.id == "prov-42"
+
+
+def test_synthetic_call_ids_are_never_echoed_to_the_provider():
+ """When the provider sent NO id we mint one for local pairing — and it must
+ never travel back to Gemini (an id the provider did not issue)."""
+ from codeverse.models.gemini_convert import is_synthetic_call_id
+
+ tool = ToolSpec(name="measure", description="m", parameters={"type": "object", "properties": {}})
+ m, log, _ = make_model([call_response("measure", {"part": "all"}), text_response("ok")])
+ req = ChatRequest(messages=[ChatMessage.user("x")], tools=[tool])
+ r = m.generate(req)
+ call = r.tool_calls[0]
+ assert is_synthetic_call_id(call.id)
+ msgs = [
+ *req.messages,
+ ChatMessage(role="assistant", parts=[call]),
+ ChatMessage(role="tool", parts=[ToolResultPart(call_id=call.id, name="measure", content="{}")]),
+ ]
+ m.generate(ChatRequest(messages=msgs, tools=[tool]))
+ contents = log[1]["contents"]
+ assert contents[1].parts[0].function_call.id is None, "an invented id must not reach Gemini"
+ assert contents[2].parts[0].function_response.id is None
+
+
+def test_two_parallel_same_name_calls_keep_their_provider_ids():
+ """Two calls to the SAME tool in one turn: only the echoed ids pair each
+ response with its call, and each call keeps its own thought_signature."""
+ parts = []
+ for cid, target in (("id-a", "seat"), ("id-b", "leg")):
+ p = types.Part(function_call=types.FunctionCall(name="measure", args={"part": target}, id=cid))
+ p.thought_signature = f"sig-{cid}".encode()
+ parts.append(p)
+ tool = ToolSpec(
+ name="measure",
+ description="m",
+ parameters={"type": "object", "properties": {"part": {"type": "string"}}},
+ )
+ m, log, _ = make_model([_resp_with_calls(parts), text_response("done")])
+ req = ChatRequest(messages=[ChatMessage.user("measure both")], tools=[tool])
+ r = m.generate(req)
+ a, b = r.tool_calls
+ assert (a.id, b.id) == ("id-a", "id-b") and a.name == b.name == "measure"
+ assert SIGNATURES.get("id-a") == b"sig-id-a" and SIGNATURES.get("id-b") == b"sig-id-b"
+ msgs = [
+ *req.messages,
+ ChatMessage(role="assistant", parts=[a, b]),
+ ChatMessage(
+ role="tool",
+ parts=[ # results deliberately out of order: the ids do the pairing
+ ToolResultPart(call_id=b.id, name="measure", content=json.dumps({"part": "leg"})),
+ ToolResultPart(call_id=a.id, name="measure", content=json.dumps({"part": "seat"})),
+ ],
+ ),
+ ]
+ m.generate(ChatRequest(messages=msgs, tools=[tool]))
+ model_parts = log[1]["contents"][1].parts
+ assert [fp.function_call.id for fp in model_parts] == ["id-a", "id-b"]
+ assert [fp.thought_signature for fp in model_parts] == [b"sig-id-a", b"sig-id-b"]
+ responses = log[1]["contents"][2].parts
+ assert [(fp.function_response.id, fp.function_response.response["part"]) for fp in responses] == [
+ ("id-b", "leg"),
+ ("id-a", "seat"),
+ ]
diff --git a/harness/tests/orchestrator_tracks/test_fix_batch2.py b/harness/tests/orchestrator_tracks/test_fix_batch2.py
index 2f5ceff6..9a03e0ce 100644
--- a/harness/tests/orchestrator_tracks/test_fix_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_fix_batch2.py
@@ -152,6 +152,17 @@ def test_degraded_verdict_never_scores_and_run_stops_as_judge_unavailable(tmp_pa
assert "judge.degraded" in kinds and "judge.retry" in kinds
# the degraded summary never reaches a refine prompt
assert len(judge.all_calls) == 3
+ # audit f1/f3: both degraded verdicts were PAID and the glitch note survives the
+ # round's notes join — money and note reach the persisted record and cost.round
+ r1 = rec.rounds[1]
+ assert "judge degraded" in r1.notes
+ assert r1.usage.cost_usd == pytest.approx(0.016, abs=1e-6) # agent 0.01 + 2 x degraded 0.003
+ saved = json.loads((ws.root / "rounds" / "r01.json").read_text())
+ assert "judge degraded" in saved["notes"]
+ assert saved["usage"]["cost_usd"] == pytest.approx(0.016, abs=1e-6)
+ r1_cost = next(e for e in EventLog(ws.events_path).read()
+ if e["event"] == "cost.round" and e["round"] == 1)
+ assert r1_cost["judge_usd"] == pytest.approx(0.003) and r1_cost["waste_reason"] == "unjudged"
def test_degraded_verdict_recovers_via_rejudge_of_same_commit(tmp_path, chair_plan, settings):
@@ -169,6 +180,17 @@ def test_degraded_verdict_recovers_via_rejudge_of_same_commit(tmp_path, chair_pl
assert [j.extra.get("round") for j in agent.jobs] == [0, 1]
saved = RoundRecord.model_validate(json.loads((ws.root / "rounds" / "r01.json").read_text()))
assert saved.judgment is not None and saved.judgment.overall == pytest.approx(0.85)
+ # audit f2: the recovery re-emits a CORRECTED cost.round so the audit stream stops
+ # counting r01 as wasted=unjudged; f1: every paid verdict is on the round's usage
+ r1_costs = [e for e in EventLog(ws.events_path).read()
+ if e["event"] == "cost.round" and e["round"] == 1]
+ assert len(r1_costs) == 2
+ first, corrected = r1_costs
+ assert first["wasted"] is True and first["waste_reason"] == "unjudged" and first["corrected"] is False
+ assert corrected["corrected"] is True and corrected["wasted"] is False
+ assert corrected["score"] == pytest.approx(0.85) and corrected["judge_usd"] == pytest.approx(0.003)
+ assert rec.rounds[1].usage.cost_usd == pytest.approx(0.016, abs=1e-6) # agent + degraded + rejudge
+ assert "judge degraded" in rec.rounds[1].notes and "judge degraded" in saved.notes
# --------------------------------------------------------------------- finding: plan stage hash covers the whole Spec
diff --git a/harness/tests/orchestrator_tracks/test_resume_reconcile.py b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
new file mode 100644
index 00000000..fda68222
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
@@ -0,0 +1,192 @@
+"""reconcile_resume: on resume, the durable round journal is the truth.
+
+Covers the audited failure modes:
+
+* the crash window between the round-journal write and the state save — a stale
+ best used to be restored and DELIVERED although a newer PAID round was on disk;
+* a planner outage on resume serializing record.json with ``rounds=[]`` (the
+ flywheel-visible history wipe);
+* ``resume --force`` after a spec edit pairing old rounds with the new spec/plan
+ (fake prompt→code provenance) — the old rounds are archived under
+ ``rounds/pre_force/`` instead;
+* a spec edit WITHOUT ``--force`` → typed :class:`SpecChanged` refusal, before
+ any model call and without damaging the run;
+* judge-failure notes lost to ``_run_round``'s local notes list.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from codeverse.contracts.common import Language
+from codeverse.events import EventLog
+from codeverse.orchestrator.rounds import RoundPolicy
+from codeverse.orchestrator.state import RunState
+from codeverse.tracks.lifecycle import SpecChanged
+from codeverse.tracks.static_object import StaticObjectTrack
+from codeverse.workspace import Workspace
+from tests.orchestrator_tracks.conftest import make_spec
+from tests.orchestrator_tracks.fakes import (
+ FakeAgent,
+ FakeChatModel,
+ FakeJudge,
+ FakeRuntime,
+ FakeServices,
+)
+
+
+def _planner(plan_dict):
+ return FakeChatModel(lambda req: plan_dict)
+
+
+def _writer(job, ws):
+ return {"src/model.py": f"import bpy # {job.label} r{job.extra.get('round', 0)}\n"}
+
+
+def _track(planner, settings, *, judge=None, policy=None):
+ return StaticObjectTrack(services=FakeServices(), judge=judge or FakeJudge(scores=(0.5, 0.7)),
+ agent=FakeAgent(_writer), planner_model=planner, settings=settings,
+ runtime=FakeRuntime(Language.BLENDER), policy=policy)
+
+
+# --------------------------------------------------------------------- (a) crash window
+def test_crash_between_round_write_and_state_save_delivers_the_newer_round(tmp_path, chair_plan, settings):
+ """rNN.json + its commit are durable; mark_round_done/update_best used to live only
+ in memory until _save_budget — with a possible PAID pairwise call in between. A
+ crash in that window plus an immediate stop on resume restored and delivered a
+ STALE best although the newer paid round was on disk."""
+ spec = make_spec(language=Language.BLENDER, max_rounds=1)
+ ws = Workspace(tmp_path / "runs" / "r")
+ planner = _planner(chair_plan.model_dump(mode="json"))
+ pol = RoundPolicy(max_rounds=1, target=0.9)
+ rec1 = _track(planner, settings, policy=pol).run(spec, ws)
+ assert rec1.best_round == 1 and len(rec1.rounds) == 2
+ # rewind state.json to what a crash between the r01 write and the state save leaves
+ state = json.loads(ws.state_path.read_text())
+ state["completed_rounds"], state["current_round"] = [0], 1
+ state["round_commits"] = {"0": rec1.rounds[0].commit}
+ state["best_round"], state["best_commit"], state["best_score"] = 0, rec1.rounds[0].commit, 0.5
+ ws.state_path.write_text(json.dumps(state))
+ ws.restore(rec1.rounds[0].commit) # ...and the tree at the stale best, as delivered
+ ws.commit("stale delivery")
+
+ rec2 = _track(planner, settings, policy=pol).run(spec, ws, resume=True)
+ # the immediate stop (max_rounds) now delivers the NEWER paid round
+ assert rec2.best_round == 1 and rec2.final_score == pytest.approx(0.7)
+ assert [r.index for r in rec2.rounds] == [0, 1]
+ assert "refine" in (ws.src / "model.py").read_text(), "round 1's code must be restored"
+ st = RunState.load(ws)
+ assert st.completed_rounds == [0, 1] and st.best_round == 1
+ assert st.best_commit == rec1.rounds[1].commit
+ ev = [e for e in EventLog(ws.events_path).read() if e["event"] == "resume.reconciled"]
+ assert ev and ev[-1]["state_was_stale"] is True and ev[-1]["best_changed"] is True
+
+
+# --------------------------------------------------------------------- (b) history wipe
+def test_a_planner_outage_on_resume_keeps_the_recorded_history(tmp_path, chair_plan, settings):
+ """The FAILED handler used to serialize record.json with rounds=[] because the
+ journal was only loaded AFTER plan/prepare — one planner outage on resume wiped
+ record.rounds (and with it the flywheel's rounds_summary)."""
+ spec = make_spec(language=Language.BLENDER, max_rounds=0)
+ ws = Workspace(tmp_path / "runs" / "r")
+ rec1 = _track(_planner(chair_plan.model_dump(mode="json")), settings,
+ policy=RoundPolicy(max_rounds=0, target=0.9)).run(spec, ws)
+ assert len(rec1.rounds) == 1
+ (ws.root / "stages" / "plan.json").unlink() # clobbered cache → the resume re-plans
+
+ def _boom(req):
+ raise RuntimeError("planner 503 storm")
+
+ with pytest.raises(RuntimeError, match="503 storm"):
+ _track(FakeChatModel(_boom), settings,
+ policy=RoundPolicy(max_rounds=1, target=0.9)).run(spec, ws, resume=True)
+ saved = json.loads(ws.record_path.read_text())
+ assert saved["status"] == "failed" and "503 storm" in saved["error"]
+ assert [r["index"] for r in saved["rounds"]] == [0], "a planner outage must not wipe the history"
+ assert saved["rounds"][0]["commit"] == rec1.rounds[0].commit
+ assert RunState.load(ws).completed_rounds == [0]
+
+
+# --------------------------------------------------------------------- (d) force provenance
+def test_force_resume_with_an_edited_prompt_archives_the_old_rounds(tmp_path, chair_plan, settings):
+ """`resume --force` after a prompt edit used to re-plan and then pair the OLD
+ rounds with the NEW spec/plan in record.json — fake prompt→code provenance.
+ The old journal + record are archived under rounds/pre_force/ instead."""
+ spec = make_spec(language=Language.BLENDER, max_rounds=0)
+ ws = Workspace(tmp_path / "runs" / "r")
+ pol = RoundPolicy(max_rounds=0, target=0.9)
+ rec1 = _track(_planner(chair_plan.model_dump(mode="json")), settings, policy=pol).run(spec, ws)
+ old_commit = rec1.rounds[0].commit
+
+ spec2 = make_spec(language=Language.BLENDER, max_rounds=0,
+ prompt="a completely different bar stool")
+ ws.write_json(ws.spec_path, spec2) # the user edited the prompt
+ p2 = _planner(chair_plan.model_dump(mode="json"))
+ rec2 = _track(p2, settings, policy=pol).run(spec2, ws, resume=True, force=True)
+
+ # the old journal + record are ARCHIVED, never paired with the new spec
+ assert (ws.root / "rounds" / "pre_force" / "r00.json").is_file()
+ assert (ws.root / "rounds" / "pre_force" / "record.json").is_file()
+ archived = json.loads((ws.root / "rounds" / "pre_force" / "r00.json").read_text())
+ assert archived["commit"] == old_commit
+ # a fresh plan and a fresh round 0 under the new spec
+ assert len(p2.requests) >= 1, "--force after a spec edit must re-plan"
+ assert rec2.spec.prompt == "a completely different bar stool"
+ assert [r.index for r in rec2.rounds] == [0] and rec2.rounds[0].commit != old_commit
+ on_disk = json.loads(ws.record_path.read_text())
+ assert on_disk["spec"]["prompt"] == "a completely different bar stool"
+ assert [r["commit"] for r in on_disk["rounds"]] == [rec2.rounds[0].commit]
+ kinds = [e["event"] for e in EventLog(ws.events_path).read()]
+ assert "resume.spec_changed" in kinds and "resume.reconciled" in kinds
+
+
+def test_a_spec_edit_without_force_is_refused_before_anything_runs(tmp_path, chair_plan, settings):
+ spec = make_spec(language=Language.BLENDER, max_rounds=0)
+ ws = Workspace(tmp_path / "runs" / "r")
+ pol = RoundPolicy(max_rounds=0, target=0.9)
+ _track(_planner(chair_plan.model_dump(mode="json")), settings, policy=pol).run(spec, ws)
+
+ spec2 = make_spec(language=Language.BLENDER, max_rounds=0,
+ prompt="a completely different bar stool")
+ ws.write_json(ws.spec_path, spec2)
+ p2 = _planner(chair_plan.model_dump(mode="json"))
+ with pytest.raises(SpecChanged, match="--force"):
+ _track(p2, settings, policy=pol).run(spec2, ws, resume=True)
+ # refused cleanly: nothing archived, nothing spent, nothing overwritten
+ assert p2.requests == [], "no model call before the refusal"
+ assert not (ws.root / "rounds" / "pre_force").exists()
+ assert (ws.root / "rounds" / "r00.json").is_file()
+ st = RunState.load(ws)
+ assert st.best_round == 0 and st.completed_rounds == [0]
+ assert json.loads(ws.record_path.read_text())["status"] != "failed"
+
+ # the sanctioned budget raise (outside the fingerprint) still resumes plainly
+ spec3 = make_spec(language=Language.BLENDER, max_rounds=0, max_usd=50.0)
+ ws.write_json(ws.spec_path, spec3)
+ rec3 = _track(_planner(chair_plan.model_dump(mode="json")), settings, policy=pol).run(
+ spec3, ws, resume=True)
+ assert [r.index for r in rec3.rounds] == [0]
+
+
+# --------------------------------------------------------------------- (f3) notes survive
+def test_notes_survive_a_judge_crash(tmp_path, chair_plan, settings):
+ """_judge used to append 'judge failed: ...' to rec.notes, which _run_round then
+ overwrote with its own local notes list — the note never reached rNN.json."""
+ class BoomJudge(FakeJudge):
+ def judge(self, inp):
+ if inp.round_index >= 1:
+ raise RuntimeError("vlm 500")
+ return super().judge(inp)
+
+ spec = make_spec(language=Language.BLENDER, max_rounds=2)
+ ws = Workspace(tmp_path / "runs" / "r")
+ rec = _track(_planner(chair_plan.model_dump(mode="json")), settings,
+ judge=BoomJudge(scores=(0.5,))).run(spec, ws)
+ assert rec.extra["stop_reason"] == "judge_unavailable"
+ r1 = rec.rounds[1]
+ assert r1.judgment is None
+ assert "judge failed: RuntimeError: vlm 500" in r1.notes
+ saved = json.loads((ws.root / "rounds" / "r01.json").read_text())
+ assert "judge failed: RuntimeError: vlm 500" in saved["notes"]
diff --git a/harness/tests/orchestrator_tracks/test_round_economics.py b/harness/tests/orchestrator_tracks/test_round_economics.py
index a29e8005..7b735712 100644
--- a/harness/tests/orchestrator_tracks/test_round_economics.py
+++ b/harness/tests/orchestrator_tracks/test_round_economics.py
@@ -94,6 +94,23 @@ def test_a_regression_buys_a_change_of_shape_then_stops():
assert twice.reason == "regression"
+def test_a_sub_noise_dip_never_burns_the_strategy_switch():
+ """Audit (g): the regression COUNTER used to count ANY drop while the gate used
+ policy.regression_delta — a -0.02 blip (inside pro sigma 0.030) plus one real
+ regression made the counter 2 and stopped the run as "regression" without ever
+ offering the single strategy switch. One predicate now serves both."""
+ from codeverse.orchestrator.rounds import meaningful_regression
+
+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, judge_model="gemini:gemini-3.1-pro-preview",
+ marginal_from_round=99))
+ d = sp.evaluate([_round(0, 0.60), _round(1, 0.58), _round(2, 0.50)])
+ assert (d.reason, d.strategy) == ("continue", "switch"), "the single switch must still be offered"
+ pol = RoundPolicy(judge_model="gemini:gemini-3.1-pro-preview")
+ assert not meaningful_regression(0.58, 0.60, pol) # sub-noise dip: not a regression
+ assert meaningful_regression(0.50, 0.60, pol) # a real one
+ assert not meaningful_regression(None, 0.60, pol) and not meaningful_regression(0.5, None, pol)
+
+
def test_regression_can_be_configured_to_stop_outright():
sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99, regression_allow_switch=False))
d = sp.evaluate([_round(0, 0.60), _round(1, 0.52)])