Skip to content

Official Codebase of 3DCodeVerse

Python

candidates: a paid pairwise rejection is final — live and replay share one sequential rule

gaoypeng205fb066bf75
diff --git a/harness/codeverse/tracks/candidates.py b/harness/codeverse/tracks/candidates.py
index 8bfbc5f7..07d8cef1 100644
--- a/harness/codeverse/tracks/candidates.py
+++ b/harness/codeverse/tracks/candidates.py
@@ -245,47 +245,49 @@ def choose_best_round(ctx: RunContext, rounds: list[RoundRecord], selector: Best
comparison of the two render sets decides, and the new round replaces the
best only when it wins with ``confidence ≥ policy.pairwise_min_confidence``.
The verdict is stored on the round (``RoundRecord.pairwise``, plus a notes line)
- and re-persisted, so :func:`replay_best_round` can reproduce it on resume."""
+ and re-persisted; the decision itself is :func:`replay_best_round` over the journal,
+ so the live pick and any later resume replay the SAME sequential rule."""
incumbent = ctx.state.best_round
- if incumbent is None or incumbent >= len(rounds) or incumbent == new_index:
- return selector.pick(rounds)
- inc, new = rounds[incumbent], rounds[new_index]
- if new.score is None or inc.score is None:
- return selector.pick(rounds)
- compare = _pairwise_fn(ctx, inc.renders, new.renders) if (inc.renders and new.renders) else None
- decision, note = decide_best(inc.score, new.score, margin=ctx.policy.pairwise_margin,
- min_confidence=ctx.policy.pairwise_min_confidence, compare=compare,
- labels=(f"r{incumbent:02d}", f"r{new_index:02d}"))
- if note is not None:
- ctx.budget.add(note.usage, stage=Stage.PAIRWISE) # post-hoc
- new.usage = new.usage + note.usage
- new.pairwise = note
- new.notes = (new.notes + "; " if new.notes else "") + note.line()
- ctx.ws.write_json(round_record_path(ctx, new_index), new)
- ctx.events.emit("pairwise.done", stage="rounds", a=note.a, b=note.b, winner=note.winner, confidence=note.confidence,
- accepted=note.accepted, error=note.error, cost_usd=round(note.usage.cost_usd, 4))
- if decision == "score":
- return selector.pick(rounds)
- return new_index if decision == "pairwise" else incumbent
-
-
-def replay_best_round(journal: Sequence[RoundRecord]) -> int | None:
- """The best index the loop chose, replaying the paid pairwise verdicts it stored.
+ if incumbent is not None and incumbent < len(rounds) and incumbent != new_index:
+ inc, new = rounds[incumbent], rounds[new_index]
+ if new.score is not None and inc.score is not None:
+ compare = _pairwise_fn(ctx, inc.renders, new.renders) if (inc.renders and new.renders) else None
+ _, note = decide_best(inc.score, new.score, margin=ctx.policy.pairwise_margin,
+ min_confidence=ctx.policy.pairwise_min_confidence, compare=compare,
+ labels=(f"r{incumbent:02d}", f"r{new_index:02d}"))
+ if note is not None:
+ ctx.budget.add(note.usage, stage=Stage.PAIRWISE) # post-hoc
+ new.usage = new.usage + note.usage
+ new.pairwise = note
+ new.notes = (new.notes + "; " if new.notes else "") + note.line()
+ ctx.ws.write_json(round_record_path(ctx, new_index), new)
+ ctx.events.emit("pairwise.done", stage="rounds", a=note.a, b=note.b, winner=note.winner, confidence=note.confidence,
+ accepted=note.accepted, error=note.error, cost_usd=round(note.usage.cost_usd, 4))
+ return replay_best_round(rounds, selector=selector)
+
+
+def replay_best_round(journal: Sequence[RoundRecord], selector: BestSelector | None = None) -> int | None:
+ """The best index, walking the journal sequentially and honouring stored verdicts.
``rNN.json`` + the git commit are durable BEFORE the best is promoted and the state
saved (``lifecycle._round_loop``), so a kill in that gap — or any resume of a run
whose state predates ``best_considered_through`` — used to re-rank on score alone
and reverse a ~$0.05 judgement the run had already bought (18 such runs on disk,
- 2026-08-30). Walking the journal the way the live loop did keeps it: a round that
- carries a :class:`PairwiseNote` was decided inside ``policy.pairwise_margin``, where
- score ranking has nothing to say, so the stored verdict — replace or keep — wins;
- every other round falls back to the same ``BestSelector`` ranking."""
- selector = BestSelector()
+ 2026-08-30). Each round is an incumbent-vs-challenger step: a round that carries a
+ :class:`PairwiseNote` was decided inside ``policy.pairwise_margin``, where score
+ ranking has nothing to say, so the stored verdict — replace or keep — is FINAL
+ (a later, worse round can never revive a rejected challenger by global re-ranking);
+ every other round advances by the same two-way ``BestSelector`` rule, whose key is
+ a total order — so absent verdicts this equals the global pick."""
+ selector = selector or BestSelector()
best: int | None = None
for i, rec in enumerate(journal):
- if rec.pairwise is None or best is None or best == i:
+ if rec.pairwise is not None and best is not None and best != i:
+ if rec.pairwise.accepted:
+ best = i
+ elif best is None:
best = selector.pick(journal[:i + 1])
- elif rec.pairwise.accepted:
+ elif selector.pick([journal[best], rec]) == 1:
best = i
return best
diff --git a/harness/tests/orchestrator_tracks/test_resume_reconcile.py b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
index b6014266..6bf71633 100644
--- a/harness/tests/orchestrator_tracks/test_resume_reconcile.py
+++ b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
@@ -157,6 +157,39 @@ def test_a_self_consistent_state_with_an_unranked_round_still_re_ranks(completed
assert ev["state_was_stale"] is False and ev["unranked_rounds"] is True and ev["best_changed"] is True
+def test_a_stored_pairwise_rejection_is_final(chair_plan):
+ """A paid pairwise REJECTION of a challenger (r1 within margin, judge said keep r0)
+ used to be silently overturned once ANY later round landed: both the live
+ ``choose_best_round`` and the resume ``replay_best_round`` re-ranked the WHOLE
+ journal on score alone and crowned the rejected r1. The verdict is final: live
+ and replay agree, whether the later round is worse or failed to score at all."""
+ from codeverse.contracts.artifacts import BuildResult, Judgment
+ from codeverse.contracts.run import PairwiseNote, RoundRecord
+ from codeverse.orchestrator import BestSelector
+ from codeverse.tracks.candidates import choose_best_round, replay_best_round
+
+ def rec(i, score, pairwise=None):
+ j = None if score is None else Judgment(rubric="r", scores={}, overall=score, passed=False)
+ return RoundRecord(index=i, kind="baseline" if i == 0 else "refine", commit=f"c{i}",
+ build=BuildResult(ok=score is not None, language="blender", entrypoint="src/main.py"),
+ judgment=j, pairwise=pairwise)
+
+ r0 = rec(0, 0.70)
+ rejected = PairwiseNote(a="r00", b="r01", winner="a", confidence=0.9, accepted=False)
+ r1 = rec(1, 0.72, pairwise=rejected) # outscores r0, but the paid verdict said keep r0
+ r2 = rec(2, 0.50)
+ assert replay_best_round([r0, r1]) == 0
+ assert replay_best_round([r0, r1, r2]) == 0, "a worse later round must not revive the rejected r1"
+ ctx = SimpleNamespace(state=SimpleNamespace(best_round=0),
+ policy=SimpleNamespace(pairwise_margin=0.03, pairwise_min_confidence=0.6))
+ assert choose_best_round(ctx, [r0, r1, r2], BestSelector(), 2) == 0, "live must agree with replay"
+ r2b = rec(2, None) # the new round never scored (build crash)
+ assert choose_best_round(ctx, [r0, r1, r2b], BestSelector(), 2) == 0
+ # an ACCEPTED verdict still promotes the challenger, and survives later worse rounds
+ accepted = PairwiseNote(a="r00", b="r01", winner="b", confidence=0.9, accepted=True)
+ assert replay_best_round([r0, rec(1, 0.72, pairwise=accepted), r2]) == 1
+
+
# --------------------------------------------------------------------- (b) history wipe
def test_a_planner_outage_on_resume_keeps_the_recorded_history(completed_run):
"""The FAILED handler serialized record.json with rounds=[] because the journal was