Official Codebase of 3DCodeVerse
Python
candidates: a paid pairwise rejection is final — live and replay share one sequential rule
gaoypeng205fb066bf75
diff --git a/harness/codeverse/tracks/candidates.py b/harness/codeverse/tracks/candidates.pyindex 8bfbc5f7..07d8cef1 100644--- a/harness/codeverse/tracks/candidates.py+++ b/harness/codeverse/tracks/candidates.py@@ -245,47 +245,49 @@ def choose_best_round(ctx: RunContext, rounds: list[RoundRecord], selector: Bestcomparison of the two render sets decides, and the new round replaces thebest only when it wins with ``confidence ≥ policy.pairwise_min_confidence``.The verdict is stored on the round (``RoundRecord.pairwise``, plus a notes line)- and re-persisted, so :func:`replay_best_round` can reproduce it on resume."""+ and re-persisted; the decision itself is :func:`replay_best_round` over the journal,+ so the live pick and any later resume replay the SAME sequential rule."""incumbent = ctx.state.best_round- if incumbent is None or incumbent >= len(rounds) or incumbent == new_index:- return selector.pick(rounds)- inc, new = rounds[incumbent], rounds[new_index]- if new.score is None or inc.score is None:- return selector.pick(rounds)- compare = _pairwise_fn(ctx, inc.renders, new.renders) if (inc.renders and new.renders) else None- decision, note = decide_best(inc.score, new.score, margin=ctx.policy.pairwise_margin,- min_confidence=ctx.policy.pairwise_min_confidence, compare=compare,- labels=(f"r{incumbent:02d}", f"r{new_index:02d}"))- if note is not None:- ctx.budget.add(note.usage, stage=Stage.PAIRWISE) # post-hoc- new.usage = new.usage + note.usage- new.pairwise = note- new.notes = (new.notes + "; " if new.notes else "") + note.line()- ctx.ws.write_json(round_record_path(ctx, new_index), new)- ctx.events.emit("pairwise.done", stage="rounds", a=note.a, b=note.b, winner=note.winner, confidence=note.confidence,- accepted=note.accepted, error=note.error, cost_usd=round(note.usage.cost_usd, 4))- if decision == "score":- return selector.pick(rounds)- return new_index if decision == "pairwise" else incumbent---def replay_best_round(journal: Sequence[RoundRecord]) -> int | None:- """The best index the loop chose, replaying the paid pairwise verdicts it stored.+ if incumbent is not None and incumbent < len(rounds) and incumbent != new_index:+ inc, new = rounds[incumbent], rounds[new_index]+ if new.score is not None and inc.score is not None:+ compare = _pairwise_fn(ctx, inc.renders, new.renders) if (inc.renders and new.renders) else None+ _, note = decide_best(inc.score, new.score, margin=ctx.policy.pairwise_margin,+ min_confidence=ctx.policy.pairwise_min_confidence, compare=compare,+ labels=(f"r{incumbent:02d}", f"r{new_index:02d}"))+ if note is not None:+ ctx.budget.add(note.usage, stage=Stage.PAIRWISE) # post-hoc+ new.usage = new.usage + note.usage+ new.pairwise = note+ new.notes = (new.notes + "; " if new.notes else "") + note.line()+ ctx.ws.write_json(round_record_path(ctx, new_index), new)+ ctx.events.emit("pairwise.done", stage="rounds", a=note.a, b=note.b, winner=note.winner, confidence=note.confidence,+ accepted=note.accepted, error=note.error, cost_usd=round(note.usage.cost_usd, 4))+ return replay_best_round(rounds, selector=selector)+++def replay_best_round(journal: Sequence[RoundRecord], selector: BestSelector | None = None) -> int | None:+ """The best index, walking the journal sequentially and honouring stored verdicts.``rNN.json`` + the git commit are durable BEFORE the best is promoted and the statesaved (``lifecycle._round_loop``), so a kill in that gap — or any resume of a runwhose state predates ``best_considered_through`` — used to re-rank on score aloneand reverse a ~$0.05 judgement the run had already bought (18 such runs on disk,- 2026-08-30). Walking the journal the way the live loop did keeps it: a round that- carries a :class:`PairwiseNote` was decided inside ``policy.pairwise_margin``, where- score ranking has nothing to say, so the stored verdict — replace or keep — wins;- every other round falls back to the same ``BestSelector`` ranking."""- selector = BestSelector()+ 2026-08-30). Each round is an incumbent-vs-challenger step: a round that carries a+ :class:`PairwiseNote` was decided inside ``policy.pairwise_margin``, where score+ ranking has nothing to say, so the stored verdict — replace or keep — is FINAL+ (a later, worse round can never revive a rejected challenger by global re-ranking);+ every other round advances by the same two-way ``BestSelector`` rule, whose key is+ a total order — so absent verdicts this equals the global pick."""+ selector = selector or BestSelector()best: int | None = Nonefor i, rec in enumerate(journal):- if rec.pairwise is None or best is None or best == i:+ if rec.pairwise is not None and best is not None and best != i:+ if rec.pairwise.accepted:+ best = i+ elif best is None:best = selector.pick(journal[:i + 1])- elif rec.pairwise.accepted:+ elif selector.pick([journal[best], rec]) == 1:best = ireturn bestdiff --git a/harness/tests/orchestrator_tracks/test_resume_reconcile.py b/harness/tests/orchestrator_tracks/test_resume_reconcile.pyindex b6014266..6bf71633 100644--- a/harness/tests/orchestrator_tracks/test_resume_reconcile.py+++ b/harness/tests/orchestrator_tracks/test_resume_reconcile.py@@ -157,6 +157,39 @@ def test_a_self_consistent_state_with_an_unranked_round_still_re_ranks(completedassert ev["state_was_stale"] is False and ev["unranked_rounds"] is True and ev["best_changed"] is True+def test_a_stored_pairwise_rejection_is_final(chair_plan):+ """A paid pairwise REJECTION of a challenger (r1 within margin, judge said keep r0)+ used to be silently overturned once ANY later round landed: both the live+ ``choose_best_round`` and the resume ``replay_best_round`` re-ranked the WHOLE+ journal on score alone and crowned the rejected r1. The verdict is final: live+ and replay agree, whether the later round is worse or failed to score at all."""+ from codeverse.contracts.artifacts import BuildResult, Judgment+ from codeverse.contracts.run import PairwiseNote, RoundRecord+ from codeverse.orchestrator import BestSelector+ from codeverse.tracks.candidates import choose_best_round, replay_best_round++ def rec(i, score, pairwise=None):+ j = None if score is None else Judgment(rubric="r", scores={}, overall=score, passed=False)+ return RoundRecord(index=i, kind="baseline" if i == 0 else "refine", commit=f"c{i}",+ build=BuildResult(ok=score is not None, language="blender", entrypoint="src/main.py"),+ judgment=j, pairwise=pairwise)++ r0 = rec(0, 0.70)+ rejected = PairwiseNote(a="r00", b="r01", winner="a", confidence=0.9, accepted=False)+ r1 = rec(1, 0.72, pairwise=rejected) # outscores r0, but the paid verdict said keep r0+ r2 = rec(2, 0.50)+ assert replay_best_round([r0, r1]) == 0+ assert replay_best_round([r0, r1, r2]) == 0, "a worse later round must not revive the rejected r1"+ ctx = SimpleNamespace(state=SimpleNamespace(best_round=0),+ policy=SimpleNamespace(pairwise_margin=0.03, pairwise_min_confidence=0.6))+ assert choose_best_round(ctx, [r0, r1, r2], BestSelector(), 2) == 0, "live must agree with replay"+ r2b = rec(2, None) # the new round never scored (build crash)+ assert choose_best_round(ctx, [r0, r1, r2b], BestSelector(), 2) == 0+ # an ACCEPTED verdict still promotes the challenger, and survives later worse rounds+ accepted = PairwiseNote(a="r00", b="r01", winner="b", confidence=0.9, accepted=True)+ assert replay_best_round([r0, rec(1, 0.72, pairwise=accepted), r2]) == 1++# --------------------------------------------------------------------- (b) history wipedef test_a_planner_outage_on_resume_keeps_the_recorded_history(completed_run):"""The FAILED handler serialized record.json with rounds=[] because the journal was