Official Codebase of 3DCodeVerse
Python
select: a pairwise verdict is cached for the pictures it compared — a re-rendered round is asked again
gaoypeng93eea5b3fa11
diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.pyindex 28805f2b..519c2041 100644--- a/harness/codeverse3d/addons/select.py+++ b/harness/codeverse3d/addons/select.py@@ -26,6 +26,7 @@ track returns (unless ``--no-pick``).from __future__ import annotations+import hashlibimport loggingfrom datetime import UTC, datetimefrom pathlib import Path@@ -38,7 +39,7 @@ from codeverse3d.contracts.run import RoundRecord, RunRecordfrom codeverse3d.cost.context import call_contextfrom codeverse3d.cost.types import Role, Stagefrom codeverse3d.judges.base import judged_subset, resolve_paths-from codeverse3d.proc import EventLog, read_json_or_none+from codeverse3d.proc import EventLog, read_json_or_none, sha256_filefrom codeverse3d.record.deliverable import build_deliverable, round_outputs, texture_report_forfrom codeverse3d.record.record import effective_judgment, load_record, package_run, round_complexityfrom codeverse3d.workspace import Workspace@@ -175,15 +176,19 @@ def pairwise_verdict(ws: Workspace, rec: RunRecord, a: int, b: int, *, model: stper (pair, model), cached at ``artifacts/judge/rAA_vs_rBB_pairwise.json``. A judgeoutage keeps ``a`` (never a worse pick for a failed call)."""cache = ws.judge_path(a, f"_vs_r{b:02d}_pairwise")- stored = read_json_or_none(cache) or {}- if stored.get("model") == model and isinstance(stored.get("note"), dict):- return PairwiseNote.model_validate(stored["note"])rounds = {r.index: r for r in rec.rounds}renders = [judged_subset(resolve_paths(ws, rounds[i].renders)) for i in (a, b)]note = PairwiseNote(a=f"r{a:02d}", b=f"r{b:02d}")if renders[0] is None or renders[1] is None:note.error = "a round has no renders to compare"return note+ # the verdict is about THESE pictures: a re-rendered round is a new comparison+ seen = hashlib.sha256("\n".join(sha256_file(v.path) if Path(v.path).is_file() else v.path+ for rs in renders for v in rs.views).encode()).hexdigest()+ stored = read_json_or_none(cache) or {}+ if (stored.get("model") == model and stored.get("renders", seen) == seen # pre-2026-09-23 caches kept no fingerprint+ and isinstance(stored.get("note"), dict)):+ return PairwiseNote.model_validate(stored["note"])try:if judge is None:from codeverse3d.judges.pairwise import PairwiseJudge@@ -199,7 +204,7 @@ def pairwise_verdict(ws: Workspace, rec: RunRecord, a: int, b: int, *, model: stnote.winner, note.confidence = res.winner, float(res.confidence or 0.0)note.reasons, note.usage, note.error = list(res.reasons or [])[:6], res.usage or Usage(), res.error or ""note.accepted = note.winner == "b" and note.confidence >= PAIRWISE_MIN_CONFIDENCE- ws.write_json(cache, {"model": model, "note": note.model_dump(mode="json")})+ ws.write_json(cache, {"model": model, "renders": seen, "note": note.model_dump(mode="json")})EventLog(ws.events_path).emit("pick.pairwise", a=a, b=b, winner=note.winner, confidence=note.confidence,accepted=note.accepted, cost_usd=round(note.usage.cost_usd, 4))return notediff --git a/harness/tests/flywheel_cli/test_select.py b/harness/tests/flywheel_cli/test_select.pyindex eea354b8..9aa8e4f0 100644--- a/harness/tests/flywheel_cli/test_select.py+++ b/harness/tests/flywheel_cli/test_select.py@@ -118,6 +118,8 @@ def test_pick_by_pairwise_asks_only_inside_the_margin_and_never_buys_a_verdict_tassert judge.calls and "/r01/" in judge.calls[0][0] and judge.calls[0][2] == "static_object_v1"assert select.pick(ws.root, by="pairwise", judge=judge) == 0 and len(judge.calls) == 1 # cached, not re-boughtassert ws.judge_path(1, "_vs_r00_pairwise").is_file()+ Path(judge.calls[0][0]).write_bytes(b"re-rendered") # new pictures are a new comparison+ assert select.pick(ws.root, by="pairwise", judge=judge) == 0 and len(judge.calls) == 2# an unsure verdict, an outage or a clear gap keep the top scoreassert select.pick(_run(tmp_path / "u", [(0.70, 0), (0.72, 0)]).root, by="pairwise", judge=_Pairwise("b", 0.5)) == 1assert select.pick(_run(tmp_path / "f", [(0.70, 0), (0.72, 0)]).root, by="pairwise",