Skip to content

Official Codebase of 3DCodeVerse

Python

select: a pairwise verdict is cached for the pictures it compared — a re-rendered round is asked again

gaoypeng93eea5b3fa11
diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.py
index 28805f2b..519c2041 100644
--- a/harness/codeverse3d/addons/select.py
+++ b/harness/codeverse3d/addons/select.py
@@ -26,6 +26,7 @@ track returns (unless ``--no-pick``).
from __future__ import annotations
+import hashlib
import logging
from datetime import UTC, datetime
from pathlib import Path
@@ -38,7 +39,7 @@ from codeverse3d.contracts.run import RoundRecord, RunRecord
from codeverse3d.cost.context import call_context
from codeverse3d.cost.types import Role, Stage
from codeverse3d.judges.base import judged_subset, resolve_paths
-from codeverse3d.proc import EventLog, read_json_or_none
+from codeverse3d.proc import EventLog, read_json_or_none, sha256_file
from codeverse3d.record.deliverable import build_deliverable, round_outputs, texture_report_for
from codeverse3d.record.record import effective_judgment, load_record, package_run, round_complexity
from codeverse3d.workspace import Workspace
@@ -175,15 +176,19 @@ def pairwise_verdict(ws: Workspace, rec: RunRecord, a: int, b: int, *, model: st
per (pair, model), cached at ``artifacts/judge/rAA_vs_rBB_pairwise.json``. A judge
outage keeps ``a`` (never a worse pick for a failed call)."""
cache = ws.judge_path(a, f"_vs_r{b:02d}_pairwise")
- stored = read_json_or_none(cache) or {}
- if stored.get("model") == model and isinstance(stored.get("note"), dict):
- return PairwiseNote.model_validate(stored["note"])
rounds = {r.index: r for r in rec.rounds}
renders = [judged_subset(resolve_paths(ws, rounds[i].renders)) for i in (a, b)]
note = PairwiseNote(a=f"r{a:02d}", b=f"r{b:02d}")
if renders[0] is None or renders[1] is None:
note.error = "a round has no renders to compare"
return note
+ # the verdict is about THESE pictures: a re-rendered round is a new comparison
+ seen = hashlib.sha256("\n".join(sha256_file(v.path) if Path(v.path).is_file() else v.path
+ for rs in renders for v in rs.views).encode()).hexdigest()
+ stored = read_json_or_none(cache) or {}
+ if (stored.get("model") == model and stored.get("renders", seen) == seen # pre-2026-09-23 caches kept no fingerprint
+ and isinstance(stored.get("note"), dict)):
+ return PairwiseNote.model_validate(stored["note"])
try:
if judge is None:
from codeverse3d.judges.pairwise import PairwiseJudge
@@ -199,7 +204,7 @@ def pairwise_verdict(ws: Workspace, rec: RunRecord, a: int, b: int, *, model: st
note.winner, note.confidence = res.winner, float(res.confidence or 0.0)
note.reasons, note.usage, note.error = list(res.reasons or [])[:6], res.usage or Usage(), res.error or ""
note.accepted = note.winner == "b" and note.confidence >= PAIRWISE_MIN_CONFIDENCE
- ws.write_json(cache, {"model": model, "note": note.model_dump(mode="json")})
+ ws.write_json(cache, {"model": model, "renders": seen, "note": note.model_dump(mode="json")})
EventLog(ws.events_path).emit("pick.pairwise", a=a, b=b, winner=note.winner, confidence=note.confidence,
accepted=note.accepted, cost_usd=round(note.usage.cost_usd, 4))
return note
diff --git a/harness/tests/flywheel_cli/test_select.py b/harness/tests/flywheel_cli/test_select.py
index eea354b8..9aa8e4f0 100644
--- a/harness/tests/flywheel_cli/test_select.py
+++ b/harness/tests/flywheel_cli/test_select.py
@@ -118,6 +118,8 @@ def test_pick_by_pairwise_asks_only_inside_the_margin_and_never_buys_a_verdict_t
assert judge.calls and "/r01/" in judge.calls[0][0] and judge.calls[0][2] == "static_object_v1"
assert select.pick(ws.root, by="pairwise", judge=judge) == 0 and len(judge.calls) == 1 # cached, not re-bought
assert ws.judge_path(1, "_vs_r00_pairwise").is_file()
+ Path(judge.calls[0][0]).write_bytes(b"re-rendered") # new pictures are a new comparison
+ assert select.pick(ws.root, by="pairwise", judge=judge) == 0 and len(judge.calls) == 2
# an unsure verdict, an outage or a clear gap keep the top score
assert select.pick(_run(tmp_path / "u", [(0.70, 0), (0.72, 0)]).root, by="pairwise", judge=_Pairwise("b", 0.5)) == 1
assert select.pick(_run(tmp_path / "f", [(0.70, 0), (0.72, 0)]).root, by="pairwise",