Official Codebase of 3DCodeVerse
Python
silhouette: the gate and the reference judge read one measure_reference, on the best-matching view
gaoypengf206b63ac2ae
diff --git a/harness/codeverse3d/judges/vlm_judge.py b/harness/codeverse3d/judges/vlm_judge.pyindex e655469d..696ef6b1 100644--- a/harness/codeverse3d/judges/vlm_judge.py+++ b/harness/codeverse3d/judges/vlm_judge.py@@ -21,7 +21,6 @@ import contextlibimport loggingimport shutilimport time-from collections.abc import Callablefrom dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Literal@@ -30,7 +29,7 @@ from codeverse3d.config import get_settingsfrom codeverse3d.contracts.artifacts import Judgment, RenderViewfrom codeverse3d.contracts.chat import ChatRequest, ChatResponsefrom codeverse3d.contracts.common import TRACK_INFO, Track, Usage-from codeverse3d.conventions import front_view, views_by_preference+from codeverse3d.conventions import views_by_preferencefrom codeverse3d.judges.base import SLICE_TRACKS, JudgeInputfrom codeverse3d.judges.prompt_builder import (_key,@@ -54,7 +53,12 @@ from codeverse3d.models import get_chat_modelfrom codeverse3d.models.base import ChatModel, ModelErrorfrom codeverse3d.proc import fan_out, unique_tmpfrom codeverse3d.reference import compare, conflict_note, dimension_conflict-from codeverse3d.spatial.silhouette import best_view_match, compare_silhouette+from codeverse3d.spatial.silhouette import (+ CompareFn,+ ReferenceMatch,+ compare_silhouette,+ measure_reference,+)log = logging.getLogger(__name__)@@ -332,8 +336,6 @@ class VlmJudge:# ===================================================================== reference-SilhouetteFn = Callable[[str, str], dict[str, Any]]-#: IoU → score mapping: ≤ 0.25 → 0, ≥ 0.85 → 1, linear in between (matches the rubric anchors).IOU_LOW, IOU_HIGH = 0.25, 0.85@@ -371,8 +373,7 @@ class ReferenceJudge(VlmJudge):temperature: float = 0.2,*,rubric: str | Rubric = "reference_v1",- silhouette_fn: SilhouetteFn | None = None,- best_view: bool = True,+ silhouette_fn: CompareFn | None = None,diff: bool = True,diff_model: Any | None = None,diff_model_id: str = "",@@ -380,7 +381,6 @@ class ReferenceJudge(VlmJudge):):super().__init__(rubric, model_id, n_samples, temperature, **kwargs)self.silhouette_fn = silhouette_fn or compare_silhouette- self.best_view = best_viewself.diff = diffself._diff_model = diff_modelself.diff_model_id = diff_model_id@@ -405,22 +405,19 @@ class ReferenceJudge(VlmJudge):if note:blocks.append(note)measured_ids = [c.id for c in self.rubric.measured_criteria()]- info: dict[str, Any] = {}measured_scores: dict[str, float] = {}+ match = measure_reference(inp.renders.views, inp.spec, compare=self.silhouette_fn) if refs else Noneif measured_ids:- info = self.measure_silhouette(inp)- if "iou" not in info:- raise ReferenceJudgeError(- f"cannot score measured criteria {measured_ids}: {info.get('error', 'no iou')}")- score, why = self.silhouette_score(info, conflict)+ if match is None:+ raise ReferenceJudgeError(f"cannot score measured criteria {measured_ids}: "+ + ("no render view compared" if refs else "spec has no readable reference images"))+ score, why = self.silhouette_score(match)measured_scores = {cid: score for cid in measured_ids}+ extra = match.model_dump(exclude={"iou", "reference", "conflict", "per_view"}, exclude_none=True)blocks.append(- f"MEASURED SILHOUETTE (harness): {info['render']} render vs reference {info['reference']}: "- f"IoU {info['iou']:.3f} → silhouette_match score {score:.2f}{why}."- + (f" extra: {info['extra']}" if info.get("extra") else ""))- elif refs:- info = self.measure_silhouette(inp)- diff = self.reference_diff(inp, refs, info, synthesized=synth) if refs else None+ f"MEASURED SILHOUETTE (harness): {match.view} render vs reference {Path(match.reference).name}: "+ f"IoU {match.iou:.3f} → silhouette_match score {score:.2f}{why}. extra: {extra}")+ diff = self.reference_diff(inp, refs, match, synthesized=synth) if refs else Noneif diff is not None and diff.as_text():blocks.append(diff.as_text())return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images,@@ -442,26 +439,25 @@ class ReferenceJudge(VlmJudge):log.warning("dimension-conflict check failed: %s", e)return {"conflict": False, "error": str(e)}- def silhouette_score(self, info: dict[str, Any], conflict: dict[str, Any]) -> tuple[float, str]:+ def silhouette_score(self, match: ReferenceMatch) -> tuple[float, str]:"""IoU → measured score, NEUTRAL when the reference contradicts the brief or themask is unreliable. Returns ``(score, ' (why)')``."""- if conflict.get("conflict"):+ if match.conflict:return NEUTRAL_SCORE, " (NEUTRAL: the reference's proportions contradict the brief's dimensions)"- if info.get("reliable") is False:+ if not match.reliable:return NEUTRAL_SCORE, " (NEUTRAL: the background mask was unreliable)"- return iou_to_score(info["iou"]), ""+ return iou_to_score(match.iou), ""# ------------------------------------------------------------------ mismatch pass- def reference_diff(self, inp: JudgeInput, refs: list[Any], info: dict[str, Any], *, synthesized: bool) -> Any | None:+ def reference_diff(self, inp: JudgeInput, refs: list[Any], match: ReferenceMatch | None, *, synthesized: bool) -> Any | None:"""One extra vision call naming concrete mismatches. ``None`` when disabled;never raises (a failed diff simply contributes no text)."""if not _diff_enabled(self.diff):return Nonetargets = [r.path for r in refs if r.role == "target"] or [r.path for r in refs]renders = self.diff_views(list(inp.renders.views))- measured = dict(info) if "iou" in info else None- if measured is not None:- measured.setdefault("view", info.get("render", ""))+ measured = None if match is None else {"iou": match.iou, "aspect_ratio_err": match.aspect_ratio_err, "reliable": match.reliable,+ "view": match.view, "reference": Path(match.reference).name}try:return compare(inp.spec, targets, renders, model=self._pick_diff_model(),part_names=inp.part_names, measured=measured, synthesized=synthesized)@@ -488,45 +484,6 @@ class ReferenceJudge(VlmJudge):picked.append(v.path)return picked[:3]- # ------------------------------------------------------------------ silhouette- def measure_silhouette(self, inp: JudgeInput) -> dict[str, Any]:- """{iou, render, reference, extra} or {error}.-- With ``best_view=True`` (default) the IoU is taken from the render view that- best matches the reference's camera instead of a hardcoded ``front``: a- reference photograph has ONE camera, and comparing a three-quarter product- shot to a straight-on elevation measures camera agreement, not shape. The- winning view name travels in ``render`` so the number stays auditable.- """- targets = [r for r in inp.spec.references if r.role == "target" and Path(r.path).is_file()] or [- r for r in inp.spec.references if Path(r.path).is_file()]- if not targets:- return {"error": "spec has no readable reference images"}- views = list(inp.renders.views)- if not views:- return {"error": "render set has no views"}- res, name = self._measure(views, targets[0].path)- if not isinstance(res, dict) or "iou" not in res:- return {"error": f"compare_silhouette returned no iou: {res!r}"}- extra = {k: v for k, v in res.items() if k not in ("iou", "per_view") and isinstance(v, (int, float, str, bool))}- out = {"iou": float(res["iou"]), "render": name, "reference": Path(targets[0].path).name,- "reliable": bool(res.get("reliable", True)),- "aspect_ratio_err": res.get("aspect_ratio_err"), "extra": extra}- if res.get("per_view"):- out["per_view"] = res["per_view"]- return out-- def _measure(self, views: list[RenderView], reference: str) -> tuple[dict[str, Any], str]:- """(result, view name). Best-matching view when enabled and available."""- if self.best_view and self.silhouette_fn is compare_silhouette:- res = best_view_match(views, reference)- if "iou" in res:- return res, str(res.get("view", ""))- view = front_view(views)- if view is None: # pragma: no cover - guarded by the caller- return {}, ""- return self.silhouette_fn(view.path, reference), view.name-LIKENESS_NOTE = ("REAL-WORLD REFERENCE PHOTOS are attached (labelled REAL-WORLD REFERENCE n/N). They are not a "@@ -545,7 +502,7 @@ LIKENESS_NOTE = (class LikenessJudge(VlmJudge):"""``VlmJudge`` + the reference photos of the REAL thing — for tracks with no silhouette.- ``ReferenceJudge`` is built for objects: it measures a front-view silhouette IoU against+ ``ReferenceJudge`` is built for objects: it measures a best-view silhouette IoU againstthe target photo and runs a part-inventory mismatch pass. Neither means anything for afragment shader or a scene, where a reference photo answers a different question — doesthis LOOK like the thing? Measured 2026-08-26 (teaser aurora, three versions, flash anddiff --git a/harness/codeverse3d/reference.py b/harness/codeverse3d/reference.pyindex b721e29d..6fc9fdeb 100644--- a/harness/codeverse3d/reference.py+++ b/harness/codeverse3d/reference.py@@ -314,9 +314,9 @@ VIEW_CLAUSE: dict[str, str] = {}#: order views are requested in as ``n_views`` grows. The straight-on ``front``-#: elevation comes second on purpose: it is the SILHOUETTE TARGET, and the harness-#: measures IoU against the run's ``front`` render (``tracks.static_object.silhouette_gate``),-#: so the two cameras must agree. The 3/4 shot carries the part inventory.+#: elevation comes second on purpose: it is the SILHOUETTE TARGET, and a straight-on+#: elevation has a render camera to match (``silhouette.measure_reference`` takes the+#: best-matching view — the gate and the judge both). The 3/4 shot carries the part inventory.VIEW_ORDER: tuple[str, ...] = ("three_quarter", "front", "side", "back")diff --git a/harness/codeverse3d/spatial/silhouette.py b/harness/codeverse3d/spatial/silhouette.pyindex ccbdb707..11526f0a 100644--- a/harness/codeverse3d/spatial/silhouette.py+++ b/harness/codeverse3d/spatial/silhouette.py@@ -22,14 +22,16 @@ signal separately.from __future__ import annotationsfrom collections import deque-from collections.abc import Sequence+from collections.abc import Callable, Sequencefrom pathlib import Pathfrom typing import Anyimport numpy as npfrom PIL import Image+from pydantic import BaseModel, Fieldfrom codeverse3d.contracts.artifacts import RenderView+from codeverse3d.contracts.spec import ReferenceImage, Specfrom codeverse3d.conventions import views_by_preferenceMASK_SIZE = 256@@ -213,12 +215,18 @@ CANDIDATE_VIEWS: tuple[str, ...] = ()+#: ``compare_silhouette``'s shape: (render png, reference png) → its result dict. Injectable, so+#: the track's services and the judge's tests can stand in for the mask arithmetic.+CompareFn = Callable[[str, str], dict[str, Any]]++def best_view_match(renders: Sequence[RenderView],reference: Path | str,*,candidates: Sequence[str] = CANDIDATE_VIEWS,diff_png: Path | str | None = None,+ compare: CompareFn | None = None,) -> dict[str, Any]:"""IoU of the render view that best matches ``reference``, and which one it was.@@ -230,15 +238,18 @@ def best_view_match(Returns the :func:`compare_silhouette` dict plus ``view`` (the winning view name)and ``per_view`` (name → IoU). ``{"error": ...}`` when nothing could be compared."""+ compare = compare or compare_silhouetteper_view: dict[str, float] = {}best: tuple[float, RenderView, dict[str, Any]] | None = Nonefor view in views_by_preference(renders, candidates) or list(renders):if not Path(view.path).is_file():continuetry:- res = compare_silhouette(view.path, reference)+ res = compare(str(view.path), str(reference))except (OSError, ValueError):continue+ if not isinstance(res, dict) or "iou" not in res:+ continueper_view[view.name] = res["iou"]if best is None or res["iou"] > best[0]:best = (res["iou"], view, res)@@ -251,3 +262,52 @@ def best_view_match(out.update(compare_silhouette(view.path, reference, diff_png=diff_png))out["view"] = view.namereturn out+++def target_reference(references: Sequence[ReferenceImage]) -> str | None:+ """THE silhouette target: the first readable reference with role ``target``, else the first+ readable one; ``None`` when no reference file exists."""+ readable = [r for r in references if Path(r.path).is_file()]+ return next((r.path for r in readable if r.role == "target"), readable[0].path if readable else None)+++class ReferenceMatch(BaseModel):+ """The one silhouette measurement of a round against its target reference: the+ ``reference_silhouette`` gate records it (and refines on it), the reference judge scores+ ``silhouette_match`` from it. Until 2026-09-24 the gate compared the ``front`` render only+ while the judge took the best-matching view — 0.26 vs 0.995 on one reference (audit N51)."""++ iou: float+ ref_fill: float | None = None+ render_fill: float | None = None+ aspect_ratio_err: float | None = None+ ref_aspect: float | None = None+ render_aspect: float | None = None+ reliable: bool = Field(default=True, description="both foreground masks localised something")+ view: str = Field(description="the render view that best matches the reference's camera")+ reference: str = Field(description="path of the target reference image")+ conflict: bool = Field(default=False, description="the reference's own proportions contradict the brief's dimensions")+ per_view: dict[str, float] = Field(default_factory=dict)++ @property+ def scorable(self) -> bool:+ """May the IoU be read as a shape verdict? Not on an unreliable mask, and not when+ the reference contradicts the brief (the brief wins; the score is neutral)."""+ return self.reliable and not self.conflict+++def measure_reference(renders: Sequence[RenderView], spec: Spec, *, compare: CompareFn | None = None) -> ReferenceMatch | None:+ """:class:`ReferenceMatch` of ``renders`` against ``spec``'s target reference, or ``None``+ when the spec has no readable reference or no render view could be compared. An error in+ ``compare`` itself propagates: the caller decides what "unavailable" means."""+ from codeverse3d.reference import dimension_conflict # reference.py imports this module++ ref = target_reference(spec.references)+ if ref is None:+ return None+ res = best_view_match(renders, ref, compare=compare)+ if "iou" not in res:+ return None+ num = {k: float(res[k]) for k in ("ref_fill", "render_fill", "aspect_ratio_err", "ref_aspect", "render_aspect") if isinstance(res.get(k), int | float)}+ return ReferenceMatch(iou=float(res["iou"]), view=str(res.get("view", "")), reference=ref, reliable=bool(res.get("reliable", True)),+ conflict=bool(dimension_conflict(spec, ref).get("conflict")), per_view=res.get("per_view") or {}, **num)diff --git a/harness/codeverse3d/tracks/prompting.py b/harness/codeverse3d/tracks/prompting.pyindex 800f8fe7..397d2b81 100644--- a/harness/codeverse3d/tracks/prompting.py+++ b/harness/codeverse3d/tracks/prompting.py@@ -414,8 +414,8 @@ def reference_note(ctx: RunContext) -> str:return _likeness_note(ctx, refs)lines = [f"REFERENCE IMAGES ({len(refs)}): match their silhouette, proportions and visible details — they "- "outrank the text when the two disagree. A harness measures the front-view outline IoU against the "- "target reference."+ "outrank the text when the two disagree. A harness measures the outline IoU of the best-matching "+ "render view against the target reference."]for i, r in enumerate(refs, 1):lines.append(f"- reference {i} ({r.role}): `{r.path}`" + (f" — {r.note}" if r.note else ""))diff --git a/harness/codeverse3d/tracks/static_object.py b/harness/codeverse3d/tracks/static_object.pyindex 8a0a78cf..69f808ba 100644--- a/harness/codeverse3d/tracks/static_object.py+++ b/harness/codeverse3d/tracks/static_object.py@@ -26,11 +26,11 @@ from codeverse3d.conventions import (BBOX_TOLERANCE_M,OBJECT_CLAY_VIEWS,OBJECT_VIEWS,- front_view,)from codeverse3d.orchestrator import RefineTask, TaskGroup, compact_instructionsfrom codeverse3d.prompts import renderfrom codeverse3d.spatial.contract import planned_joins+from codeverse3d.spatial.silhouette import measure_referencefrom codeverse3d.tracks.common import RunContextfrom codeverse3d.tracks.depth import (PartScope,@@ -102,7 +102,7 @@ class ObjectPipeline:def post_render_gates(self, ctx: RunContext, round_index: int, renders: RenderSet) -> list[GateReport]:- """Reference-image runs: front-view outline IoU vs the target image (WARN finding with the number)."""+ """Reference-image runs: outline IoU vs the target image (WARN finding with the number)."""gate = silhouette_gate(ctx, renders)return [gate] if gate is not None else []@@ -299,36 +299,26 @@ SILHOUETTE_GATE = "reference_silhouette"IOU_REFINE_THRESHOLD = 0.6-def target_reference(ctx: RunContext) -> str | None:- """Path of the reference image to match the silhouette against (role ``target`` first)."""- refs = [r for r in ctx.spec.references if Path(r.path).is_file()]- if not refs:- return None- return next((r.path for r in refs if r.role == "target"), refs[0].path)--def silhouette_gate(ctx: RunContext, renders: RenderSet) -> GateReport | None:- """``None`` when the spec has no reference images or nothing could be compared."""- ref = target_reference(ctx)- view = front_view(renders.views)- if ref is None or view is None:- return None+ """The round's :func:`~codeverse3d.spatial.silhouette.measure_reference` — the measurement the+ reference judge scores — as a finding: WARN below the refine threshold when the IoU is a shape+ verdict (reliable mask, reference agrees with the brief), else INFO. ``None`` when the spec+ has no reference or nothing could be compared."""try:- res = ctx.services.silhouette(view.path, ref)+ m = measure_reference(renders.views, ctx.spec, compare=ctx.services.silhouette)except Exception as e: # noqa: BLE001 — advisory measurement; never fails a roundlog.warning("compare_silhouette failed: %s", e)return GateReport.of(SILHOUETTE_GATE, [GateFinding(gate=SILHOUETTE_GATE, severity=Severity.INFO, target="overall", message=f"silhouette comparison unavailable: {e}")])- iou = float(res.get("iou", 0.0)) if isinstance(res, dict) else 0.0- reliable = bool(res.get("reliable", True)) if isinstance(res, dict) else False- data = {k: v for k, v in (res.items() if isinstance(res, dict) else []) if isinstance(v, (int, float, str, bool))}- data["view"] = view.name- data["reference"] = ref- low = reliable and iou < IOU_REFINE_THRESHOLD- msg = (f"front-view outline IoU vs reference = {iou:.3f}" + ("" if reliable else " (unreliable mask)")+ if m is None:+ return None+ low = m.scorable and m.iou < IOU_REFINE_THRESHOLD+ msg = (f"outline IoU vs reference = {m.iou:.3f} (best-matching view: {m.view})"+ + ("" if m.reliable else " (unreliable mask)")+ + (" (the reference's proportions contradict the brief: not a shape verdict)" if m.conflict else "")+ (f" — below {IOU_REFINE_THRESHOLD:.1f}" if low else ""))finding = GateFinding(gate=SILHOUETTE_GATE, severity=Severity.WARN if low else Severity.INFO, target="overall",- message=msg, data=data,+ message=msg, data=m.model_dump(exclude={"per_view"}),fix_hint=("match the reference outline: compare proportions (aspect ratio), overall extents and the ""silhouette of each major part against the reference image" if low else ""))return GateReport.of(SILHOUETTE_GATE, [finding])@@ -356,9 +346,10 @@ def reference_refine_tasks(ctx: RunContext, last: RoundRecord) -> list[RefineTasif hit is None:return []iou, data = hit- if iou >= IOU_REFINE_THRESHOLD or data.get("reliable") is False:+ if iou >= IOU_REFINE_THRESHOLD or data.get("reliable") is False or data.get("conflict"):return []- bits = [f"Front-view silhouette IoU vs the reference image is {iou:.2f} (target ≥ {IOU_REFINE_THRESHOLD:.1f})."]+ seen = f" (best-matching view: {data['view']})" if data.get("view") else ""+ bits = [f"Silhouette IoU vs the reference image is {iou:.2f}{seen} (target ≥ {IOU_REFINE_THRESHOLD:.1f})."]aspect = data.get("aspect_ratio_err")if isinstance(aspect, (int, float)):ra, rr = data.get("ref_aspect"), data.get("render_aspect")diff --git a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.pyindex 1623cd0a..d360bfe3 100644--- a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py+++ b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py@@ -125,7 +125,7 @@ def test_reference_images_wire_judge_prompts_and_silhouette(tmp_path, chair_plan# silhouette gate recorded with the IoU, and the refine round got the reference taskgate = next(g for g in rec.rounds[0].gates if g.gate == "reference_silhouette")assert gate.passed and gate.findings[0].data["iou"] == pytest.approx(0.42) and services.silhouette_calls- assert any("silhouette IoU vs the reference image is 0.42" in i for i in rec.rounds[1].instructions)+ assert any("Silhouette IoU vs the reference image is 0.42" in i for i in rec.rounds[1].instructions)assert any("wider relative to its height" in i for i in rec.rounds[1].instructions)refine_prompt = model.requests[-1].messages[0].textassert "## Reference images" in refine_prompt and "compare_reference" not in refine_prompt # single-shot: no toolsdiff --git a/harness/tests/reference/test_silhouette_best_view.py b/harness/tests/reference/test_silhouette_best_view.pyindex 469a14b2..809428b3 100644--- a/harness/tests/reference/test_silhouette_best_view.py+++ b/harness/tests/reference/test_silhouette_best_view.py@@ -3,11 +3,18 @@from __future__ import annotationsfrom pathlib import Path+from types import SimpleNamespacefrom PIL import Image, ImageDraw-from codeverse3d.contracts.artifacts import RenderView-from codeverse3d.spatial.silhouette import CANDIDATE_VIEWS, best_view_match+from codeverse3d.contracts.artifacts import RenderSet, RenderView, Severity+from codeverse3d.contracts.spec import ReferenceImage+from codeverse3d.judges.base import JudgeInput+from codeverse3d.judges.vlm_judge import ReferenceJudge+from codeverse3d.spatial.silhouette import CANDIDATE_VIEWS, best_view_match, compare_silhouette+from codeverse3d.tracks.static_object import silhouette_gate+from tests.judges.conftest import FakeChatModel+from tests.reference.conftest import make_specdef _shape(path: Path, box, bg=(240, 240, 240), fg=(30, 30, 30)) -> Path:@@ -45,3 +52,17 @@ def test_a_pre_d47_run_matches_its_own_cameras_through_one_alias_map(tmp_path: Pold = [RenderView(name=n, path=views[2].path, width=256, height=256) for n in ("front_right_34", "low_front_left")]res = best_view_match(old, _shape(tmp_path / "ref.png", (20, 95, 236, 165)))assert res["view"] == "front_right_34" and set(res["per_view"]) == {"front_right_34"}+++def test_the_gate_and_the_judge_read_one_measurement(tmp_path: Path) -> None:+ """Audit 2026-09-24 N51: the gate compared the ``front`` render (IoU 0.26 → a WARN and a+ refine round) while the judge scored the best-matching view (0.995). Both now read+ ``silhouette.measure_reference``."""+ views = _views(tmp_path / "r")+ ref = _shape(tmp_path / "ref.png", (20, 95, 236, 165))+ spec = make_spec(references=[ReferenceImage(path=str(ref), role="target")])+ ctx = SimpleNamespace(spec=spec, services=SimpleNamespace(silhouette=compare_silhouette))+ (finding,) = silhouette_gate(ctx, RenderSet(views=views)).findings+ assert finding.data["view"] == "right" and finding.data["iou"] > 0.8 and finding.severity == Severity.INFO+ jc = ReferenceJudge("fake:fake-1", chat_model=FakeChatModel([]), diff=False).context(JudgeInput(spec=spec, renders=RenderSet(views=views)))+ assert f"right render vs reference ref.png: IoU {finding.data['iou']:.3f}" in jc.extra_text