Official Codebase of 3DCodeVerse
Python
judges: thirteen modules become six, and the high-traffic names keep their addresses
gaoypenga82f941f6ee5
diff --git a/harness/codeverse/cli/_judge.py b/harness/codeverse/cli/_judge.pyindex e8294ea6..6c891585 100644--- a/harness/codeverse/cli/_judge.py+++ b/harness/codeverse/cli/_judge.py@@ -21,7 +21,7 @@ from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderViewfrom codeverse.contracts.common import TRACK_INFOfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.run import RoundRecord, RunRecord-from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_paths+from codeverse.judges.base import judged_subset, plan_digest, resolve_pathsfrom codeverse.proc import read_json_or_nonefrom codeverse.workspace import Workspace@@ -115,7 +115,7 @@ def make_judge(rec: RunRecord, rubric_name: str, model_id: str, n: int) -> Any:raise C.CliError(f"cannot load rubric {rubric_name!r}: {e}") from emeasured = bool(rubric.measured_criteria())if measured or rec.spec.references:- ReferenceJudge = C.lazy("codeverse.judges.reference", "ReferenceJudge")+ ReferenceJudge = C.lazy("codeverse.judges.vlm_judge", "ReferenceJudge")return ReferenceJudge(model_id=model_id, n_samples=n, rubric=rubric_name)VlmJudge = C.lazy("codeverse.judges.vlm_judge", "VlmJudge")return VlmJudge(rubric=rubric_name, model_id=model_id, n_samples=n)diff --git a/harness/codeverse/cli/inspect_cmd.py b/harness/codeverse/cli/inspect_cmd.pyindex 89261ab5..4f15a319 100644--- a/harness/codeverse/cli/inspect_cmd.py+++ b/harness/codeverse/cli/inspect_cmd.py@@ -228,7 +228,7 @@ def judge(except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothingraise C.CliError(f"judge failed: {e}") from eexcept Exception as e:- ReferenceJudgeError = C.lazy("codeverse.judges.reference", "ReferenceJudgeError")+ ReferenceJudgeError = C.lazy("codeverse.judges.vlm_judge", "ReferenceJudgeError")if isinstance(e, ReferenceJudgeError):raise C.CliError(f"judge failed: {e}") from eraisediff --git a/harness/codeverse/flywheel/record.py b/harness/codeverse/flywheel/record.pyindex a0eb02e5..92857812 100644--- a/harness/codeverse/flywheel/record.py+++ b/harness/codeverse/flywheel/record.py@@ -128,7 +128,7 @@ def effective_judgment(r: RoundRecord):j = r.judgmentif j is None:return None- from codeverse.judges.scoring import is_degraded+ from codeverse.judges.rubrics import is_degradedreturn None if is_degraded(j) else jdiff --git a/harness/codeverse/judges/base.py b/harness/codeverse/judges/base.pyindex 6d4edbd4..4f703bcc 100644--- a/harness/codeverse/judges/base.py+++ b/harness/codeverse/judges/base.py@@ -2,7 +2,7 @@from __future__ import annotations-from typing import Protocol+from typing import Any, Protocolfrom pydantic import BaseModel, Field@@ -10,6 +10,7 @@ from codeverse.contracts.artifacts import GateReport, Measurement, RenderSetfrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.spec import Spec+from codeverse.workspace import Workspaceclass JudgeInput(BaseModel):@@ -31,3 +32,58 @@ class Judge(Protocol):name: strdef judge(self, inp: JudgeInput) -> Judgment: ...+++# ===================================================================== replay_input+# (merged from codeverse/judges/replay_input.py, 2026-08-28)+def plan_digest(plan: dict[str, Any]) -> str:+ """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""+ bits: list[str] = []+ title = plan.get("object_name") or plan.get("title") or ""+ if title:+ bits.append(f"{title}: {plan.get('summary', '')}".strip())+ bbox = plan.get("overall_bbox") or {}+ if isinstance(bbox, dict) and bbox.get("extents"):+ e = bbox["extents"]+ bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")+ parts = plan.get("parts") or []+ if parts:+ bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")+ if plan.get("root_link"):+ bits.append(f"Root link {plan['root_link']}.")+ joints = plan.get("joints") or []+ if joints:+ bits.append("Joints: " + "; ".join(+ f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"+ for j in joints) + ".")+ for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):+ items = plan.get(key) or []+ if items:+ bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")+ if plan.get("setting"):+ bits.append(f"Setting: {plan['setting']}.")+ return " ".join(bits)+++def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:+ """Render paths out of a round record, resolved against THIS workspace.++ Delegates to ``Workspace.rebase``: this used to rebase only paths for which+ ``is_absolute()`` was False, which made it a no-op against every record the harness+ itself writes (they are all absolute) — so a moved or archived run kept pointing at+ the host it was produced on.+ """+ if rs is None:+ return None+ fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]+ sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet+ out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir+ return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})+++def judged_subset(rs: RenderSet | None) -> RenderSet | None:+ """The views the in-run judge actually saw: the per-view ``judge`` flags+ stamped at render time; legacy rounds (no flags) keep every stored view."""+ if rs is None or not any(v.judge is not None for v in rs.views):+ return rs+ return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})diff --git a/harness/codeverse/judges/calibration.py b/harness/codeverse/judges/calibration.pyindex 78f3f87b..805f8e13 100644--- a/harness/codeverse/judges/calibration.py+++ b/harness/codeverse/judges/calibration.py@@ -32,9 +32,8 @@ from codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.run import RoundRecordfrom codeverse.contracts.spec import Spec-from codeverse.judges.base import JudgeInput-from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_paths-from codeverse.judges.scoring import is_degraded+from codeverse.judges.base import JudgeInput, judged_subset, plan_digest, resolve_paths+from codeverse.judges.rubrics import is_degradedfrom codeverse.judges.vlm_judge import VlmJudgefrom codeverse.workspace import Workspacediff --git a/harness/codeverse/judges/caps.py b/harness/codeverse/judges/caps.pydeleted file mode 100644index 91bb717d..00000000--- a/harness/codeverse/judges/caps.py+++ /dev/null@@ -1,186 +0,0 @@-"""Deterministic caps: bound the overall score from gate findings / acceptance.--The VLM never sees or computes these. ``apply_caps`` returns the capped overall-and a ledger of which rules fired and why, so the verdict is auditable.-"""--from __future__ import annotations--from fnmatch import fnmatch--from pydantic import BaseModel, Field--from codeverse.contracts.artifacts import GateFinding, GateReport, RenderView, Severity-from codeverse.contracts.plan import AcceptanceItem-from codeverse.judges.rubrics import CapRule, Rubric--_SEV_RANK = {Severity.INFO: 0, Severity.WARN: 1, Severity.ERROR: 2}---class CapApplied(BaseModel):- rule: str- cap: float- evidence: str = Field(description="which finding / acceptance item triggered it")---class CapResult(BaseModel):- overall: float- caps_applied: list[CapApplied] = Field(default_factory=list)-- @property- def capped(self) -> bool:- return bool(self.caps_applied)---def _finding_tokens(f: GateFinding) -> str:- """Searchable lower-cased text for a finding: data.kind / data.code / message / target."""- bits = [str(f.data.get("kind", "")), str(f.data.get("code", "")), f.message, f.target or ""]- return " ".join(bits).lower()---def _finding_matches(rule: CapRule, report: GateReport, f: GateFinding) -> bool:- if not fnmatch(report.gate, rule.gate) and not fnmatch(f.gate, rule.gate):- return False- if _SEV_RANK[f.severity] < _SEV_RANK[rule.severity]:- return False- if rule.kinds:- text = _finding_tokens(f)- return any(k.lower() in text for k in rule.kinds)- return True---def missing_must_items(- acceptance_items: list[AcceptanceItem], acceptance_results: dict[str, bool]-) -> list[str]:- """Ids of ``must`` acceptance items that are not verified (missing counts as failed)."""- return [- a.id- for a in acceptance_items- if a.priority == "must" and not acceptance_results.get(a.id, False)- ]---def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) -> bool:- """True when a deterministic gate MEASURED the thing this checklist defect claims, and found nothing.-- A checklist defect is a VLM's reading of a picture. Some of them — a floating part, an- interpenetration — are also what a gate measures on the exported mesh, and the rubric- already says which: a ``when="gate"`` cap rule with the same id and a ``kinds`` list.- When every gate that rule watches ran and passed with no ERROR finding of those kinds,- the measurement contradicts the perception, and the perception must not cap.-- Measured 2026-08-26 on a plan-pinned pair (fancy_v1 gas_street_lamp): the connectivity- gate said "all 9 parts connected, gap <= 2 mm"; the judge read the dark seam under the- pedestal as "floating in mid-air, a clear daylight gap", marked floating_part present, and- ``defect:floating_part`` capped the run at 0.6 (uncapped 0.72) against 0.96 for a sibling- the eye cannot tell apart. CLAUDE.md law 3: gates decide geometry, the VLM is perception.-- A defect with no matching gate rule (wrong_object, missing_named_part …) is never- overridden: nothing measured it. A gate that did not run overrides nothing either.- """- rule = next(- (r for r in rubric.caps if r.when == "gate" and r.id == defect_id and r.kinds), None- )- if rule is None:- return False- watched = [g for g in gates if fnmatch(g.gate, rule.gate)]- if not watched:- return False- for g in watched:- if not g.passed:- return False- for f in g.findings:- if f.severity == Severity.ERROR and any(- k in (f.message or "").lower() for k in rule.kinds- ):- return False- return True---def veto_measured_defects(- rubric: Rubric, defects: dict[str, bool], gates: list[GateReport]-) -> tuple[dict[str, bool], list[str]]:- """``defects`` with every measured-absent one switched off, plus the ids switched off."""- overridden = [did for did, on in defects.items() if on and measured_absent(rubric, did, gates)]- return {did: (on and did not in overridden) for did, on in defects.items()}, overridden---def apply_caps(- rubric: Rubric,- overall: float,- gates: list[GateReport],- acceptance_results: dict[str, bool],- acceptance_items: list[AcceptanceItem] | None = None,- *,- console_errors: list[str] | None = None,- views: list[RenderView] | None = None,- defects_present: dict[str, bool] | None = None,-) -> CapResult:- """Apply every rubric cap rule; the overall becomes ``min(overall, caps...)``.-- Only the first matching finding per rule is recorded as evidence (one ledger- line per rule), but all rules are evaluated. ``defects_present`` (checklist- id → True) adds one ``defect:<id>`` ledger line per present defect that- carries a ``cap``.- """- applied: list[CapApplied] = []- for rule in rubric.caps:- evidence = _rule_evidence(- rule,- gates,- acceptance_results,- acceptance_items or [],- console_errors or [],- views or [],- )- if evidence is not None:- applied.append(CapApplied(rule=rule.id, cap=rule.cap, evidence=evidence))- for did, present in (defects_present or {}).items():- if not present:- continue- try:- item = rubric.defect(did)- except KeyError:- continue- if item.cap is not None:- applied.append(- CapApplied(- rule=f"defect:{did}",- cap=item.cap,- evidence=f"judge checklist: {item.text[:120]}",- )- )- capped = overall- for a in applied:- capped = min(capped, a.cap)- return CapResult(overall=max(0.0, min(1.0, capped)), caps_applied=applied)---def _rule_evidence(- rule: CapRule,- gates: list[GateReport],- acceptance_results: dict[str, bool],- acceptance_items: list[AcceptanceItem],- console_errors: list[str],- views: list[RenderView],-) -> str | None:- if rule.when == "missing_views":- tokens = [k.lower() for k in rule.kinds]- if any(any(t in f"{v.name} {v.mode}".lower() for t in tokens) for v in views):- return None- return f"no render view matching {rule.kinds} among {len(views)} view(s)"- if rule.when == "acceptance":- missing = missing_must_items(acceptance_items, acceptance_results)- if missing:- return f"must items not verified: {', '.join(missing[:6])}"- return None- if rule.when == "console":- if console_errors:- return f"{len(console_errors)} console error(s); first: {console_errors[0][:160]}"- return None- for report in gates:- for f in report.findings:- if _finding_matches(rule, report, f):- target = f" [{f.target}]" if f.target else ""- return f"{report.gate}{target}: {f.message[:200]}"- return Nonediff --git a/harness/codeverse/judges/images.py b/harness/codeverse/judges/images.pydeleted file mode 100644index 564b51b4..00000000--- a/harness/codeverse/judges/images.py+++ /dev/null@@ -1,107 +0,0 @@-"""Judge image preparation: downscale, label strips, caching, view geometry.--Every image sent to a judge goes through ``prepare_image`` so the token cost is-bounded (≤ ``max_px`` on the long side) and every view carries a burnt-in label-(``VIEW 3/8 — front · az 0° el 8°``) that the model can cite as evidence.-Prepared files are cached under ``<cache_dir>/<sha>.png`` keyed by source path,-mtime, size, max_px and label text.-"""--from __future__ import annotations--import hashlib-import math-import os-import threading-from pathlib import Path--from PIL import Image, ImageDraw--from codeverse.config import get_settings-from codeverse.contracts.artifacts import RenderView-from codeverse.contracts.chat import ImagePart-from codeverse.conventions import OBJECT_VIEWS, SCENE_VIEWS, ViewPreset-from codeverse.spatial.sheet import load_font--_PRESETS: dict[str, ViewPreset] = {v.name: v for v in (*OBJECT_VIEWS, *SCENE_VIEWS)}---class JudgeImageError(FileNotFoundError):- """A render referenced by a RenderSet is missing or unreadable."""---def default_cache_dir() -> Path:- return get_settings().cache_dir / "judge_images"---def view_az_el(view: RenderView) -> tuple[float, float] | None:- """Azimuth/elevation (deg) of a view: from the preset name, else from camera geometry (Y-up)."""- if view.name in _PRESETS:- p = _PRESETS[view.name]- return p.azimuth_deg, p.elevation_deg- if view.camera_position is not None:- cx, cy, cz = view.camera_position- lx, ly, lz = view.look_at or (0.0, 0.0, 0.0)- dx, dy, dz = cx - lx, cy - ly, cz - lz- horiz = math.hypot(dx, dz)- if horiz < 1e-9 and abs(dy) < 1e-9:- return None- az = math.degrees(math.atan2(dx, dz)) % 360.0 # 0 = +Z front, CCW from above- el = math.degrees(math.atan2(dy, horiz))- return round(az, 1), round(el, 1)- return None---def _cache_key(src: Path, max_px: int, label: str) -> str:- st = src.stat()- h = hashlib.sha1(f"{src.resolve()}|{st.st_mtime_ns}|{st.st_size}|{max_px}|{label}".encode())- return h.hexdigest()[:20]---def prepare_image(- src: str | Path, *, label: str = "", max_px: int = 768, cache_dir: Path | None = None-) -> Path:- """Downscale ``src`` to ≤ ``max_px`` on its long side and burn ``label`` into a top strip.-- Returns the cached PNG path. Raises ``JudgeImageError`` if the source is missing.- """- src = Path(src)- if not src.is_file():- raise JudgeImageError(f"judge image missing: {src}")- cache = Path(cache_dir) if cache_dir else default_cache_dir()- cache.mkdir(parents=True, exist_ok=True)- out = cache / f"{_cache_key(src, max_px, label)}.png"- if out.is_file():- return out- with Image.open(src) as im:- im = im.convert("RGB")- w, h = im.size- scale = min(1.0, max_px / max(w, h))- if scale < 1.0:- im = im.resize((max(1, round(w * scale)), max(1, round(h * scale))), Image.LANCZOS)- if label:- im = _with_label_strip(im, label)- tmp = out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png") # unique per writer: concurrent judges share the cache- im.save(tmp, format="PNG", optimize=True)- tmp.replace(out) # atomic; a concurrent identical write simply wins last- return out---def _with_label_strip(im: Image.Image, label: str) -> Image.Image:- w, h = im.size- strip_h = max(22, int(h * 0.055))- font = load_font(max(12, int(strip_h * 0.62)), bold=True)- canvas = Image.new("RGB", (w, h + strip_h), (18, 18, 22))- canvas.paste(im, (0, strip_h))- draw = ImageDraw.Draw(canvas)- text = label- # truncate to fit- while text and draw.textlength(text, font=font) > w - 12 and len(text) > 8:- text = text[:-2]- font_px = getattr(font, "size", 11)- draw.text((6, max(2, (strip_h - font_px) // 2)), text, fill=(245, 245, 240), font=font)- return canvas---def image_part(path: Path, label: str) -> ImagePart:- return ImagePart(path=str(path), mime="image/png", label=label)diff --git a/harness/codeverse/judges/montage.py b/harness/codeverse/judges/montage.pydeleted file mode 100644index 3e8ba01a..00000000--- a/harness/codeverse/judges/montage.py+++ /dev/null@@ -1,274 +0,0 @@-"""Judge montages: ≤ 2×2 labelled grids built from a ``RenderSet``.--Research finding applied here: VLM judges get position-biased and saturate when-shown many tiles, so every judge image carries **at most four views**. A judge-call sees, in priority order and capped at ``max_montages``:--1. ``shaded`` primary — the 4 most informative shaded views (3/4 views, top,- underside for objects; authored cameras for scenes);-2. ``poses`` — the articulation pose sheet (harness-made) or a 2×2 of ``pose_*``- views, when the render set has them;-3. ``geometry`` — a 2×2 of clay / normals renders (from ``geometry_views`` or from- non-shaded views inside the set) so holes, inverted faces and intersections- are visible without shading/material noise;-4. ``shaded`` secondary — the remaining shaded views (then a 2×2 of ``pose_*``- views when a pose sheet was already shown);--plus 0–2 ``detail`` crops (centre of the best 3/4 view at 2×; the ground-contact-band of the lowest view). ``shuffle_montages`` permutes montage order *and* tile-order per n-sample so judge noise from position bias averages out.-"""--from __future__ import annotations--import hashlib-import os-import random-import threading-from dataclasses import dataclass, replace-from pathlib import Path-from typing import Literal--from codeverse.contracts.artifacts import RenderSet, RenderView-from codeverse.conventions import SCENE_VIEWS-from codeverse.judges.images import JudgeImageError, default_cache_dir, view_az_el-from codeverse.spatial.sheet import crop_region, montage_2x2--MontageKind = Literal["shaded", "geometry", "poses", "pose_sheet", "detail"]--#: modes that show geometry without material/lighting noise-GEOMETRY_MODES = ("clay", "normals", "wire", "silhouette", "depth")-#: most-informative-first order for the object rig (names from conventions.OBJECT_VIEWS)-OBJECT_RANK = ("front_right_34", "back_left_34", "top", "low_front_left", "front", "right", "back", "left")-#: scene rig: authored cameras first (graded for composition), then the overview rig.-#: The rig names come from ``conventions.SCENE_VIEWS`` — anything else in a scene render-#: set is a camera the SCENE authored, whatever it is called (they are PascalCase plan-#: names such as ``Establishing``, never a fixed prefix).-SCENE_OVERVIEW_RANK = ("overview_front_right", "overview_back_left", "overview_top", "eye_front", "eye_right", "eye_back_left")-SCENE_RIG_NAMES = frozenset(v.name for v in SCENE_VIEWS) | frozenset(SCENE_OVERVIEW_RANK)-POSE_PREFIX = "pose_"-POSE_SHEET_NAMES = ("articulation_sheet", "pose_sheet")-_POSITIONS = {- 1: ("single",),- 2: ("left", "right"),- 3: ("top-left", "top-right", "bottom-left"),- 4: ("top-left", "top-right", "bottom-left", "bottom-right"),-}-MAX_TILES = 4---@dataclass(frozen=True)-class Montage:- """One judge image: 1–4 tiles (or a pass-through sheet / a detail crop)."""-- kind: MontageKind- tiles: tuple[RenderView, ...]- title: str- passthrough: bool = False- crop: tuple[float, float, float, float] | None = None- hint: str = ""-- @property- def is_detail(self) -> bool:- return self.kind == "detail"---# --------------------------------------------------------------------------- view grouping / ranking-def is_pose_view(v: RenderView) -> bool:- return v.name.startswith(POSE_PREFIX)---def is_pose_sheet(v: RenderView) -> bool:- return v.name in POSE_SHEET_NAMES---def is_geometry_view(v: RenderView) -> bool:- return v.mode in GEOMETRY_MODES---def rank_views(views: list[RenderView], *, scene: bool) -> list[RenderView]:- """Most informative first; unknown names keep their given order after the known ones."""- if scene:- def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:- i, v = iv- if v.name not in SCENE_RIG_NAMES:- return (0, 0, i) # the scene's OWN cameras: the pictures being graded- if v.name in SCENE_OVERVIEW_RANK:- return (1, SCENE_OVERVIEW_RANK.index(v.name), i)- return (2, 0, i)- else:- def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:- i, v = iv- return (0, OBJECT_RANK.index(v.name), i) if v.name in OBJECT_RANK else (1, 0, i)- return [v for _, v in sorted(enumerate(views), key=key)]---def _chunks(views: list[RenderView], n: int) -> list[list[RenderView]]:- return [views[i : i + n] for i in range(0, len(views), n)]---def plan_montages(- renders: RenderSet,- *,- geometry_views: RenderSet | None = None,- scene: bool = False,- max_montages: int = 3,- detail_crops: int = 2,-) -> list[Montage]:- """Decide which montages a judge call gets (see module docstring for the priority)."""- shaded = [v for v in renders.views if not (is_pose_view(v) or is_pose_sheet(v) or is_geometry_view(v))]- poses = [v for v in renders.views if is_pose_view(v)]- sheets = [v for v in renders.views if is_pose_sheet(v)]- geometry = [v for v in renders.views if is_geometry_view(v)]- if geometry_views is not None:- geometry = list(geometry_views.views) + geometry- shaded = rank_views(shaded, scene=scene)- shaded_groups = _chunks(shaded, MAX_TILES)-- candidates: list[Montage] = []- if shaded_groups:- candidates.append(Montage("shaded", tuple(shaded_groups[0]), "SHADED views"))- if sheets:- candidates.append(Montage("pose_sheet", (sheets[0],), "POSE SHEET (rows = poses, columns = views; tile labels name the joint and its value)", passthrough=True))- elif poses:- candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))- if geometry:- geo = _match_geometry(geometry, candidates[0].tiles if candidates else ())- candidates.append(Montage("geometry", tuple(geo), f"GEOMETRY-ONLY views ({geo[0].mode}: no materials — look for holes, black/inverted faces, intersections, gaps)"))- for extra in shaded_groups[1:]:- candidates.append(Montage("shaded", tuple(extra), "SHADED views (remaining)"))- if sheets and poses: # the sheet already shows the poses; the 2×2 is a larger-tile duplicate, lowest priority- candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))- chosen = candidates[: max(0, max_montages)]- chosen.extend(plan_detail_crops(shaded, scene=scene)[: max(0, detail_crops)])- return chosen---def _match_geometry(geometry: list[RenderView], primary: tuple[RenderView, ...]) -> list[RenderView]:- """Prefer geometry views whose names match the primary shaded montage, then the rest."""- names = [v.name for v in primary]- by_name = {v.name: v for v in geometry}- out = [by_name[n] for n in names if n in by_name]- out += [v for v in geometry if v not in out]- return out[:MAX_TILES]---def plan_detail_crops(shaded: list[RenderView], *, scene: bool) -> list[Montage]:- """Centre crop of the best view at 2× (detail legibility) + ground band of the lowest view (objects only)."""- if not shaded:- return []- out = [Montage("detail", (shaded[0],), f"DETAIL CROP — centre of {shaded[0].name} at 2×",- crop=(0.25, 0.25, 0.75, 0.75),- hint="look for bevels, seams, gaps and surface artefacts; judge detail at this scale")]- if scene:- return out- lowest = min(shaded, key=lambda v: (view_az_el(v) or (0.0, 90.0))[1])- if lowest is not shaded[0] or len(shaded) == 1:- out.append(Montage("detail", (lowest,), f"DETAIL CROP — ground-contact band of {lowest.name}",- crop=(0.0, 0.55, 1.0, 1.0),- hint="look for feet/legs that do not reach the ground and parts hovering above their support"))- return out---# --------------------------------------------------------------------------- shuffling-def shuffle_montages(montages: list[Montage], seed: int | None) -> list[Montage]:- """Permute montage order and the tile order inside each grid; detail crops stay last."""- if seed is None:- return list(montages)- rng = random.Random(seed)- grids = [m for m in montages if not m.is_detail]- details = [m for m in montages if m.is_detail]- rng.shuffle(grids)- out = []- for m in grids:- if m.passthrough or len(m.tiles) < 2:- out.append(m)- continue- tiles = list(m.tiles)- rng.shuffle(tiles)- out.append(replace(m, tiles=tuple(tiles)))- return out + details---# --------------------------------------------------------------------------- labels-def tile_label(v: RenderView) -> str:- bits = [v.name]- azel = view_az_el(v)- if azel is not None:- bits.append(f"az {azel[0]:.0f}° el {azel[1]:.0f}°")- if v.mode and v.mode != "shaded":- bits.append(v.mode)- if v.time_s is not None:- bits.append(f"t={v.time_s:g}s")- return " · ".join(bits)---def montage_strip(m: Montage, index: int, total: int) -> str:- """Short label burnt into the image's top strip (``MONTAGE 2/3 — SHADED views``)."""- if m.is_detail:- return f"{m.title} [{index}/{total}]"- title = m.title.split(" (")[0]- return f"MONTAGE {index}/{total} — {title}"---def montage_label(m: Montage, index: int, total: int) -> str:- """Full label for the text part: strip + tile layout (``…: top-left = front_right_34 · az 35° el 22°, …``)."""- head = montage_strip(m, index, total)- if m.is_detail:- return head + (f" — {m.hint}" if m.hint else "")- if m.passthrough:- return f"MONTAGE {index}/{total} — {m.title}"- pos = _POSITIONS[len(m.tiles)]- return f"MONTAGE {index}/{total} — {m.title}: " + ", ".join(f"{p} = {tile_label(v)}" for p, v in zip(pos, m.tiles, strict=True))---# --------------------------------------------------------------------------- rendering-def _key(parts: list[str]) -> str:- return hashlib.sha1("|".join(parts).encode()).hexdigest()[:20]---def _stat(p: str) -> str:- path = Path(p)- if not path.is_file():- raise JudgeImageError(f"judge image missing: {path}")- st = path.stat()- return f"{path.resolve()}|{st.st_mtime_ns}|{st.st_size}"---def _tmp(out: Path) -> Path:- return out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png")---def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int = 512) -> Path:- """Build (or fetch from cache) the PNG for one montage; pass-through sheets return their source.-- Writes go through a per-writer temp file + atomic replace so concurrent- judges sharing a cache directory never read a half-written PNG.- """- if m.passthrough:- _stat(m.tiles[0].path)- return Path(m.tiles[0].path)- cache = Path(cache_dir) if cache_dir else default_cache_dir()- cache.mkdir(parents=True, exist_ok=True)- if m.is_detail:- v = m.tiles[0]- out = cache / f"crop_{_key([_stat(v.path), str(m.crop), str(tile_px)])}.png"- if not out.is_file():- tmp = crop_region(v.path, _tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)- tmp.replace(out)- return out- images = [(tile_label(v), v.path) for v in m.tiles]- out = cache / f"montage_{_key([_stat(p) + '|' + lbl for lbl, p in images] + [str(tile_px)])}.png"- if not out.is_file():- tmp = montage_2x2(images, _tmp(out), tile=tile_px)- tmp.replace(out)- return out---def describe_montages(montages: list[Montage]) -> str:- """One line per image for the VIEW RIG paragraph (what each image is, in send order)."""- grids = [m for m in montages if not m.is_detail]- details = [m for m in montages if m.is_detail]- lines = [f"- image {i}: {m.title}" for i, m in enumerate(grids, 1)]- lines += [f"- detail crop {i}: {m.title.replace('DETAIL CROP — ', '')}" for i, m in enumerate(details, 1)]- return "\n".join(lines)diff --git a/harness/codeverse/judges/output_schema.py b/harness/codeverse/judges/output_schema.pydeleted file mode 100644index 79840f4a..00000000--- a/harness/codeverse/judges/output_schema.py+++ /dev/null@@ -1,216 +0,0 @@-"""The judge's wire format: a pydantic model built per rubric (+ acceptance ids).--The model forces the VLM to score EVERY visual criterion and to answer EVERY-acceptance item; ``overall`` is deliberately absent — it is computed in code.-Parsing is lenient about shape (dict-of-records or list-of-records, fenced-JSON in plain text) but strict about content: a missing criterion is a-``JudgeParseError`` so the caller can retry instead of scoring a hole as 0.-"""--from __future__ import annotations--from typing import Any--from pydantic import BaseModel, Field, ValidationError, create_model--from codeverse.contracts.judgment import ImprovementItem, JudgeIssue-from codeverse.judges.rubrics import Rubric-from codeverse.models.schema_utils import JsonParseError, parse_json_lenient---class JudgeParseError(ValueError):- """The model's reply could not be turned into a complete JudgeOutput."""---class CriterionScore(BaseModel):- score: float = Field(ge=0.0, le=1.0, description="0..1 against the anchors")- evidence: str = Field(description="what you saw, citing VIEW labels / measurements")---class AcceptanceVerdict(BaseModel):- verified: bool = Field(description="true only if the renders/measurements prove it")- evidence: str = Field(default="", description="which view / number proves or disproves it")---class DefectVerdict(BaseModel):- present: bool = Field(description="true only if the defect is VISIBLE in the images or stated by a gate/measurement")- evidence: str = Field(default="", description="which image / tile / number shows it (or why it is absent)")---class JudgeOutput(BaseModel):- """Provider-neutral parsed judge reply (criteria + acceptance + defect checklist as dicts)."""-- criteria: dict[str, CriterionScore]- summary: str = ""- strengths: list[str] = Field(default_factory=list)- issues: list[JudgeIssue] = Field(default_factory=list)- improvement_plan: list[ImprovementItem] = Field(default_factory=list)- acceptance: dict[str, AcceptanceVerdict] = Field(default_factory=dict)- defects: dict[str, bool] = Field(default_factory=dict, description="checklist id → defect present")- defect_evidence: dict[str, str] = Field(default_factory=dict)-- @property- def scores(self) -> dict[str, float]:- return {k: v.score for k, v in self.criteria.items()}-- @property- def acceptance_bools(self) -> dict[str, bool]:- return {k: v.verified for k, v in self.acceptance.items()}---def build_wire_model(rubric: Rubric, acceptance_ids: list[str]) -> type[BaseModel]:- """A pydantic model whose ``criteria`` / ``acceptance`` objects have FIXED keys.-- Criterion ids are valid identifiers (enforced by the rubric). Acceptance ids- are arbitrary strings → fields ``acc_<i>`` aliased to the id.- """- crit_fields: dict[str, Any] = {- c.id: (CriterionScore, Field(description=f"{c.label}: {c.description.strip()[:200]}"))- for c in rubric.visual_criteria()- }- Criteria = create_model("Criteria", **crit_fields)- # Field order = generation order for JSON-mode models: observe (summary, strengths, issues, defect- # checklist, acceptance) BEFORE scoring, so the scores follow the evidence rather than the reverse.- fields: dict[str, Any] = {- "summary": (str, Field(description="2-4 sentences: what it is, what is right, what is most wrong")),- "strengths": (list[str], Field(default_factory=list, description="≤ 5 short bullets")),- "issues": (list[JudgeIssue], Field(default_factory=list, description="observable defects, most severe first")),- }- if rubric.defects:- def_fields: dict[str, Any] = {- d.id: (DefectVerdict, Field(description=d.text.strip()[:200])) for d in rubric.defects- }- Defects = create_model("Defects", **def_fields)- fields["defects"] = (Defects, Field(description="binary checklist: one entry per defect id, present=true ONLY when seen"))- if acceptance_ids:- acc_fields: dict[str, Any] = {- f"acc_{i}": (AcceptanceVerdict, Field(alias=aid, description=f"acceptance item {aid}"))- for i, aid in enumerate(acceptance_ids)- }- Acceptance = create_model("Acceptance", **acc_fields)- fields["acceptance"] = (Acceptance, Field(description="one entry per acceptance item id"))- fields["criteria"] = (Criteria, Field(description="one entry per criterion id, scored AFTER the observations above"))- fields["improvement_plan"] = (- list[ImprovementItem],- Field(default_factory=list, description="≤ 6 concrete instructions for the builder, priority 1 first"),- )- return create_model("JudgeReply", **fields)---def wire_schema(rubric: Rubric, acceptance_ids: list[str]) -> dict[str, Any]:- return build_wire_model(rubric, acceptance_ids).model_json_schema(by_alias=True)---# --------------------------------------------------------------------------- parsing-def _records_to_dict(value: Any, key_names: tuple[str, ...]) -> dict[str, Any]:- """Accept ``{id: {...}}`` or ``[{id: ..., ...}, ...]`` and return ``{id: {...}}``."""- if isinstance(value, dict):- return value- if isinstance(value, list):- out: dict[str, Any] = {}- for rec in value:- if not isinstance(rec, dict):- raise JudgeParseError(f"record is not an object: {rec!r}")- key = next((rec[k] for k in key_names if k in rec), None)- if key is None:- raise JudgeParseError(f"record lacks an id field {key_names}: {rec!r}")- out[str(key)] = {k: v for k, v in rec.items() if k not in key_names}- return out- raise JudgeParseError(f"expected object or list, got {type(value).__name__}")---def _coerce_acceptance(value: Any) -> dict[str, Any]:- recs = _records_to_dict(value, ("id", "item_id", "item"))- out: dict[str, Any] = {}- for k, v in recs.items():- if isinstance(v, bool):- out[k] = {"verified": v, "evidence": ""}- elif isinstance(v, dict):- vv = dict(v)- if "verified" not in vv and "passed" in vv:- vv["verified"] = vv.pop("passed")- out[k] = vv- else:- raise JudgeParseError(f"acceptance item {k!r}: unexpected value {v!r}")- return out---def _coerce_defects(value: Any) -> tuple[dict[str, bool], dict[str, str]]:- """``{id: bool}`` or ``{id: {present, evidence}}`` or a list of records → (present, evidence)."""- if value is None:- return {}, {}- recs = _records_to_dict(value, ("id", "defect", "item"))- present: dict[str, bool] = {}- evidence: dict[str, str] = {}- for k, v in recs.items():- if isinstance(v, bool):- present[k] = v- elif isinstance(v, dict):- flag = v.get("present", v.get("value", v.get("verified")))- if not isinstance(flag, bool):- raise JudgeParseError(f"defect {k!r}: 'present' must be a boolean, got {flag!r}")- present[k] = flag- evidence[k] = str(v.get("evidence", "") or "")- else:- raise JudgeParseError(f"defect item {k!r}: unexpected value {v!r}")- return present, evidence---def parse_judge_output(- payload: Any, rubric: Rubric, acceptance_ids: list[str], *, measured_scores: dict[str, float] | None = None-) -> JudgeOutput:- """Validate a parsed reply (dict) or raw text into a complete ``JudgeOutput``.-- ``measured_scores`` (criterion id → score) are injected for ``kind: measured``- criteria so the output always covers the full rubric.- """- if isinstance(payload, str):- try:- payload = parse_json_lenient(payload)- except JsonParseError as e:- raise JudgeParseError(f"no JSON object in reply: {e}") from e- if not isinstance(payload, dict):- raise JudgeParseError(f"reply is not a JSON object: {type(payload).__name__}")- data = dict(payload)- if "criteria" not in data and "per_criterion" in data:- data["criteria"] = data.pop("per_criterion")- try:- criteria = _records_to_dict(data.get("criteria", {}), ("id", "criterion", "name"))- criteria = {- k: ({"score": v, "evidence": ""} if isinstance(v, (int, float)) else v) for k, v in criteria.items()- }- acceptance = _coerce_acceptance(data.get("acceptance", {}))- defects, defect_evidence = _coerce_defects(data.get("defects", {}))- out = JudgeOutput(- criteria=criteria,- summary=str(data.get("summary", "") or ""),- strengths=[str(s) for s in data.get("strengths", []) or []],- issues=data.get("issues", []) or [],- improvement_plan=data.get("improvement_plan", []) or [],- acceptance=acceptance,- defects=defects,- defect_evidence=defect_evidence,- )- except ValidationError as e:- raise JudgeParseError(f"judge reply failed validation: {e}") from e- measured = measured_scores or {}- for cid, s in measured.items():- out.criteria[cid] = CriterionScore(score=max(0.0, min(1.0, float(s))), evidence="measured in code")- missing = [c.id for c in rubric.criteria if c.id not in out.criteria]- if missing:- raise JudgeParseError(f"judge reply missing criteria {missing}")- extra = [k for k in out.criteria if k not in rubric.weights]- for k in extra: # unknown criteria are ignored, not scored- out.criteria.pop(k)- missing_acc = [a for a in acceptance_ids if a not in out.acceptance]- for a in missing_acc: # unanswered acceptance items count as NOT verified- out.acceptance[a] = AcceptanceVerdict(verified=False, evidence="not answered by judge")- known = {d.id for d in rubric.defects}- for k in [k for k in out.defects if k not in known]: # unknown checklist ids are ignored- out.defects.pop(k)- out.defect_evidence.pop(k, None)- for d in rubric.defects:- if d.id not in out.defects: # unanswered defect → absent (the conservative direction for a penalty)- out.defects[d.id] = False- out.defect_evidence[d.id] = "not answered by judge"- return outdiff --git a/harness/codeverse/judges/pairwise.py b/harness/codeverse/judges/pairwise.pyindex 4ee50688..3290c47a 100644--- a/harness/codeverse/judges/pairwise.py+++ b/harness/codeverse/judges/pairwise.py@@ -27,11 +27,16 @@ from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart, TextPafrom codeverse.contracts.common import Usagefrom codeverse.contracts.spec import Specfrom codeverse.fanout import fan_out-from codeverse.judges.images import image_part, prepare_image-from codeverse.judges.montage import montage_label, plan_montages, render_montage-from codeverse.judges.output_schema import JudgeParseError-from codeverse.judges.prompt_builder import MONTAGE_TILE_PX, brief_section-from codeverse.judges.rubrics import Rubric, load_rubric+from codeverse.judges.prompt_builder import (+ MONTAGE_TILE_PX,+ brief_section,+ image_part,+ montage_label,+ plan_montages,+ prepare_image,+ render_montage,+)+from codeverse.judges.rubrics import JudgeParseError, Rubric, load_rubricfrom codeverse.models.base import ChatModel, ModelErrorfrom codeverse.models.schema_utils import JsonParseError, parse_json_lenientdiff --git a/harness/codeverse/judges/prompt_builder.py b/harness/codeverse/judges/prompt_builder.pyindex 5ae63081..14f3aa3c 100644--- a/harness/codeverse/judges/prompt_builder.py+++ b/harness/codeverse/judges/prompt_builder.py@@ -1,38 +1,364 @@-"""Build the judge conversation: system (role + rubric + defect checklist +-output rules) and ONE user message (brief, plan digest, acceptance checklist,-measurement table, gate digest, previous-verdict framing, view-rig paragraph,-then the labelled images).--Images are ≤ 2×2 **montages** (see ``montage.py``): at most ``max_montages``-grids (shaded primary, pose sheet, geometry-only, shaded secondary) plus 0–2-detail crops, every tile labelled, whole-image label strips burnt in. Text is-budgeted to ≈6k tokens. The builder's code and reasoning are never included-(blind judge).+"""Judge image preparation: downscale, label strips, caching, view geometry.++Every image sent to a judge goes through ``prepare_image`` so the token cost is+bounded (≤ ``max_px`` on the long side) and every view carries a burnt-in label+(``VIEW 3/8 — front · az 0° el 8°``) that the model can cite as evidence.+Prepared files are cached under ``<cache_dir>/<sha>.png`` keyed by source path,+mtime, size, max_px and label text."""from __future__ import annotations+import hashlib+import math+import os+import random+import threading+from dataclasses import dataclass, replacefrom pathlib import Path-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Literal++from PIL import Image, ImageDraw-from codeverse.contracts.artifacts import GateReport, Measurement, RenderSet, Severity+from codeverse.config import get_settings+from codeverse.contracts.artifacts import GateReport, Measurement, RenderSet, RenderView, Severityfrom codeverse.contracts.chat import ChatMessage, ImagePart, TextPartfrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.spec import Spec-from codeverse.judges.images import image_part, prepare_image-from codeverse.judges.montage import (- Montage,- describe_montages,- montage_label,- montage_strip,- plan_montages,- render_montage,- shuffle_montages,-)+from codeverse.conventions import OBJECT_VIEWS, SCENE_VIEWS, ViewPresetfrom codeverse.judges.rubrics import Rubricfrom codeverse.spatial.measure import measure_summary_table+from codeverse.spatial.sheet import crop_region, load_font, montage_2x2++_PRESETS: dict[str, ViewPreset] = {v.name: v for v in (*OBJECT_VIEWS, *SCENE_VIEWS)}+++class JudgeImageError(FileNotFoundError):+ """A render referenced by a RenderSet is missing or unreadable."""+++def default_cache_dir() -> Path:+ return get_settings().cache_dir / "judge_images"+++def view_az_el(view: RenderView) -> tuple[float, float] | None:+ """Azimuth/elevation (deg) of a view: from the preset name, else from camera geometry (Y-up)."""+ if view.name in _PRESETS:+ p = _PRESETS[view.name]+ return p.azimuth_deg, p.elevation_deg+ if view.camera_position is not None:+ cx, cy, cz = view.camera_position+ lx, ly, lz = view.look_at or (0.0, 0.0, 0.0)+ dx, dy, dz = cx - lx, cy - ly, cz - lz+ horiz = math.hypot(dx, dz)+ if horiz < 1e-9 and abs(dy) < 1e-9:+ return None+ az = math.degrees(math.atan2(dx, dz)) % 360.0 # 0 = +Z front, CCW from above+ el = math.degrees(math.atan2(dy, horiz))+ return round(az, 1), round(el, 1)+ return None+++def _cache_key(src: Path, max_px: int, label: str) -> str:+ st = src.stat()+ h = hashlib.sha1(f"{src.resolve()}|{st.st_mtime_ns}|{st.st_size}|{max_px}|{label}".encode())+ return h.hexdigest()[:20]+++def prepare_image(+ src: str | Path, *, label: str = "", max_px: int = 768, cache_dir: Path | None = None+) -> Path:+ """Downscale ``src`` to ≤ ``max_px`` on its long side and burn ``label`` into a top strip.++ Returns the cached PNG path. Raises ``JudgeImageError`` if the source is missing.+ """+ src = Path(src)+ if not src.is_file():+ raise JudgeImageError(f"judge image missing: {src}")+ cache = Path(cache_dir) if cache_dir else default_cache_dir()+ cache.mkdir(parents=True, exist_ok=True)+ out = cache / f"{_cache_key(src, max_px, label)}.png"+ if out.is_file():+ return out+ with Image.open(src) as im:+ im = im.convert("RGB")+ w, h = im.size+ scale = min(1.0, max_px / max(w, h))+ if scale < 1.0:+ im = im.resize((max(1, round(w * scale)), max(1, round(h * scale))), Image.LANCZOS)+ if label:+ im = _with_label_strip(im, label)+ tmp = out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png") # unique per writer: concurrent judges share the cache+ im.save(tmp, format="PNG", optimize=True)+ tmp.replace(out) # atomic; a concurrent identical write simply wins last+ return out+++def _with_label_strip(im: Image.Image, label: str) -> Image.Image:+ w, h = im.size+ strip_h = max(22, int(h * 0.055))+ font = load_font(max(12, int(strip_h * 0.62)), bold=True)+ canvas = Image.new("RGB", (w, h + strip_h), (18, 18, 22))+ canvas.paste(im, (0, strip_h))+ draw = ImageDraw.Draw(canvas)+ text = label+ # truncate to fit+ while text and draw.textlength(text, font=font) > w - 12 and len(text) > 8:+ text = text[:-2]+ font_px = getattr(font, "size", 11)+ draw.text((6, max(2, (strip_h - font_px) // 2)), text, fill=(245, 245, 240), font=font)+ return canvas+++def image_part(path: Path, label: str) -> ImagePart:+ return ImagePart(path=str(path), mime="image/png", label=label)+++# ===================================================================== montage+# (merged from codeverse/judges/montage.py, 2026-08-28)+MontageKind = Literal["shaded", "geometry", "poses", "pose_sheet", "detail"]++#: modes that show geometry without material/lighting noise+GEOMETRY_MODES = ("clay", "normals", "wire", "silhouette", "depth")+#: most-informative-first order for the object rig (names from conventions.OBJECT_VIEWS)+OBJECT_RANK = ("front_right_34", "back_left_34", "top", "low_front_left", "front", "right", "back", "left")+#: scene rig: authored cameras first (graded for composition), then the overview rig.+#: The rig names come from ``conventions.SCENE_VIEWS`` — anything else in a scene render+#: set is a camera the SCENE authored, whatever it is called (they are PascalCase plan+#: names such as ``Establishing``, never a fixed prefix).+SCENE_OVERVIEW_RANK = ("overview_front_right", "overview_back_left", "overview_top", "eye_front", "eye_right", "eye_back_left")+SCENE_RIG_NAMES = frozenset(v.name for v in SCENE_VIEWS) | frozenset(SCENE_OVERVIEW_RANK)+POSE_PREFIX = "pose_"+POSE_SHEET_NAMES = ("articulation_sheet", "pose_sheet")+_POSITIONS = {+ 1: ("single",),+ 2: ("left", "right"),+ 3: ("top-left", "top-right", "bottom-left"),+ 4: ("top-left", "top-right", "bottom-left", "bottom-right"),+}+MAX_TILES = 4+++@dataclass(frozen=True)+class Montage:+ """One judge image: 1–4 tiles (or a pass-through sheet / a detail crop)."""++ kind: MontageKind+ tiles: tuple[RenderView, ...]+ title: str+ passthrough: bool = False+ crop: tuple[float, float, float, float] | None = None+ hint: str = ""++ @property+ def is_detail(self) -> bool:+ return self.kind == "detail"+++# --------------------------------------------------------------------------- view grouping / ranking+def is_pose_view(v: RenderView) -> bool:+ return v.name.startswith(POSE_PREFIX)+++def is_pose_sheet(v: RenderView) -> bool:+ return v.name in POSE_SHEET_NAMES+++def is_geometry_view(v: RenderView) -> bool:+ return v.mode in GEOMETRY_MODES+++def rank_views(views: list[RenderView], *, scene: bool) -> list[RenderView]:+ """Most informative first; unknown names keep their given order after the known ones."""+ if scene:+ def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:+ i, v = iv+ if v.name not in SCENE_RIG_NAMES:+ return (0, 0, i) # the scene's OWN cameras: the pictures being graded+ if v.name in SCENE_OVERVIEW_RANK:+ return (1, SCENE_OVERVIEW_RANK.index(v.name), i)+ return (2, 0, i)+ else:+ def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:+ i, v = iv+ return (0, OBJECT_RANK.index(v.name), i) if v.name in OBJECT_RANK else (1, 0, i)+ return [v for _, v in sorted(enumerate(views), key=key)]+++def _chunks(views: list[RenderView], n: int) -> list[list[RenderView]]:+ return [views[i : i + n] for i in range(0, len(views), n)]+++def plan_montages(+ renders: RenderSet,+ *,+ geometry_views: RenderSet | None = None,+ scene: bool = False,+ max_montages: int = 3,+ detail_crops: int = 2,+) -> list[Montage]:+ """Decide which montages a judge call gets (see module docstring for the priority)."""+ shaded = [v for v in renders.views if not (is_pose_view(v) or is_pose_sheet(v) or is_geometry_view(v))]+ poses = [v for v in renders.views if is_pose_view(v)]+ sheets = [v for v in renders.views if is_pose_sheet(v)]+ geometry = [v for v in renders.views if is_geometry_view(v)]+ if geometry_views is not None:+ geometry = list(geometry_views.views) + geometry+ shaded = rank_views(shaded, scene=scene)+ shaded_groups = _chunks(shaded, MAX_TILES)++ candidates: list[Montage] = []+ if shaded_groups:+ candidates.append(Montage("shaded", tuple(shaded_groups[0]), "SHADED views"))+ if sheets:+ candidates.append(Montage("pose_sheet", (sheets[0],), "POSE SHEET (rows = poses, columns = views; tile labels name the joint and its value)", passthrough=True))+ elif poses:+ candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))+ if geometry:+ geo = _match_geometry(geometry, candidates[0].tiles if candidates else ())+ candidates.append(Montage("geometry", tuple(geo), f"GEOMETRY-ONLY views ({geo[0].mode}: no materials — look for holes, black/inverted faces, intersections, gaps)"))+ for extra in shaded_groups[1:]:+ candidates.append(Montage("shaded", tuple(extra), "SHADED views (remaining)"))+ if sheets and poses: # the sheet already shows the poses; the 2×2 is a larger-tile duplicate, lowest priority+ candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))+ chosen = candidates[: max(0, max_montages)]+ chosen.extend(plan_detail_crops(shaded, scene=scene)[: max(0, detail_crops)])+ return chosen+++def _match_geometry(geometry: list[RenderView], primary: tuple[RenderView, ...]) -> list[RenderView]:+ """Prefer geometry views whose names match the primary shaded montage, then the rest."""+ names = [v.name for v in primary]+ by_name = {v.name: v for v in geometry}+ out = [by_name[n] for n in names if n in by_name]+ out += [v for v in geometry if v not in out]+ return out[:MAX_TILES]+++def plan_detail_crops(shaded: list[RenderView], *, scene: bool) -> list[Montage]:+ """Centre crop of the best view at 2× (detail legibility) + ground band of the lowest view (objects only)."""+ if not shaded:+ return []+ out = [Montage("detail", (shaded[0],), f"DETAIL CROP — centre of {shaded[0].name} at 2×",+ crop=(0.25, 0.25, 0.75, 0.75),+ hint="look for bevels, seams, gaps and surface artefacts; judge detail at this scale")]+ if scene:+ return out+ lowest = min(shaded, key=lambda v: (view_az_el(v) or (0.0, 90.0))[1])+ if lowest is not shaded[0] or len(shaded) == 1:+ out.append(Montage("detail", (lowest,), f"DETAIL CROP — ground-contact band of {lowest.name}",+ crop=(0.0, 0.55, 1.0, 1.0),+ hint="look for feet/legs that do not reach the ground and parts hovering above their support"))+ return out+++# --------------------------------------------------------------------------- shuffling+def shuffle_montages(montages: list[Montage], seed: int | None) -> list[Montage]:+ """Permute montage order and the tile order inside each grid; detail crops stay last."""+ if seed is None:+ return list(montages)+ rng = random.Random(seed)+ grids = [m for m in montages if not m.is_detail]+ details = [m for m in montages if m.is_detail]+ rng.shuffle(grids)+ out = []+ for m in grids:+ if m.passthrough or len(m.tiles) < 2:+ out.append(m)+ continue+ tiles = list(m.tiles)+ rng.shuffle(tiles)+ out.append(replace(m, tiles=tuple(tiles)))+ return out + details+++# --------------------------------------------------------------------------- labels+def tile_label(v: RenderView) -> str:+ bits = [v.name]+ azel = view_az_el(v)+ if azel is not None:+ bits.append(f"az {azel[0]:.0f}° el {azel[1]:.0f}°")+ if v.mode and v.mode != "shaded":+ bits.append(v.mode)+ if v.time_s is not None:+ bits.append(f"t={v.time_s:g}s")+ return " · ".join(bits)+++def montage_strip(m: Montage, index: int, total: int) -> str:+ """Short label burnt into the image's top strip (``MONTAGE 2/3 — SHADED views``)."""+ if m.is_detail:+ return f"{m.title} [{index}/{total}]"+ title = m.title.split(" (")[0]+ return f"MONTAGE {index}/{total} — {title}"+++def montage_label(m: Montage, index: int, total: int) -> str:+ """Full label for the text part: strip + tile layout (``…: top-left = front_right_34 · az 35° el 22°, …``)."""+ head = montage_strip(m, index, total)+ if m.is_detail:+ return head + (f" — {m.hint}" if m.hint else "")+ if m.passthrough:+ return f"MONTAGE {index}/{total} — {m.title}"+ pos = _POSITIONS[len(m.tiles)]+ return f"MONTAGE {index}/{total} — {m.title}: " + ", ".join(f"{p} = {tile_label(v)}" for p, v in zip(pos, m.tiles, strict=True))+++# --------------------------------------------------------------------------- rendering+def _key(parts: list[str]) -> str:+ return hashlib.sha1("|".join(parts).encode()).hexdigest()[:20]+++def _stat(p: str) -> str:+ path = Path(p)+ if not path.is_file():+ raise JudgeImageError(f"judge image missing: {path}")+ st = path.stat()+ return f"{path.resolve()}|{st.st_mtime_ns}|{st.st_size}"+++def _tmp(out: Path) -> Path:+ return out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png")+++def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int = 512) -> Path:+ """Build (or fetch from cache) the PNG for one montage; pass-through sheets return their source.++ Writes go through a per-writer temp file + atomic replace so concurrent+ judges sharing a cache directory never read a half-written PNG.+ """+ if m.passthrough:+ _stat(m.tiles[0].path)+ return Path(m.tiles[0].path)+ cache = Path(cache_dir) if cache_dir else default_cache_dir()+ cache.mkdir(parents=True, exist_ok=True)+ if m.is_detail:+ v = m.tiles[0]+ out = cache / f"crop_{_key([_stat(v.path), str(m.crop), str(tile_px)])}.png"+ if not out.is_file():+ tmp = crop_region(v.path, _tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)+ tmp.replace(out)+ return out+ images = [(tile_label(v), v.path) for v in m.tiles]+ out = cache / f"montage_{_key([_stat(p) + '|' + lbl for lbl, p in images] + [str(tile_px)])}.png"+ if not out.is_file():+ tmp = montage_2x2(images, _tmp(out), tile=tile_px)+ tmp.replace(out)+ return out+++def describe_montages(montages: list[Montage]) -> str:+ """One line per image for the VIEW RIG paragraph (what each image is, in send order)."""+ grids = [m for m in montages if not m.is_detail]+ details = [m for m in montages if m.is_detail]+ lines = [f"- image {i}: {m.title}" for i, m in enumerate(grids, 1)]+ lines += [f"- detail crop {i}: {m.title.replace('DETAIL CROP — ', '')}" for i, m in enumerate(details, 1)]+ return "\n".join(lines)++# ===================================================================== prompt_builder+# (merged from codeverse/judges/prompt_builder.py, 2026-08-28)if TYPE_CHECKING: # pragma: no coverfrom codeverse.judges.base import JudgeInput@@ -248,7 +574,7 @@ def judge_prompt_hash(rubric: Rubric) -> str:"""import json- from codeverse.judges.output_schema import wire_schema+ from codeverse.judges.rubrics import wire_schemafrom codeverse.prompts import prompt_hashpayload = "\n".join([diff --git a/harness/codeverse/judges/reference.py b/harness/codeverse/judges/reference.pydeleted file mode 100644index 8c41bb51..00000000--- a/harness/codeverse/judges/reference.py+++ /dev/null@@ -1,358 +0,0 @@-"""``ReferenceJudge``: image-conditioned judging (rubric ``reference_v1``).--Three things the base judge does not do:--* the spec's reference images go in BEFORE the renders, labelled ``REFERENCE k``- (and labelled **SYNTHESIZED** when they were generated by- :mod:`codeverse.reference`, so the judge knows the target is not ground truth);-* ``silhouette_match`` is scored IN CODE from the front-render-vs-reference- outline IoU (:func:`codeverse.spatial.silhouette.compare_silhouette`) — the VLM- only scores the perceptual criteria;-* one extra vision pass (:func:`codeverse.reference.mismatch.compare`) names the- CONCRETE differences — missing feature, wrong part count, wrong proportion —- and puts them in front of the scoring call, so the verdict's issues and- improvement plan (and therefore the next round's refine tasks) are specific- instead of "looks blocky". Set ``diff=False`` (or ``CV3D_REFERENCE_DIFF=off``)- to skip that call; it is fail-soft either way.-"""--from __future__ import annotations--import logging-import os-import re-from collections.abc import Callable-from pathlib import Path-from typing import Any--from codeverse.contracts.artifacts import RenderView-from codeverse.judges.base import JudgeInput-from codeverse.judges.rubrics import Rubric-from codeverse.judges.vlm_judge import JudgeContext, VlmJudge--log = logging.getLogger(__name__)--SilhouetteFn = Callable[[str, str], dict[str, Any]]--#: IoU → score mapping: ≤ 0.25 → 0, ≥ 0.85 → 1, linear in between (matches the rubric anchors).-IOU_LOW, IOU_HIGH = 0.25, 0.85--#: score the measured criterion gets when the measurement cannot be trusted-#: (unreliable mask, or a reference whose proportions contradict the brief)-NEUTRAL_SCORE = 0.5--#: views handed to the mismatch pass (front-ish first, then a 3/4 and a side)-DIFF_VIEW_NAMES: tuple[str, ...] = ("front", "front_right_34", "front_left_34", "right", "left")-_ENV_DIFF = "CV3D_REFERENCE_DIFF"---def iou_to_score(iou: float) -> float:- return max(0.0, min(1.0, (float(iou) - IOU_LOW) / (IOU_HIGH - IOU_LOW)))---def _default_silhouette_fn() -> SilhouetteFn | None:- try:- from codeverse.spatial.silhouette import compare_silhouette # type: ignore[attr-defined]- except (ImportError, AttributeError):- return None- return compare_silhouette---def _diff_enabled(flag: bool) -> bool:- return flag and os.environ.get(_ENV_DIFF, "on").strip().lower() not in ("0", "off", "false", "no")---def _conflict_note(info: dict[str, Any]) -> str:- try:- from codeverse.reference import conflict_note-- return conflict_note(info)- except ImportError: # pragma: no cover- return ""---def _is_synth(note: str) -> bool:- return note.startswith("SYNTHESIZED")---class ReferenceJudge(VlmJudge):- """VlmJudge + reference images + measured silhouette + named mismatches."""-- name = "reference"-- def __init__(- self,- model_id: str | None = None,- n_samples: int = 1,- temperature: float = 0.2,- *,- rubric: str | Rubric = "reference_v1",- silhouette_fn: SilhouetteFn | None = None,- front_view_names: tuple[str, ...] = ("front", "front_right_34"),- best_view: bool = True,- diff: bool = True,- diff_model: Any | None = None,- diff_model_id: str = "",- **kwargs: Any,- ):- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)- self.silhouette_fn = silhouette_fn or _default_silhouette_fn()- self.front_view_names = front_view_names- self.best_view = best_view- self.diff = diff- self._diff_model = diff_model- self.diff_model_id = diff_model_id- #: the last diff computed (tests / callers that want the mismatches themselves)- self.last_diff: Any | None = None-- # ------------------------------------------------------------------ API- def judge(self, inp: JudgeInput, **kwargs: Any) -> Any:- """``VlmJudge.judge`` plus the mismatch pass's own spend.-- The diff is a real model call made from :meth:`context`; folding its- ``Usage`` into the verdict keeps ``BudgetGuard`` and ``record.total_usage``- honest (the ledger already sees it through the metered chat model)."""- verdict = super().judge(inp, **kwargs)- diff = self.last_diff- if diff is not None and getattr(diff, "usage", None) is not None:- verdict.usage = verdict.usage + diff.usage- return verdict-- # ------------------------------------------------------------------ hook- def context(self, inp: JudgeInput) -> JudgeContext:- refs = [r for r in inp.spec.references if Path(r.path).is_file()]- synth = bool(refs) and all(_is_synth(r.note) for r in refs)- images: list[tuple[str, str]] = []- for i, r in enumerate(refs[:3], 1):- tag = "SYNTHESIZED " if _is_synth(r.note) else ""- note = f" — {r.note}" if r.note else ""- images.append((f"{tag}REFERENCE {i}/{min(len(refs), 3)} ({r.role}){note}", r.path))- blocks: list[str] = []- if synth:- blocks.append(- "NOTE ON THE REFERENCE: it was SYNTHESIZED from the brief by an image model, not photographed. "- "It is a shape/part-inventory target only. Where it disagrees with the brief or the stated "- "dimensions, the BRIEF is correct and the render must follow the brief.")- conflict = self.dimension_conflict(inp, refs)- note = _conflict_note(conflict)- if note:- blocks.append(note)- measured_ids = [c.id for c in self.rubric.measured_criteria()]- info: dict[str, Any] = {}- measured_scores: dict[str, float] = {}- if measured_ids:- info = self.measure_silhouette(inp)- if "iou" not in info:- raise ReferenceJudgeError(- f"cannot score measured criteria {measured_ids}: {info.get('error', 'no iou')}")- score, why = self.silhouette_score(info, conflict)- measured_scores = {cid: score for cid in measured_ids}- blocks.append(- f"MEASURED SILHOUETTE (harness): {info['render']} render vs reference {info['reference']}: "- f"IoU {info['iou']:.3f} → silhouette_match score {score:.2f}{why}."- + (f" extra: {info['extra']}" if info.get("extra") else ""))- elif refs:- info = self.measure_silhouette(inp)- diff = self.reference_diff(inp, refs, info, synthesized=synth) if refs else None- self.last_diff = diff- if diff is not None and diff.as_text():- blocks.append(diff.as_text())- return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images)-- # ------------------------------------------------------------------ proportion guard- def dimension_conflict(self, inp: JudgeInput, refs: list[Any]) -> dict[str, Any]:- """Does the target reference's own outline contradict the brief's dimensions?-- When it does, no numeric proportion signal derived from that picture may be- used against the object — the brief wins (see- :mod:`codeverse.reference.proportions`).- """- targets = [r for r in refs if r.role == "target"] or refs- if not targets:- return {"conflict": False}- try:- from codeverse.reference import dimension_conflict-- return dimension_conflict(inp.spec, targets[0].path)- except Exception as e: # noqa: BLE001 — advisory- log.warning("dimension-conflict check failed: %s", e)- return {"conflict": False, "error": str(e)}-- def silhouette_score(self, info: dict[str, Any], conflict: dict[str, Any]) -> tuple[float, str]:- """IoU → measured score, NEUTRAL when the reference contradicts the brief or the- mask is unreliable. Returns ``(score, ' (why)')``."""- if conflict.get("conflict"):- return NEUTRAL_SCORE, " (NEUTRAL: the reference's proportions contradict the brief's dimensions)"- if info.get("reliable") is False:- return NEUTRAL_SCORE, " (NEUTRAL: the background mask was unreliable)"- return iou_to_score(info["iou"]), ""-- # ------------------------------------------------------------------ mismatch pass- def reference_diff(self, inp: JudgeInput, refs: list[Any], info: dict[str, Any], *, synthesized: bool) -> Any | None:- """One extra vision call naming concrete mismatches. ``None`` when disabled;- never raises (a failed diff simply contributes no text)."""- if not _diff_enabled(self.diff):- return None- try:- from codeverse.reference import compare- except ImportError: # pragma: no cover - the package is part of the wheel- return None- targets = [r.path for r in refs if r.role == "target"] or [r.path for r in refs]- renders = self.diff_views(list(inp.renders.views))- measured = dict(info) if "iou" in info else None- if measured is not None:- measured.setdefault("view", info.get("render", ""))- try:- return compare(inp.spec, targets, renders, model=self._pick_diff_model(),- part_names=_plan_part_names(inp), measured=measured, synthesized=synthesized)- except Exception as e: # noqa: BLE001 — an advisory pass must never fail a verdict- log.warning("reference diff pass failed: %s", e)- return None-- def _pick_diff_model(self) -> Any:- """``diff_model`` > ``diff_model_id`` (a cheaper perception model) > the judge's own model."""- if self._diff_model is not None:- return self._diff_model- if self.diff_model_id:- from codeverse.models import get_chat_model-- self._diff_model = get_chat_model(self.diff_model_id)- return self._diff_model- return self.model-- def diff_views(self, views: list[RenderView]) -> list[str]:- """Up to 3 shaded views for the diff, front-ish first."""- by_name = {v.name: v for v in views}- picked = [by_name[n].path for n in DIFF_VIEW_NAMES if n in by_name]- for v in views:- if len(picked) >= 3:- break- if v.path not in picked:- picked.append(v.path)- return picked[:3]-- # ------------------------------------------------------------------ silhouette- def pick_front_view(self, views: list[RenderView]) -> RenderView | None:- for name in self.front_view_names:- for v in views:- if v.name == name:- return v- return views[0] if views else None-- def measure_silhouette(self, inp: JudgeInput) -> dict[str, Any]:- """{iou, render, reference, extra} or {error}.-- With ``best_view=True`` (default) the IoU is taken from the render view that- best matches the reference's camera instead of a hardcoded ``front``: a- reference photograph has ONE camera, and comparing a three-quarter product- shot to a straight-on elevation measures camera agreement, not shape. The- winning view name travels in ``render`` so the number stays auditable.- """- targets = [r for r in inp.spec.references if r.role == "target" and Path(r.path).is_file()] or [- r for r in inp.spec.references if Path(r.path).is_file()]- if not targets:- return {"error": "spec has no readable reference images"}- views = list(inp.renders.views)- if not views:- return {"error": "render set has no views"}- if self.silhouette_fn is None:- return {"error": "compare_silhouette unavailable (codeverse.spatial.silhouette not importable)"}- res, name = self._measure(views, targets[0].path)- if not isinstance(res, dict) or "iou" not in res:- return {"error": f"compare_silhouette returned no iou: {res!r}"}- extra = {k: v for k, v in res.items() if k not in ("iou", "per_view") and isinstance(v, (int, float, str, bool))}- out = {"iou": float(res["iou"]), "render": name, "reference": Path(targets[0].path).name,- "reliable": bool(res.get("reliable", True)),- "aspect_ratio_err": res.get("aspect_ratio_err"), "extra": extra}- if res.get("per_view"):- out["per_view"] = res["per_view"]- return out-- def _measure(self, views: list[RenderView], reference: str) -> tuple[dict[str, Any], str]:- """(result, view name). Best-matching view when enabled and available."""- if self.best_view and self.silhouette_fn is _default_silhouette_fn():- try:- from codeverse.spatial.silhouette import best_view_match- except ImportError: # pragma: no cover- best_view_match = None # type: ignore[assignment]- if best_view_match is not None:- res = best_view_match(views, reference)- if "iou" in res:- return res, str(res.get("view", ""))- view = self.pick_front_view(views)- if view is None: # pragma: no cover - guarded by the caller- return {}, ""- return self.silhouette_fn(view.path, reference), view.name # type: ignore[misc]---def _plan_part_names(inp: JudgeInput) -> list[str]:- """Part names from the plan digest the judge already receives (best effort).-- ``judges.replay_input.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;- a multi-line digest lists one ``- Name · role · …`` per line. Both are handled, and- an unrecognised digest simply yields no names (the diff prompt then says so).- """- text = getattr(inp, "plan_summary", "") or ""- names: list[str] = []- for chunk in re.split(r"(?i)\bparts\s*:", text)[1:]:- for raw in chunk.split("\n")[0].split(","):- names.append(raw.split("\u00d7")[0].split("(")[0].strip().rstrip("."))- for line in text.splitlines():- line = line.strip()- if not line.startswith(("-", "*", "\u00b7")):- continue- head = line.lstrip("-*\u00b7 ").split("\u00b7")[0].split("(")[0].split(":")[0].strip()- names.append(head)- seen: list[str] = []- for n in names:- if n and " " not in n and 2 < len(n) <= 40 and n[:1].isupper() and n not in seen:- seen.append(n)- return seen[:40]----LIKENESS_NOTE = (- "REAL-WORLD REFERENCE PHOTOS are attached (labelled REAL-WORLD REFERENCE n/N). They are not a "- "composition to copy; they are what the thing the brief names ACTUALLY looks like. Score "- "brief fidelity, colour/light and richness against them: dominant colour and where the "- "secondary colours sit, how the structure folds / layers / thins out, where the brightness "- "concentrates and how much of the frame stays dark, what the real texture is at fine scale. A "- "frame that ticks every noun in the brief but would never be mistaken for the photographed "- "thing (a row of evenly spaced bars standing in for curtains, a flat band standing in for a "- "gradient of light, cartoon-saturated colour where the photo is subtle) is a 0.4-0.5 on brief "- "fidelity, not a 1.0. Foreground, framing and landscape in the photos are incidental unless "- "the brief asks for them."-)---class LikenessJudge(VlmJudge):- """``VlmJudge`` + the reference photos of the REAL thing — for tracks with no silhouette.-- ``ReferenceJudge`` is built for objects: it measures a front-view silhouette IoU against- the target photo and runs a part-inventory mismatch pass. Neither means anything for a- fragment shader or a scene, where a reference photo answers a different question — does- this LOOK like the thing? Measured 2026-08-26 (teaser aurora, three versions, flash and- codex): every version scored 0.78-0.94 with an empty issues list while none resembled an- aurora (a comb of straight teal bars; pink cotton-wool lobes), because the brief is a- checklist of nouns and the rubric had nothing to say about likeness. This judge attaches- up to three photos beside the frames with :data:`LIKENESS_NOTE` and leaves the rubric alone.- """-- name = "likeness"-- def __init__(self, model_id: str | None = None, n_samples: int = 1, temperature: float = 0.2, *,- rubric: str | Rubric = "shader_v2", max_refs: int = 3, **kwargs: Any):- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)- self.max_refs = max(1, int(max_refs))-- def context(self, inp: JudgeInput) -> JudgeContext:- if self.rubric.measured_criteria():- raise ValueError(f"rubric {self.rubric.name} has measured criteria; LikenessJudge measures nothing")- refs = [r for r in inp.spec.references if Path(r.path).is_file()][: self.max_refs]- images = [(f"REAL-WORLD REFERENCE {i}/{len(refs)}" + (f" — {r.note}" if r.note else ""), r.path)- for i, r in enumerate(refs, 1)]- return JudgeContext(extra_text=LIKENESS_NOTE if refs else "", extra_images=images)---class ReferenceJudgeError(RuntimeError):- """The reference judge cannot compute its measured criteria."""diff --git a/harness/codeverse/judges/replay_input.py b/harness/codeverse/judges/replay_input.pydeleted file mode 100644index cdf4cdd0..00000000--- a/harness/codeverse/judges/replay_input.py+++ /dev/null@@ -1,82 +0,0 @@-"""Judge-replay inputs shared by ``3dcv judge`` (``cli/_judge.py``) and-``judges/calibration.py``.--Both re-judge a stored round from ``rounds/rNN.json`` + ``plan.json`` and must hand-the judge what the in-run judge saw. The pure pieces of that reconstruction live-here so the two replayers cannot drift (no cli / tracks / flywheel imports —-``judges/`` stays below all of them):--* ``plan_digest`` — was ``judges/calibration.py:plan_digest``;- ``cli/_judge.py:plan_summary_for`` lazily imported it from there, a cli→calibration- edge for a pure function.-* ``resolve_paths`` — was ``cli/_judge.py:resolve_paths``; calibration fed the stored- render paths straight in, so a moved or archived run replayed against the host it- was produced on.-* ``judged_subset`` — was ``cli/_judge.py:judged_subset``; calibration judged every- stored view, so whenever a track stamps ``RenderView.judge`` its numbers were not- those of the in-run judge or ``3dcv judge``.--The pieces that need ``RunRecord`` / ``tracks.get_track`` / ``flywheel.record``-(``previous``, ``extra_context``, clay views) stay in ``cli/_judge.py``: nothing in-``judges/`` imports those packages and they all import ``judges/``.-"""--from __future__ import annotations--from typing import Any--from codeverse.contracts.artifacts import RenderSet-from codeverse.workspace import Workspace---def plan_digest(plan: dict[str, Any]) -> str:- """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""- bits: list[str] = []- title = plan.get("object_name") or plan.get("title") or ""- if title:- bits.append(f"{title}: {plan.get('summary', '')}".strip())- bbox = plan.get("overall_bbox") or {}- if isinstance(bbox, dict) and bbox.get("extents"):- e = bbox["extents"]- bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")- parts = plan.get("parts") or []- if parts:- bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")- if plan.get("root_link"):- bits.append(f"Root link {plan['root_link']}.")- joints = plan.get("joints") or []- if joints:- bits.append("Joints: " + "; ".join(- f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"- for j in joints) + ".")- for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):- items = plan.get(key) or []- if items:- bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")- if plan.get("setting"):- bits.append(f"Setting: {plan['setting']}.")- return " ".join(bits)---def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:- """Render paths out of a round record, resolved against THIS workspace.-- Delegates to ``Workspace.rebase``: this used to rebase only paths for which- ``is_absolute()`` was False, which made it a no-op against every record the harness- itself writes (they are all absolute) — so a moved or archived run kept pointing at- the host it was produced on.- """- if rs is None:- return None- fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]- sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet- out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir- return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})---def judged_subset(rs: RenderSet | None) -> RenderSet | None:- """The views the in-run judge actually saw: the per-view ``judge`` flags- stamped at render time; legacy rounds (no flags) keep every stored view."""- if rs is None or not any(v.judge is not None for v in rs.views):- return rs- return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})diff --git a/harness/codeverse/judges/rubrics.py b/harness/codeverse/judges/rubrics.pyindex 491c5008..2767fade 100644--- a/harness/codeverse/judges/rubrics.py+++ b/harness/codeverse/judges/rubrics.py@@ -21,14 +21,27 @@ from __future__ import annotationsimport hashlibimport jsonimport re+import statistics+from fnmatch import fnmatchfrom functools import lru_cachefrom pathlib import Path-from typing import Literal+from typing import Any, Literalimport yaml-from pydantic import BaseModel, Field, field_validator, model_validator--from codeverse.contracts.artifacts import Severity+from pydantic import (+ BaseModel,+ Field,+ ValidationError,+ create_model,+ field_validator,+ model_validator,+)++from codeverse.contracts.artifacts import GateFinding, GateReport, RenderView, Severity+from codeverse.contracts.common import Usage+from codeverse.contracts.judgment import ImprovementItem, JudgeIssue, Judgment+from codeverse.contracts.plan import AcceptanceItem+from codeverse.models.schema_utils import JsonParseError, parse_json_lenientRUBRICS_DIR = Path(__file__).resolve().parent / "rubrics"ANCHOR_LEVELS = ("1.0", "0.7", "0.4", "0.1")@@ -211,3 +224,638 @@ def load_rubric(name: str) -> Rubric:raise RubricError(f"rubric file {path} must be a mapping")data.setdefault("name", path.stem)return rubric_from_dict(data, name_hint=path.stem)+++# ===================================================================== output_schema+# (merged from codeverse/judges/output_schema.py, 2026-08-28)+class JudgeParseError(ValueError):+ """The model's reply could not be turned into a complete JudgeOutput."""+++class CriterionScore(BaseModel):+ score: float = Field(ge=0.0, le=1.0, description="0..1 against the anchors")+ evidence: str = Field(description="what you saw, citing VIEW labels / measurements")+++class AcceptanceVerdict(BaseModel):+ verified: bool = Field(description="true only if the renders/measurements prove it")+ evidence: str = Field(default="", description="which view / number proves or disproves it")+++class DefectVerdict(BaseModel):+ present: bool = Field(description="true only if the defect is VISIBLE in the images or stated by a gate/measurement")+ evidence: str = Field(default="", description="which image / tile / number shows it (or why it is absent)")+++class JudgeOutput(BaseModel):+ """Provider-neutral parsed judge reply (criteria + acceptance + defect checklist as dicts)."""++ criteria: dict[str, CriterionScore]+ summary: str = ""+ strengths: list[str] = Field(default_factory=list)+ issues: list[JudgeIssue] = Field(default_factory=list)+ improvement_plan: list[ImprovementItem] = Field(default_factory=list)+ acceptance: dict[str, AcceptanceVerdict] = Field(default_factory=dict)+ defects: dict[str, bool] = Field(default_factory=dict, description="checklist id → defect present")+ defect_evidence: dict[str, str] = Field(default_factory=dict)++ @property+ def scores(self) -> dict[str, float]:+ return {k: v.score for k, v in self.criteria.items()}++ @property+ def acceptance_bools(self) -> dict[str, bool]:+ return {k: v.verified for k, v in self.acceptance.items()}+++def build_wire_model(rubric: Rubric, acceptance_ids: list[str]) -> type[BaseModel]:+ """A pydantic model whose ``criteria`` / ``acceptance`` objects have FIXED keys.++ Criterion ids are valid identifiers (enforced by the rubric). Acceptance ids+ are arbitrary strings → fields ``acc_<i>`` aliased to the id.+ """+ crit_fields: dict[str, Any] = {+ c.id: (CriterionScore, Field(description=f"{c.label}: {c.description.strip()[:200]}"))+ for c in rubric.visual_criteria()+ }+ Criteria = create_model("Criteria", **crit_fields)+ # Field order = generation order for JSON-mode models: observe (summary, strengths, issues, defect+ # checklist, acceptance) BEFORE scoring, so the scores follow the evidence rather than the reverse.+ fields: dict[str, Any] = {+ "summary": (str, Field(description="2-4 sentences: what it is, what is right, what is most wrong")),+ "strengths": (list[str], Field(default_factory=list, description="≤ 5 short bullets")),+ "issues": (list[JudgeIssue], Field(default_factory=list, description="observable defects, most severe first")),+ }+ if rubric.defects:+ def_fields: dict[str, Any] = {+ d.id: (DefectVerdict, Field(description=d.text.strip()[:200])) for d in rubric.defects+ }+ Defects = create_model("Defects", **def_fields)+ fields["defects"] = (Defects, Field(description="binary checklist: one entry per defect id, present=true ONLY when seen"))+ if acceptance_ids:+ acc_fields: dict[str, Any] = {+ f"acc_{i}": (AcceptanceVerdict, Field(alias=aid, description=f"acceptance item {aid}"))+ for i, aid in enumerate(acceptance_ids)+ }+ Acceptance = create_model("Acceptance", **acc_fields)+ fields["acceptance"] = (Acceptance, Field(description="one entry per acceptance item id"))+ fields["criteria"] = (Criteria, Field(description="one entry per criterion id, scored AFTER the observations above"))+ fields["improvement_plan"] = (+ list[ImprovementItem],+ Field(default_factory=list, description="≤ 6 concrete instructions for the builder, priority 1 first"),+ )+ return create_model("JudgeReply", **fields)+++def wire_schema(rubric: Rubric, acceptance_ids: list[str]) -> dict[str, Any]:+ return build_wire_model(rubric, acceptance_ids).model_json_schema(by_alias=True)+++# --------------------------------------------------------------------------- parsing+def _records_to_dict(value: Any, key_names: tuple[str, ...]) -> dict[str, Any]:+ """Accept ``{id: {...}}`` or ``[{id: ..., ...}, ...]`` and return ``{id: {...}}``."""+ if isinstance(value, dict):+ return value+ if isinstance(value, list):+ out: dict[str, Any] = {}+ for rec in value:+ if not isinstance(rec, dict):+ raise JudgeParseError(f"record is not an object: {rec!r}")+ key = next((rec[k] for k in key_names if k in rec), None)+ if key is None:+ raise JudgeParseError(f"record lacks an id field {key_names}: {rec!r}")+ out[str(key)] = {k: v for k, v in rec.items() if k not in key_names}+ return out+ raise JudgeParseError(f"expected object or list, got {type(value).__name__}")+++def _coerce_acceptance(value: Any) -> dict[str, Any]:+ recs = _records_to_dict(value, ("id", "item_id", "item"))+ out: dict[str, Any] = {}+ for k, v in recs.items():+ if isinstance(v, bool):+ out[k] = {"verified": v, "evidence": ""}+ elif isinstance(v, dict):+ vv = dict(v)+ if "verified" not in vv and "passed" in vv:+ vv["verified"] = vv.pop("passed")+ out[k] = vv+ else:+ raise JudgeParseError(f"acceptance item {k!r}: unexpected value {v!r}")+ return out+++def _coerce_defects(value: Any) -> tuple[dict[str, bool], dict[str, str]]:+ """``{id: bool}`` or ``{id: {present, evidence}}`` or a list of records → (present, evidence)."""+ if value is None:+ return {}, {}+ recs = _records_to_dict(value, ("id", "defect", "item"))+ present: dict[str, bool] = {}+ evidence: dict[str, str] = {}+ for k, v in recs.items():+ if isinstance(v, bool):+ present[k] = v+ elif isinstance(v, dict):+ flag = v.get("present", v.get("value", v.get("verified")))+ if not isinstance(flag, bool):+ raise JudgeParseError(f"defect {k!r}: 'present' must be a boolean, got {flag!r}")+ present[k] = flag+ evidence[k] = str(v.get("evidence", "") or "")+ else:+ raise JudgeParseError(f"defect item {k!r}: unexpected value {v!r}")+ return present, evidence+++def parse_judge_output(+ payload: Any, rubric: Rubric, acceptance_ids: list[str], *, measured_scores: dict[str, float] | None = None+) -> JudgeOutput:+ """Validate a parsed reply (dict) or raw text into a complete ``JudgeOutput``.++ ``measured_scores`` (criterion id → score) are injected for ``kind: measured``+ criteria so the output always covers the full rubric.+ """+ if isinstance(payload, str):+ try:+ payload = parse_json_lenient(payload)+ except JsonParseError as e:+ raise JudgeParseError(f"no JSON object in reply: {e}") from e+ if not isinstance(payload, dict):+ raise JudgeParseError(f"reply is not a JSON object: {type(payload).__name__}")+ data = dict(payload)+ if "criteria" not in data and "per_criterion" in data:+ data["criteria"] = data.pop("per_criterion")+ try:+ criteria = _records_to_dict(data.get("criteria", {}), ("id", "criterion", "name"))+ criteria = {+ k: ({"score": v, "evidence": ""} if isinstance(v, (int, float)) else v) for k, v in criteria.items()+ }+ acceptance = _coerce_acceptance(data.get("acceptance", {}))+ defects, defect_evidence = _coerce_defects(data.get("defects", {}))+ out = JudgeOutput(+ criteria=criteria,+ summary=str(data.get("summary", "") or ""),+ strengths=[str(s) for s in data.get("strengths", []) or []],+ issues=data.get("issues", []) or [],+ improvement_plan=data.get("improvement_plan", []) or [],+ acceptance=acceptance,+ defects=defects,+ defect_evidence=defect_evidence,+ )+ except ValidationError as e:+ raise JudgeParseError(f"judge reply failed validation: {e}") from e+ measured = measured_scores or {}+ for cid, s in measured.items():+ out.criteria[cid] = CriterionScore(score=max(0.0, min(1.0, float(s))), evidence="measured in code")+ missing = [c.id for c in rubric.criteria if c.id not in out.criteria]+ if missing:+ raise JudgeParseError(f"judge reply missing criteria {missing}")+ extra = [k for k in out.criteria if k not in rubric.weights]+ for k in extra: # unknown criteria are ignored, not scored+ out.criteria.pop(k)+ missing_acc = [a for a in acceptance_ids if a not in out.acceptance]+ for a in missing_acc: # unanswered acceptance items count as NOT verified+ out.acceptance[a] = AcceptanceVerdict(verified=False, evidence="not answered by judge")+ known = {d.id for d in rubric.defects}+ for k in [k for k in out.defects if k not in known]: # unknown checklist ids are ignored+ out.defects.pop(k)+ out.defect_evidence.pop(k, None)+ for d in rubric.defects:+ if d.id not in out.defects: # unanswered defect → absent (the conservative direction for a penalty)+ out.defects[d.id] = False+ out.defect_evidence[d.id] = "not answered by judge"+ return out+++# ===================================================================== caps+# (merged from codeverse/judges/caps.py, 2026-08-28)+_SEV_RANK = {Severity.INFO: 0, Severity.WARN: 1, Severity.ERROR: 2}+++class CapApplied(BaseModel):+ rule: str+ cap: float+ evidence: str = Field(description="which finding / acceptance item triggered it")+++class CapResult(BaseModel):+ overall: float+ caps_applied: list[CapApplied] = Field(default_factory=list)++ @property+ def capped(self) -> bool:+ return bool(self.caps_applied)+++def _finding_tokens(f: GateFinding) -> str:+ """Searchable lower-cased text for a finding: data.kind / data.code / message / target."""+ bits = [str(f.data.get("kind", "")), str(f.data.get("code", "")), f.message, f.target or ""]+ return " ".join(bits).lower()+++def _finding_matches(rule: CapRule, report: GateReport, f: GateFinding) -> bool:+ if not fnmatch(report.gate, rule.gate) and not fnmatch(f.gate, rule.gate):+ return False+ if _SEV_RANK[f.severity] < _SEV_RANK[rule.severity]:+ return False+ if rule.kinds:+ text = _finding_tokens(f)+ return any(k.lower() in text for k in rule.kinds)+ return True+++def missing_must_items(+ acceptance_items: list[AcceptanceItem], acceptance_results: dict[str, bool]+) -> list[str]:+ """Ids of ``must`` acceptance items that are not verified (missing counts as failed)."""+ return [+ a.id+ for a in acceptance_items+ if a.priority == "must" and not acceptance_results.get(a.id, False)+ ]+++def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) -> bool:+ """True when a deterministic gate MEASURED the thing this checklist defect claims, and found nothing.++ A checklist defect is a VLM's reading of a picture. Some of them — a floating part, an+ interpenetration — are also what a gate measures on the exported mesh, and the rubric+ already says which: a ``when="gate"`` cap rule with the same id and a ``kinds`` list.+ When every gate that rule watches ran and passed with no ERROR finding of those kinds,+ the measurement contradicts the perception, and the perception must not cap.++ Measured 2026-08-26 on a plan-pinned pair (fancy_v1 gas_street_lamp): the connectivity+ gate said "all 9 parts connected, gap <= 2 mm"; the judge read the dark seam under the+ pedestal as "floating in mid-air, a clear daylight gap", marked floating_part present, and+ ``defect:floating_part`` capped the run at 0.6 (uncapped 0.72) against 0.96 for a sibling+ the eye cannot tell apart. CLAUDE.md law 3: gates decide geometry, the VLM is perception.++ A defect with no matching gate rule (wrong_object, missing_named_part …) is never+ overridden: nothing measured it. A gate that did not run overrides nothing either.+ """+ rule = next(+ (r for r in rubric.caps if r.when == "gate" and r.id == defect_id and r.kinds), None+ )+ if rule is None:+ return False+ watched = [g for g in gates if fnmatch(g.gate, rule.gate)]+ if not watched:+ return False+ for g in watched:+ if not g.passed:+ return False+ for f in g.findings:+ if f.severity == Severity.ERROR and any(+ k in (f.message or "").lower() for k in rule.kinds+ ):+ return False+ return True+++def veto_measured_defects(+ rubric: Rubric, defects: dict[str, bool], gates: list[GateReport]+) -> tuple[dict[str, bool], list[str]]:+ """``defects`` with every measured-absent one switched off, plus the ids switched off."""+ overridden = [did for did, on in defects.items() if on and measured_absent(rubric, did, gates)]+ return {did: (on and did not in overridden) for did, on in defects.items()}, overridden+++def apply_caps(+ rubric: Rubric,+ overall: float,+ gates: list[GateReport],+ acceptance_results: dict[str, bool],+ acceptance_items: list[AcceptanceItem] | None = None,+ *,+ console_errors: list[str] | None = None,+ views: list[RenderView] | None = None,+ defects_present: dict[str, bool] | None = None,+) -> CapResult:+ """Apply every rubric cap rule; the overall becomes ``min(overall, caps...)``.++ Only the first matching finding per rule is recorded as evidence (one ledger+ line per rule), but all rules are evaluated. ``defects_present`` (checklist+ id → True) adds one ``defect:<id>`` ledger line per present defect that+ carries a ``cap``.+ """+ applied: list[CapApplied] = []+ for rule in rubric.caps:+ evidence = _rule_evidence(+ rule,+ gates,+ acceptance_results,+ acceptance_items or [],+ console_errors or [],+ views or [],+ )+ if evidence is not None:+ applied.append(CapApplied(rule=rule.id, cap=rule.cap, evidence=evidence))+ for did, present in (defects_present or {}).items():+ if not present:+ continue+ try:+ item = rubric.defect(did)+ except KeyError:+ continue+ if item.cap is not None:+ applied.append(+ CapApplied(+ rule=f"defect:{did}",+ cap=item.cap,+ evidence=f"judge checklist: {item.text[:120]}",+ )+ )+ capped = overall+ for a in applied:+ capped = min(capped, a.cap)+ return CapResult(overall=max(0.0, min(1.0, capped)), caps_applied=applied)+++def _rule_evidence(+ rule: CapRule,+ gates: list[GateReport],+ acceptance_results: dict[str, bool],+ acceptance_items: list[AcceptanceItem],+ console_errors: list[str],+ views: list[RenderView],+) -> str | None:+ if rule.when == "missing_views":+ tokens = [k.lower() for k in rule.kinds]+ if any(any(t in f"{v.name} {v.mode}".lower() for t in tokens) for v in views):+ return None+ return f"no render view matching {rule.kinds} among {len(views)} view(s)"+ if rule.when == "acceptance":+ missing = missing_must_items(acceptance_items, acceptance_results)+ if missing:+ return f"must items not verified: {', '.join(missing[:6])}"+ return None+ if rule.when == "console":+ if console_errors:+ return f"{len(console_errors)} console error(s); first: {console_errors[0][:160]}"+ return None+ for report in gates:+ for f in report.findings:+ if _finding_matches(rule, report, f):+ target = f" [{f.target}]" if f.target else ""+ return f"{report.gate}{target}: {f.message[:200]}"+ return None+++# ===================================================================== scoring+# (merged from codeverse/judges/scoring.py, 2026-08-28)+class ScoreBreakdown(BaseModel):+ """Everything the pass/fail decision used (serialised into ``Judgment.raw``)."""++ status: str = Field(default="ok", description="ok | degraded")+ per_criterion_mean: dict[str, float]+ per_criterion_std: dict[str, float]+ per_sample_overall: list[float]+ overall_uncapped: float = Field(+ description="rubric-weighted mean of the per-criterion means, before defects/caps"+ )+ overall_after_defects: float = Field(+ default=0.0, description="overall_uncapped minus defect penalties"+ )+ overall: float+ floors_hit: list[dict[str, Any]] = Field(default_factory=list)+ caps: CapResult+ must_missing: list[str] = Field(default_factory=list)+ acceptance_votes: dict[str, list[bool]] = Field(default_factory=dict)+ defects: dict[str, bool] = Field(+ default_factory=dict, description="checklist id → present (majority vote)"+ )+ defect_votes: dict[str, list[bool]] = Field(default_factory=dict)+ defect_penalty: float = 0.0+ n_requested: int+ n_used: int+ sample_errors: list[str] = Field(default_factory=list)+ tie_broken: list[str] = Field(default_factory=list,+ description="defect / acceptance ids an exact vote tie handed to the representative sample")+ rubric_hash: str = ""+ judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "+ "role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")+ samples: list[dict[str, Any]] = Field(default_factory=list)+++def _std(vals: list[float]) -> float:+ return float(statistics.pstdev(vals)) if len(vals) > 1 else 0.0+++def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeOutput:+ mean = statistics.fmean(overalls)+ return min(zip(samples, overalls, strict=True), key=lambda so: abs(so[1] - mean))[0]+++def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:+ """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie)."""+ if not votes:+ return False, False+ yes = sum(1 for v in votes if v)+ if yes * 2 == len(votes):+ return bool(tie_break), True+ return yes * 2 > len(votes), False+++def aggregate_samples(+ rubric: Rubric,+ samples: list[JudgeOutput],+ *,+ gates: list[GateReport],+ acceptance_items: list[AcceptanceItem],+ console_errors: list[str] | None = None,+ views: list[RenderView] | None = None,+ usage: Usage | None = None,+ judge_backend: str = "",+ n_requested: int | None = None,+ sample_errors: list[str] | None = None,+ judge_prompt_hash: str = "",+) -> Judgment:+ """Combine ≥ 1 parsed samples into a Judgment (raises ValueError on zero samples).++ ``views`` feeds ``missing_views`` cap rules (e.g. the articulated rubric's+ required pose sheet).+ """+ if not samples:+ raise ValueError("aggregate_samples needs at least one sample")+ per_crit: dict[str, list[float]] = {c.id: [] for c in rubric.criteria}+ overalls: list[float] = []+ for s in samples:+ sc = s.scores+ for cid in per_crit:+ per_crit[cid].append(float(sc[cid]))+ overalls.append(rubric.weighted_overall(sc))+ mean_scores = {cid: round(statistics.fmean(v), 4) for cid, v in per_crit.items()}+ std_scores = {cid: round(_std(v), 4) for cid, v in per_crit.items()}+ overall_uncapped = rubric.weighted_overall(mean_scores)++ floors = [+ {"criterion": cid, "score": round(sc, 4), "floor": fl}+ for cid, sc, fl in rubric.floors_hit(mean_scores)+ ]+ rep = _representative(samples, overalls)+ tie_broken: list[str] = []+ votes: dict[str, list[bool]] = {}+ for s in samples:+ for aid, ok in s.acceptance_bools.items():+ votes.setdefault(aid, []).append(bool(ok))+ for a in acceptance_items:+ votes.setdefault(a.id, [False])+ acceptance: dict[str, bool] = {}+ for aid, v in votes.items():+ acceptance[aid], tied = _vote(v, bool(rep.acceptance_bools.get(aid, False)))+ if tied:+ tie_broken.append(aid)+ must_missing = missing_must_items(acceptance_items, acceptance)++ d_votes: dict[str, list[bool]] = {d.id: [] for d in rubric.defects}+ for s in samples:+ for did, flag in s.defects.items():+ d_votes.setdefault(did, []).append(bool(flag))+ defects: dict[str, bool] = {}+ for did, v in d_votes.items():+ defects[did], tied = _vote(v, bool(rep.defects.get(did, False)))+ if tied:+ tie_broken.append(did)+ # a checklist defect that a passed gate measured as absent neither penalises nor caps+ # (judges/caps.measured_absent); it is still named in the verdict tail+ defects, overridden = veto_measured_defects(rubric, defects, gates)+ penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)+ after_defects = max(0.0, overall_uncapped - penalty)++ caps = apply_caps(+ rubric,+ after_defects,+ gates,+ acceptance,+ acceptance_items,+ console_errors=console_errors,+ views=views,+ defects_present=defects,+ )+ overall = caps.overall+ passed = overall >= rubric.pass_threshold and not floors and not must_missing++ summary = rep.summary.strip()+ tail = _verdict_tail(+ rubric,+ overall,+ passed,+ floors,+ caps,+ must_missing,+ defects=defects,+ penalty=penalty,+ overridden=overridden,+ )+ if tail:+ summary = (summary + " " if summary else "") + tail++ breakdown = ScoreBreakdown(+ per_criterion_mean=mean_scores,+ per_criterion_std=std_scores,+ per_sample_overall=[round(o, 4) for o in overalls],+ overall_uncapped=round(overall_uncapped, 4),+ overall_after_defects=round(after_defects, 4),+ overall=round(overall, 4),+ floors_hit=floors,+ caps=caps,+ must_missing=must_missing,+ acceptance_votes=votes,+ defects=defects,+ defect_votes=d_votes,+ defect_penalty=penalty,+ n_requested=n_requested or len(samples),+ n_used=len(samples),+ sample_errors=sample_errors or [],+ tie_broken=tie_broken,+ rubric_hash=rubric.content_hash(),+ judge_prompt_hash=judge_prompt_hash,+ samples=[s.model_dump(mode="json") for s in samples],+ )+ return Judgment(+ rubric=rubric.name,+ judge_backend=judge_backend,+ scores=mean_scores,+ overall=round(overall, 4),+ passed=passed,+ summary=summary,+ strengths=list(rep.strengths),+ issues=list(rep.issues),+ improvement_plan=sorted(rep.improvement_plan, key=lambda it: it.priority),+ acceptance_results=acceptance,+ n_samples=len(samples),+ score_std=round(_std(overalls), 4),+ usage=usage or Usage(),+ raw=json.dumps(breakdown.model_dump(mode="json"), ensure_ascii=False),+ )+++def _verdict_tail(+ rubric: Rubric,+ overall: float,+ passed: bool,+ floors: list[dict[str, Any]],+ caps: CapResult,+ must_missing: list[str],+ *,+ defects: dict[str, bool] | None = None,+ penalty: float = 0.0,+ overridden: list[str] | None = None,+) -> str:+ bits = [+ f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"+ ]+ present = [d for d, on in (defects or {}).items() if on]+ if present:+ bits.append(f"defects (-{penalty:.2f}): " + ", ".join(present))+ if overridden:+ bits.append(+ "checklist claims contradicted by a passed gate (no penalty, no cap): "+ + ", ".join(overridden)+ )+ if caps.caps_applied:+ bits.append(+ "caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied)+ )+ if floors:+ bits.append(+ "floors: "+ + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors)+ )+ if must_missing:+ bits.append("must items unverified: " + ", ".join(must_missing))+ return " | ".join(bits) + "]"+++def degraded_judgment(+ rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int, judge_prompt_hash: str = ""+) -> Judgment:+ """A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""+ breakdown = {+ "status": "degraded",+ "error": error,+ "n_requested": n_requested,+ "n_used": 0,+ "rubric_hash": rubric.content_hash(),+ "judge_prompt_hash": judge_prompt_hash,+ }+ return Judgment(+ rubric=rubric.name,+ judge_backend=judge_backend,+ scores={c.id: 0.0 for c in rubric.criteria},+ overall=0.0,+ passed=False,+ summary=f"judge_error: {error}",+ n_samples=0,+ usage=usage,+ raw=json.dumps(breakdown, ensure_ascii=False),+ )+++def is_degraded(j: Judgment) -> bool:+ """True when a Judgment is a judge glitch rather than a score."""+ if j.summary.startswith("judge_error:"):+ return True+ try:+ return json.loads(j.raw).get("status") == "degraded"+ except (json.JSONDecodeError, AttributeError, TypeError):+ return Falsediff --git a/harness/codeverse/judges/scoring.py b/harness/codeverse/judges/scoring.pydeleted file mode 100644index 3a8389bd..00000000--- a/harness/codeverse/judges/scoring.py+++ /dev/null@@ -1,297 +0,0 @@-"""Turn N parsed judge samples into ONE ``Judgment`` — all arithmetic in code.--* per-criterion score = mean over samples; overall = rubric-weighted mean;- ``score_std`` = std of the per-sample overalls;-* floors: any criterion mean below its floor → verdict fails;-* defects: the rubric's binary checklist, majority vote over samples; each- present item subtracts its ``penalty`` from the overall and its ``cap`` (if- any) bounds it — arithmetic the VLM never does;-* caps: deterministic bounds from gate findings / acceptance / missing views- (``caps.py``) plus the defect caps;-* acceptance: majority vote over samples;-* an EXACT tie on a binary item (only possible with an even ``n_samples``) follows- the *representative* sample (below) instead of a fixed direction. The old rule- (defect ties → present, acceptance ties → False) made ``n_samples=2`` strictly- harsher than both n=1 and n=3 — one dissenting sample applied every penalty and- failed every must item — so scores from the ``economy`` profile (n=2) were not- comparable with the others. With the representative rule the probability that- an item is flagged at n=2 equals n=1's, while the continuous scores still average- over both samples; ids decided this way are listed in ``tie_broken``;-* narrative (summary / issues / plan / strengths) is taken from the sample whose- overall is closest to the mean (the *representative* sample) so it is coherent.-"""--from __future__ import annotations--import json-import statistics-from typing import Any--from pydantic import BaseModel, Field--from codeverse.contracts.artifacts import GateReport, RenderView-from codeverse.contracts.common import Usage-from codeverse.contracts.judgment import Judgment-from codeverse.contracts.plan import AcceptanceItem-from codeverse.judges.caps import CapResult, apply_caps, missing_must_items, veto_measured_defects-from codeverse.judges.output_schema import JudgeOutput-from codeverse.judges.rubrics import Rubric---class ScoreBreakdown(BaseModel):- """Everything the pass/fail decision used (serialised into ``Judgment.raw``)."""-- status: str = Field(default="ok", description="ok | degraded")- per_criterion_mean: dict[str, float]- per_criterion_std: dict[str, float]- per_sample_overall: list[float]- overall_uncapped: float = Field(- description="rubric-weighted mean of the per-criterion means, before defects/caps"- )- overall_after_defects: float = Field(- default=0.0, description="overall_uncapped minus defect penalties"- )- overall: float- floors_hit: list[dict[str, Any]] = Field(default_factory=list)- caps: CapResult- must_missing: list[str] = Field(default_factory=list)- acceptance_votes: dict[str, list[bool]] = Field(default_factory=dict)- defects: dict[str, bool] = Field(- default_factory=dict, description="checklist id → present (majority vote)"- )- defect_votes: dict[str, list[bool]] = Field(default_factory=dict)- defect_penalty: float = 0.0- n_requested: int- n_used: int- sample_errors: list[str] = Field(default_factory=list)- tie_broken: list[str] = Field(default_factory=list,- description="defect / acceptance ids an exact vote tie handed to the representative sample")- rubric_hash: str = ""- judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "- "role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")- samples: list[dict[str, Any]] = Field(default_factory=list)---def _std(vals: list[float]) -> float:- return float(statistics.pstdev(vals)) if len(vals) > 1 else 0.0---def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeOutput:- mean = statistics.fmean(overalls)- return min(zip(samples, overalls, strict=True), key=lambda so: abs(so[1] - mean))[0]---def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:- """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie)."""- if not votes:- return False, False- yes = sum(1 for v in votes if v)- if yes * 2 == len(votes):- return bool(tie_break), True- return yes * 2 > len(votes), False---def aggregate_samples(- rubric: Rubric,- samples: list[JudgeOutput],- *,- gates: list[GateReport],- acceptance_items: list[AcceptanceItem],- console_errors: list[str] | None = None,- views: list[RenderView] | None = None,- usage: Usage | None = None,- judge_backend: str = "",- n_requested: int | None = None,- sample_errors: list[str] | None = None,- judge_prompt_hash: str = "",-) -> Judgment:- """Combine ≥ 1 parsed samples into a Judgment (raises ValueError on zero samples).-- ``views`` feeds ``missing_views`` cap rules (e.g. the articulated rubric's- required pose sheet).- """- if not samples:- raise ValueError("aggregate_samples needs at least one sample")- per_crit: dict[str, list[float]] = {c.id: [] for c in rubric.criteria}- overalls: list[float] = []- for s in samples:- sc = s.scores- for cid in per_crit:- per_crit[cid].append(float(sc[cid]))- overalls.append(rubric.weighted_overall(sc))- mean_scores = {cid: round(statistics.fmean(v), 4) for cid, v in per_crit.items()}- std_scores = {cid: round(_std(v), 4) for cid, v in per_crit.items()}- overall_uncapped = rubric.weighted_overall(mean_scores)-- floors = [- {"criterion": cid, "score": round(sc, 4), "floor": fl}- for cid, sc, fl in rubric.floors_hit(mean_scores)- ]- rep = _representative(samples, overalls)- tie_broken: list[str] = []- votes: dict[str, list[bool]] = {}- for s in samples:- for aid, ok in s.acceptance_bools.items():- votes.setdefault(aid, []).append(bool(ok))- for a in acceptance_items:- votes.setdefault(a.id, [False])- acceptance: dict[str, bool] = {}- for aid, v in votes.items():- acceptance[aid], tied = _vote(v, bool(rep.acceptance_bools.get(aid, False)))- if tied:- tie_broken.append(aid)- must_missing = missing_must_items(acceptance_items, acceptance)-- d_votes: dict[str, list[bool]] = {d.id: [] for d in rubric.defects}- for s in samples:- for did, flag in s.defects.items():- d_votes.setdefault(did, []).append(bool(flag))- defects: dict[str, bool] = {}- for did, v in d_votes.items():- defects[did], tied = _vote(v, bool(rep.defects.get(did, False)))- if tied:- tie_broken.append(did)- # a checklist defect that a passed gate measured as absent neither penalises nor caps- # (judges/caps.measured_absent); it is still named in the verdict tail- defects, overridden = veto_measured_defects(rubric, defects, gates)- penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)- after_defects = max(0.0, overall_uncapped - penalty)-- caps = apply_caps(- rubric,- after_defects,- gates,- acceptance,- acceptance_items,- console_errors=console_errors,- views=views,- defects_present=defects,- )- overall = caps.overall- passed = overall >= rubric.pass_threshold and not floors and not must_missing-- summary = rep.summary.strip()- tail = _verdict_tail(- rubric,- overall,- passed,- floors,- caps,- must_missing,- defects=defects,- penalty=penalty,- overridden=overridden,- )- if tail:- summary = (summary + " " if summary else "") + tail-- breakdown = ScoreBreakdown(- per_criterion_mean=mean_scores,- per_criterion_std=std_scores,- per_sample_overall=[round(o, 4) for o in overalls],- overall_uncapped=round(overall_uncapped, 4),- overall_after_defects=round(after_defects, 4),- overall=round(overall, 4),- floors_hit=floors,- caps=caps,- must_missing=must_missing,- acceptance_votes=votes,- defects=defects,- defect_votes=d_votes,- defect_penalty=penalty,- n_requested=n_requested or len(samples),- n_used=len(samples),- sample_errors=sample_errors or [],- tie_broken=tie_broken,- rubric_hash=rubric.content_hash(),- judge_prompt_hash=judge_prompt_hash,- samples=[s.model_dump(mode="json") for s in samples],- )- return Judgment(- rubric=rubric.name,- judge_backend=judge_backend,- scores=mean_scores,- overall=round(overall, 4),- passed=passed,- summary=summary,- strengths=list(rep.strengths),- issues=list(rep.issues),- improvement_plan=sorted(rep.improvement_plan, key=lambda it: it.priority),- acceptance_results=acceptance,- n_samples=len(samples),- score_std=round(_std(overalls), 4),- usage=usage or Usage(),- raw=json.dumps(breakdown.model_dump(mode="json"), ensure_ascii=False),- )---def _verdict_tail(- rubric: Rubric,- overall: float,- passed: bool,- floors: list[dict[str, Any]],- caps: CapResult,- must_missing: list[str],- *,- defects: dict[str, bool] | None = None,- penalty: float = 0.0,- overridden: list[str] | None = None,-) -> str:- bits = [- f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"- ]- present = [d for d, on in (defects or {}).items() if on]- if present:- bits.append(f"defects (-{penalty:.2f}): " + ", ".join(present))- if overridden:- bits.append(- "checklist claims contradicted by a passed gate (no penalty, no cap): "- + ", ".join(overridden)- )- if caps.caps_applied:- bits.append(- "caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied)- )- if floors:- bits.append(- "floors: "- + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors)- )- if must_missing:- bits.append("must items unverified: " + ", ".join(must_missing))- return " | ".join(bits) + "]"---def degraded_judgment(- rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int, judge_prompt_hash: str = ""-) -> Judgment:- """A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""- breakdown = {- "status": "degraded",- "error": error,- "n_requested": n_requested,- "n_used": 0,- "rubric_hash": rubric.content_hash(),- "judge_prompt_hash": judge_prompt_hash,- }- return Judgment(- rubric=rubric.name,- judge_backend=judge_backend,- scores={c.id: 0.0 for c in rubric.criteria},- overall=0.0,- passed=False,- summary=f"judge_error: {error}",- n_samples=0,- usage=usage,- raw=json.dumps(breakdown, ensure_ascii=False),- )---def is_degraded(j: Judgment) -> bool:- """True when a Judgment is a judge glitch rather than a score."""- if j.summary.startswith("judge_error:"):- return True- try:- return json.loads(j.raw).get("status") == "degraded"- except (json.JSONDecodeError, AttributeError, TypeError):- return Falsediff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.pyindex dc36e8fc..70bd8e53 100644--- a/harness/codeverse/judges/vlm_judge.py+++ b/harness/codeverse/judges/vlm_judge.py@@ -18,27 +18,32 @@ to treat as a glitch, never as a score.from __future__ import annotationsimport logging+import os+import reimport time+from collections.abc import Callablefrom dataclasses import dataclass, fieldfrom pathlib import Path-from typing import Literal+from typing import Any, Literalfrom codeverse.config import get_settings-from codeverse.contracts.artifacts import RenderSet+from codeverse.contracts.artifacts import RenderSet, RenderViewfrom codeverse.contracts.chat import ChatRequest, ChatResponsefrom codeverse.contracts.common import Usagefrom codeverse.contracts.judgment import Judgmentfrom codeverse.fanout import fan_outfrom codeverse.judges.base import JudgeInput-from codeverse.judges.output_schema import (+from codeverse.judges.prompt_builder import build_judge_messages, judge_prompt_hash+from codeverse.judges.rubrics import (JudgeOutput,JudgeParseError,+ Rubric,+ aggregate_samples,+ degraded_judgment,+ load_rubric,parse_judge_output,wire_schema,)-from codeverse.judges.prompt_builder import build_judge_messages, judge_prompt_hash-from codeverse.judges.rubrics import Rubric, load_rubric-from codeverse.judges.scoring import aggregate_samples, degraded_judgmentfrom codeverse.models.base import ChatModel, ModelErrorlog = logging.getLogger(__name__)@@ -233,3 +238,333 @@ class VlmJudge:log.warning("judge %s: %s", req.label, last[:300])req = req.model_copy(update={"temperature": min(1.0, self.temperature + 0.1 * attempt)})return None, usage, last or "unknown judge failure"+++# ===================================================================== reference+# (merged from codeverse/judges/reference.py, 2026-08-28)+log = logging.getLogger(__name__)++SilhouetteFn = Callable[[str, str], dict[str, Any]]++#: IoU → score mapping: ≤ 0.25 → 0, ≥ 0.85 → 1, linear in between (matches the rubric anchors).+IOU_LOW, IOU_HIGH = 0.25, 0.85++#: score the measured criterion gets when the measurement cannot be trusted+#: (unreliable mask, or a reference whose proportions contradict the brief)+NEUTRAL_SCORE = 0.5++#: views handed to the mismatch pass (front-ish first, then a 3/4 and a side)+DIFF_VIEW_NAMES: tuple[str, ...] = ("front", "front_right_34", "front_left_34", "right", "left")+_ENV_DIFF = "CV3D_REFERENCE_DIFF"+++def iou_to_score(iou: float) -> float:+ return max(0.0, min(1.0, (float(iou) - IOU_LOW) / (IOU_HIGH - IOU_LOW)))+++def _default_silhouette_fn() -> SilhouetteFn | None:+ try:+ from codeverse.spatial.silhouette import compare_silhouette # type: ignore[attr-defined]+ except (ImportError, AttributeError):+ return None+ return compare_silhouette+++def _diff_enabled(flag: bool) -> bool:+ return flag and os.environ.get(_ENV_DIFF, "on").strip().lower() not in ("0", "off", "false", "no")+++def _conflict_note(info: dict[str, Any]) -> str:+ try:+ from codeverse.reference import conflict_note++ return conflict_note(info)+ except ImportError: # pragma: no cover+ return ""+++def _is_synth(note: str) -> bool:+ return note.startswith("SYNTHESIZED")+++class ReferenceJudge(VlmJudge):+ """VlmJudge + reference images + measured silhouette + named mismatches."""++ name = "reference"++ def __init__(+ self,+ model_id: str | None = None,+ n_samples: int = 1,+ temperature: float = 0.2,+ *,+ rubric: str | Rubric = "reference_v1",+ silhouette_fn: SilhouetteFn | None = None,+ front_view_names: tuple[str, ...] = ("front", "front_right_34"),+ best_view: bool = True,+ diff: bool = True,+ diff_model: Any | None = None,+ diff_model_id: str = "",+ **kwargs: Any,+ ):+ super().__init__(rubric, model_id, n_samples, temperature, **kwargs)+ self.silhouette_fn = silhouette_fn or _default_silhouette_fn()+ self.front_view_names = front_view_names+ self.best_view = best_view+ self.diff = diff+ self._diff_model = diff_model+ self.diff_model_id = diff_model_id+ #: the last diff computed (tests / callers that want the mismatches themselves)+ self.last_diff: Any | None = None++ # ------------------------------------------------------------------ API+ def judge(self, inp: JudgeInput, **kwargs: Any) -> Any:+ """``VlmJudge.judge`` plus the mismatch pass's own spend.++ The diff is a real model call made from :meth:`context`; folding its+ ``Usage`` into the verdict keeps ``BudgetGuard`` and ``record.total_usage``+ honest (the ledger already sees it through the metered chat model)."""+ verdict = super().judge(inp, **kwargs)+ diff = self.last_diff+ if diff is not None and getattr(diff, "usage", None) is not None:+ verdict.usage = verdict.usage + diff.usage+ return verdict++ # ------------------------------------------------------------------ hook+ def context(self, inp: JudgeInput) -> JudgeContext:+ refs = [r for r in inp.spec.references if Path(r.path).is_file()]+ synth = bool(refs) and all(_is_synth(r.note) for r in refs)+ images: list[tuple[str, str]] = []+ for i, r in enumerate(refs[:3], 1):+ tag = "SYNTHESIZED " if _is_synth(r.note) else ""+ note = f" — {r.note}" if r.note else ""+ images.append((f"{tag}REFERENCE {i}/{min(len(refs), 3)} ({r.role}){note}", r.path))+ blocks: list[str] = []+ if synth:+ blocks.append(+ "NOTE ON THE REFERENCE: it was SYNTHESIZED from the brief by an image model, not photographed. "+ "It is a shape/part-inventory target only. Where it disagrees with the brief or the stated "+ "dimensions, the BRIEF is correct and the render must follow the brief.")+ conflict = self.dimension_conflict(inp, refs)+ note = _conflict_note(conflict)+ if note:+ blocks.append(note)+ measured_ids = [c.id for c in self.rubric.measured_criteria()]+ info: dict[str, Any] = {}+ measured_scores: dict[str, float] = {}+ if measured_ids:+ info = self.measure_silhouette(inp)+ if "iou" not in info:+ raise ReferenceJudgeError(+ f"cannot score measured criteria {measured_ids}: {info.get('error', 'no iou')}")+ score, why = self.silhouette_score(info, conflict)+ measured_scores = {cid: score for cid in measured_ids}+ blocks.append(+ f"MEASURED SILHOUETTE (harness): {info['render']} render vs reference {info['reference']}: "+ f"IoU {info['iou']:.3f} → silhouette_match score {score:.2f}{why}."+ + (f" extra: {info['extra']}" if info.get("extra") else ""))+ elif refs:+ info = self.measure_silhouette(inp)+ diff = self.reference_diff(inp, refs, info, synthesized=synth) if refs else None+ self.last_diff = diff+ if diff is not None and diff.as_text():+ blocks.append(diff.as_text())+ return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images)++ # ------------------------------------------------------------------ proportion guard+ def dimension_conflict(self, inp: JudgeInput, refs: list[Any]) -> dict[str, Any]:+ """Does the target reference's own outline contradict the brief's dimensions?++ When it does, no numeric proportion signal derived from that picture may be+ used against the object — the brief wins (see+ :mod:`codeverse.reference.proportions`).+ """+ targets = [r for r in refs if r.role == "target"] or refs+ if not targets:+ return {"conflict": False}+ try:+ from codeverse.reference import dimension_conflict++ return dimension_conflict(inp.spec, targets[0].path)+ except Exception as e: # noqa: BLE001 — advisory+ log.warning("dimension-conflict check failed: %s", e)+ return {"conflict": False, "error": str(e)}++ def silhouette_score(self, info: dict[str, Any], conflict: dict[str, Any]) -> tuple[float, str]:+ """IoU → measured score, NEUTRAL when the reference contradicts the brief or the+ mask is unreliable. Returns ``(score, ' (why)')``."""+ if conflict.get("conflict"):+ return NEUTRAL_SCORE, " (NEUTRAL: the reference's proportions contradict the brief's dimensions)"+ if info.get("reliable") is False:+ return NEUTRAL_SCORE, " (NEUTRAL: the background mask was unreliable)"+ return iou_to_score(info["iou"]), ""++ # ------------------------------------------------------------------ mismatch pass+ def reference_diff(self, inp: JudgeInput, refs: list[Any], info: dict[str, Any], *, synthesized: bool) -> Any | None:+ """One extra vision call naming concrete mismatches. ``None`` when disabled;+ never raises (a failed diff simply contributes no text)."""+ if not _diff_enabled(self.diff):+ return None+ try:+ from codeverse.reference import compare+ except ImportError: # pragma: no cover - the package is part of the wheel+ return None+ targets = [r.path for r in refs if r.role == "target"] or [r.path for r in refs]+ renders = self.diff_views(list(inp.renders.views))+ measured = dict(info) if "iou" in info else None+ if measured is not None:+ measured.setdefault("view", info.get("render", ""))+ try:+ return compare(inp.spec, targets, renders, model=self._pick_diff_model(),+ part_names=_plan_part_names(inp), measured=measured, synthesized=synthesized)+ except Exception as e: # noqa: BLE001 — an advisory pass must never fail a verdict+ log.warning("reference diff pass failed: %s", e)+ return None++ def _pick_diff_model(self) -> Any:+ """``diff_model`` > ``diff_model_id`` (a cheaper perception model) > the judge's own model."""+ if self._diff_model is not None:+ return self._diff_model+ if self.diff_model_id:+ from codeverse.models import get_chat_model++ self._diff_model = get_chat_model(self.diff_model_id)+ return self._diff_model+ return self.model++ def diff_views(self, views: list[RenderView]) -> list[str]:+ """Up to 3 shaded views for the diff, front-ish first."""+ by_name = {v.name: v for v in views}+ picked = [by_name[n].path for n in DIFF_VIEW_NAMES if n in by_name]+ for v in views:+ if len(picked) >= 3:+ break+ if v.path not in picked:+ picked.append(v.path)+ return picked[:3]++ # ------------------------------------------------------------------ silhouette+ def pick_front_view(self, views: list[RenderView]) -> RenderView | None:+ for name in self.front_view_names:+ for v in views:+ if v.name == name:+ return v+ return views[0] if views else None++ def measure_silhouette(self, inp: JudgeInput) -> dict[str, Any]:+ """{iou, render, reference, extra} or {error}.++ With ``best_view=True`` (default) the IoU is taken from the render view that+ best matches the reference's camera instead of a hardcoded ``front``: a+ reference photograph has ONE camera, and comparing a three-quarter product+ shot to a straight-on elevation measures camera agreement, not shape. The+ winning view name travels in ``render`` so the number stays auditable.+ """+ targets = [r for r in inp.spec.references if r.role == "target" and Path(r.path).is_file()] or [+ r for r in inp.spec.references if Path(r.path).is_file()]+ if not targets:+ return {"error": "spec has no readable reference images"}+ views = list(inp.renders.views)+ if not views:+ return {"error": "render set has no views"}+ if self.silhouette_fn is None:+ return {"error": "compare_silhouette unavailable (codeverse.spatial.silhouette not importable)"}+ res, name = self._measure(views, targets[0].path)+ if not isinstance(res, dict) or "iou" not in res:+ return {"error": f"compare_silhouette returned no iou: {res!r}"}+ extra = {k: v for k, v in res.items() if k not in ("iou", "per_view") and isinstance(v, (int, float, str, bool))}+ out = {"iou": float(res["iou"]), "render": name, "reference": Path(targets[0].path).name,+ "reliable": bool(res.get("reliable", True)),+ "aspect_ratio_err": res.get("aspect_ratio_err"), "extra": extra}+ if res.get("per_view"):+ out["per_view"] = res["per_view"]+ return out++ def _measure(self, views: list[RenderView], reference: str) -> tuple[dict[str, Any], str]:+ """(result, view name). Best-matching view when enabled and available."""+ if self.best_view and self.silhouette_fn is _default_silhouette_fn():+ try:+ from codeverse.spatial.silhouette import best_view_match+ except ImportError: # pragma: no cover+ best_view_match = None # type: ignore[assignment]+ if best_view_match is not None:+ res = best_view_match(views, reference)+ if "iou" in res:+ return res, str(res.get("view", ""))+ view = self.pick_front_view(views)+ if view is None: # pragma: no cover - guarded by the caller+ return {}, ""+ return self.silhouette_fn(view.path, reference), view.name # type: ignore[misc]+++def _plan_part_names(inp: JudgeInput) -> list[str]:+ """Part names from the plan digest the judge already receives (best effort).++ ``judges.replay_input.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;+ a multi-line digest lists one ``- Name · role · …`` per line. Both are handled, and+ an unrecognised digest simply yields no names (the diff prompt then says so).+ """+ text = getattr(inp, "plan_summary", "") or ""+ names: list[str] = []+ for chunk in re.split(r"(?i)\bparts\s*:", text)[1:]:+ for raw in chunk.split("\n")[0].split(","):+ names.append(raw.split("\u00d7")[0].split("(")[0].strip().rstrip("."))+ for line in text.splitlines():+ line = line.strip()+ if not line.startswith(("-", "*", "\u00b7")):+ continue+ head = line.lstrip("-*\u00b7 ").split("\u00b7")[0].split("(")[0].split(":")[0].strip()+ names.append(head)+ seen: list[str] = []+ for n in names:+ if n and " " not in n and 2 < len(n) <= 40 and n[:1].isupper() and n not in seen:+ seen.append(n)+ return seen[:40]++++LIKENESS_NOTE = (+ "REAL-WORLD REFERENCE PHOTOS are attached (labelled REAL-WORLD REFERENCE n/N). They are not a "+ "composition to copy; they are what the thing the brief names ACTUALLY looks like. Score "+ "brief fidelity, colour/light and richness against them: dominant colour and where the "+ "secondary colours sit, how the structure folds / layers / thins out, where the brightness "+ "concentrates and how much of the frame stays dark, what the real texture is at fine scale. A "+ "frame that ticks every noun in the brief but would never be mistaken for the photographed "+ "thing (a row of evenly spaced bars standing in for curtains, a flat band standing in for a "+ "gradient of light, cartoon-saturated colour where the photo is subtle) is a 0.4-0.5 on brief "+ "fidelity, not a 1.0. Foreground, framing and landscape in the photos are incidental unless "+ "the brief asks for them."+)+++class LikenessJudge(VlmJudge):+ """``VlmJudge`` + the reference photos of the REAL thing — for tracks with no silhouette.++ ``ReferenceJudge`` is built for objects: it measures a front-view silhouette IoU against+ the target photo and runs a part-inventory mismatch pass. Neither means anything for a+ fragment shader or a scene, where a reference photo answers a different question — does+ this LOOK like the thing? Measured 2026-08-26 (teaser aurora, three versions, flash and+ codex): every version scored 0.78-0.94 with an empty issues list while none resembled an+ aurora (a comb of straight teal bars; pink cotton-wool lobes), because the brief is a+ checklist of nouns and the rubric had nothing to say about likeness. This judge attaches+ up to three photos beside the frames with :data:`LIKENESS_NOTE` and leaves the rubric alone.+ """++ name = "likeness"++ def __init__(self, model_id: str | None = None, n_samples: int = 1, temperature: float = 0.2, *,+ rubric: str | Rubric = "shader_v2", max_refs: int = 3, **kwargs: Any):+ super().__init__(rubric, model_id, n_samples, temperature, **kwargs)+ self.max_refs = max(1, int(max_refs))++ def context(self, inp: JudgeInput) -> JudgeContext:+ if self.rubric.measured_criteria():+ raise ValueError(f"rubric {self.rubric.name} has measured criteria; LikenessJudge measures nothing")+ refs = [r for r in inp.spec.references if Path(r.path).is_file()][: self.max_refs]+ images = [(f"REAL-WORLD REFERENCE {i}/{len(refs)}" + (f" — {r.note}" if r.note else ""), r.path)+ for i, r in enumerate(refs, 1)]+ return JudgeContext(extra_text=LIKENESS_NOTE if refs else "", extra_images=images)+++class ReferenceJudgeError(RuntimeError):+ """The reference judge cannot compute its measured criteria."""diff --git a/harness/codeverse/reference.py b/harness/codeverse/reference.pyindex 07feec45..30bfa47a 100644--- a/harness/codeverse/reference.py+++ b/harness/codeverse/reference.py@@ -5,7 +5,7 @@ synthesizes a neutral studio product shot of the brief with the image model,**validates** it (:mod:`.gate`), and hands it to the rest of the harness as thefidelity anchor: the planner sees it, the generator's ``compare_reference`` toolsees it, the silhouette gate measures against it and the judge scores against it-(:class:`codeverse.judges.reference.ReferenceJudge`).+(:class:`codeverse.judges.vlm_judge.ReferenceJudge`).A synthesized reference is never ground truth and is marked as such everywhere(:mod:`.attach`): the user's own ``--image`` wins, the brief's dimensions win,diff --git a/harness/codeverse/spatial/silhouette.py b/harness/codeverse/spatial/silhouette.pyindex 5401c979..1ad9dc95 100644--- a/harness/codeverse/spatial/silhouette.py+++ b/harness/codeverse/spatial/silhouette.py@@ -161,7 +161,7 @@ def compare_silhouette(render_png: Path | str, reference_png: Path | str, *, difreliable, diff_png_path?}``. ``reliable`` is False when either mask is empty, coversmore than 95 % of the pixels, or has a bounding box spanning the whole frame — allthree mean the background estimate failed and the IoU below is arithmetic on noise.- The caller must honour it: :class:`~codeverse.judges.reference.ReferenceJudge` scores+ The caller must honour it: :class:`~codeverse.judges.vlm_judge.ReferenceJudge` scores``silhouette_match`` NEUTRAL rather than low, because "we could not measure this" isnot the same as "it does not match"."""diff --git a/harness/codeverse/texturing/gate.py b/harness/codeverse/texturing/gate.pyindex e178cf56..73af695a 100644--- a/harness/codeverse/texturing/gate.py+++ b/harness/codeverse/texturing/gate.py@@ -91,7 +91,7 @@ def _plan_summary(plan: StaticPlan | None) -> str:def _degraded(j: Judgment) -> bool:- from codeverse.judges.scoring import is_degraded+ from codeverse.judges.rubrics import is_degradedreturn is_degraded(j)diff --git a/harness/codeverse/tracks/common.py b/harness/codeverse/tracks/common.pyindex aff03cbb..57ab927d 100644--- a/harness/codeverse/tracks/common.py+++ b/harness/codeverse/tracks/common.py@@ -61,11 +61,11 @@ class Services:def reference_judge(self, model_id: str, n_samples: int = 1, rubric: str = "reference_v1") -> Any:"""Image-conditioned judge (spec has reference images): renders + references + silhouette IoU."""- return _import("codeverse.judges.reference", "ReferenceJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)+ return _import("codeverse.judges.vlm_judge", "ReferenceJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)def likeness_judge(self, model_id: str, n_samples: int = 1, rubric: str = "shader_v2") -> Any:"""Reference photos beside the frames, no silhouette (graphics / scene with images)."""- return _import("codeverse.judges.reference", "LikenessJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)+ return _import("codeverse.judges.vlm_judge", "LikenessJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)def pairwise(self, model_id: str) -> Any:"""Position-swapped A/B judge: ``.compare(spec, renders_a, renders_b, rubric=...)``."""diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.pyindex dff22b45..84d4b242 100644--- a/harness/codeverse/tracks/steps.py+++ b/harness/codeverse/tracks/steps.py@@ -470,7 +470,7 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildResdef _is_degraded(judgment: Judgment) -> bool:try:- from codeverse.judges.scoring import is_degraded+ from codeverse.judges.rubrics import is_degradedexcept ImportError: # pragma: no cover — judges package always ships with tracksreturn Falsereturn is_degraded(judgment)diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex d045f7cb..8ec4fc88 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -141,11 +141,11 @@ codeverse/billing.py (SUBSCRIPTION_BACKENDS/bills_usd — which backends take real dollars,so max_usd guards money and not list price; docs/COST.md §25)guard.py routing.py reconstruct.py (old runs) audit.py report.py- judges/ base.py (Judge protocol), rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,- prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.py- (compare_many), reference.py, calibration.py, metrics.py, replay_input.py- (plan_digest / resolve_paths / judged_subset — the pure round-replay pieces- `3dcv judge` and calibration share)+ judges/ base.py (Judge protocol + the pure round-replay pieces `3dcv judge` and+ calibration share), rubrics.py + rubrics/*.yaml (defect checklists, the wire+ schema, caps and scoring), prompt_builder.py (image prep, montages, the+ judge messages), vlm_judge.py (+ the reference/likeness judges),+ pairwise.py (compare_many), calibration.pyreference/ reference GROUNDING — give the pipeline a picture of what it is building:one module: synthesis, THE plausibility gate that makes a synthesizedreference safe to use, Spec attachment + honesty guards, proportionsdiff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.mdindex 8fa925dc..beb7a089 100644--- a/harness/docs/INTERFACES.md+++ b/harness/docs/INTERFACES.md@@ -282,19 +282,19 @@ VlmJudge(rubric="static_object_v1", model_id=None (settings.default_judge = gemiVlmJudge.judge(inp, *, geometry_views: RenderSet | None = None) -> Judgment # also reads inp.geometry_viewsfrom codeverse.judges.base import JudgeInput # (spec, renders, measurement=None, gates=[], acceptance=[], plan_summary="",# round_index=0, previous=None, extra_context="", geometry_views=None)-from codeverse.judges.montage import plan_montages, render_montage, Montage # ≤3 2×2 montages (shaded/geometry/poses) + ≤2 detail+from codeverse.judges.prompt_builder import plan_montages, render_montage, Montage # ≤3 2×2 montages (shaded/geometry/poses) + ≤2 detail# crops @≤1024px replace sheet+9 views; clay/normals views (RenderView.mode) auto-route to the GEOMETRY montage-from codeverse.judges.scoring import is_degraded, aggregate_samples # ScoreBreakdown adds defects, defect_votes (majority,+from codeverse.judges.rubrics import is_degraded, aggregate_samples # ScoreBreakdown adds defects, defect_votes (majority,# ties→representative sample, D36), tie_broken, defect_penalty, overall_after_defects,# judge_prompt_hash (D37); overall = caps(weighted_mean − Σpenalty)-from codeverse.judges.caps import apply_caps # (rubric, overall, gates, acceptance_results, acceptance_items=None, *,+from codeverse.judges.rubrics import apply_caps # (rubric, overall, gates, acceptance_results, acceptance_items=None, *,# console_errors=None, views=None, defects_present=None) -> CapResult;# cap rules add when="missing_views" and ledger lines "defect:<id>"from codeverse.judges.pairwise import PairwiseJudge # .compare(spec, renders_a, renders_b, *, rubric=…) -> PairwiseResultPairwiseJudge.compare_many(spec, candidates: list[RenderSet], *, rubric) -> RankingResult{order, points, confidence, pairs, usage, .best}from codeverse.judges.calibration import calibrate, CalibrationTable # (run_dirs, *, model_id, n_samples=3, out_dir, geometry_mode,# rounds, …) -> rows + pearson/spearman(errors vs score), mean_std, cost; CLI: python -m codeverse.judges.calibration RUN… --n 3-from codeverse.judges.reference import ReferenceJudge # image-conditioned specs+from codeverse.judges.vlm_judge import ReferenceJudge # image-conditioned specsfrom codeverse.judges.metrics import judge_agreement, plateau, best_index```Rubrics: `static_object_v1` (0.72), `articulated_v1` (requires pose views viadiff --git a/harness/tests/compare_bench/test_fixed_eval_language.py b/harness/tests/compare_bench/test_fixed_eval_language.pyindex aee4ee8c..ac9cd08f 100644--- a/harness/tests/compare_bench/test_fixed_eval_language.py+++ b/harness/tests/compare_bench/test_fixed_eval_language.py@@ -67,7 +67,7 @@ def test_graphics_judge_is_the_track_rubric_with_the_photos():from bench._fixed_eval import FixedEvaluatorfrom codeverse.contracts.common import Trackfrom codeverse.contracts.spec import ReferenceImage, Spec- from codeverse.judges.reference import LikenessJudge+ from codeverse.judges.vlm_judge import LikenessJudgeev = FixedEvaluator("gemini:gemini-3.1-pro-preview", n_samples=2)with_photos = Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="x", references=[ReferenceImage(path="/p.png")])diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.pyindex 82706d09..edc488f8 100644--- a/harness/tests/flywheel_cli/test_cli.py+++ b/harness/tests/flywheel_cli/test_cli.py@@ -175,7 +175,7 @@ def test_judge_rejudges_with_in_run_inputs(runs_dir: Path, monkeypatch):def test_judge_uses_reference_judge_for_measured_rubrics(runs_dir: Path, monkeypatch):- import codeverse.judges.reference as ref+ import codeverse.judges.vlm_judge as refclass _FakeRef(_FakeVlm):passdiff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.pyindex ac3f5964..06827bde 100644--- a/harness/tests/judges/test_calibration.py+++ b/harness/tests/judges/test_calibration.py@@ -5,8 +5,8 @@ import pytestfrom codeverse.contracts.artifacts import GateFinding, GateReport, Severityfrom codeverse.contracts.run import RoundRecord+from codeverse.judges.base import plan_digestfrom codeverse.judges.calibration import calibrate, load_run_cases, pearson, spearman-from codeverse.judges.replay_input import plan_digestfrom codeverse.judges.rubrics import load_rubricfrom tests.judges.conftest import (ACCEPTANCE,diff --git a/harness/tests/judges/test_caps.py b/harness/tests/judges/test_caps.pyindex 92d697a3..769a9a03 100644--- a/harness/tests/judges/test_caps.py+++ b/harness/tests/judges/test_caps.py@@ -1,7 +1,6 @@from codeverse.contracts.artifacts import GateFinding, GateReport, Severityfrom codeverse.contracts.plan import AcceptanceItem-from codeverse.judges.caps import apply_caps-from codeverse.judges.rubrics import load_rubric+from codeverse.judges.rubrics import apply_caps, load_rubricR = load_rubric("static_object_v1")@@ -73,8 +72,7 @@ def test_a_floating_claim_the_connectivity_gate_measured_absent_neither_penalise'all 9 parts connected, gap <= 2 mm'; the judge marked floating_part present on a darkseam and defect:floating_part capped the run at 0.6 (uncapped 0.72) against 0.96 for asibling the eye cannot tell apart. Gates decide geometry (law 3)."""- from codeverse.judges.caps import measured_absent, veto_measured_defects- from codeverse.judges.rubrics import load_rubric+ from codeverse.judges.rubrics import load_rubric, measured_absent, veto_measured_defectsrubric = load_rubric("static_object_v1")passed = [_connectivity(True, ("info", "all 9 parts are connected (9 contacts, gap <= 2 mm)"),@@ -86,8 +84,7 @@ def test_a_floating_claim_the_connectivity_gate_measured_absent_neither_penalisedef test_a_real_floating_part_still_caps():- from codeverse.judges.caps import measured_absent- from codeverse.judges.rubrics import load_rubric+ from codeverse.judges.rubrics import load_rubric, measured_absentrubric = load_rubric("static_object_v1")failed = [_connectivity(False, ("error", "part 'Seat' is floating 12 mm above 'Leg'"))]@@ -100,8 +97,7 @@ def test_scene_placement_gate_measures_the_floating_asset_claim():"""scene_v1 (2026-08-26): the placement gate and the checklist defect share an id, so a passedgate vetoes the VLM's floating/sunken claim and an ERROR there caps the round at 0.7."""from codeverse.contracts.artifacts import GateFinding, GateReport, Severity- from codeverse.judges.caps import apply_caps, measured_absent- from codeverse.judges.rubrics import load_rubric+ from codeverse.judges.rubrics import apply_caps, load_rubric, measured_absentrubric = load_rubric("scene_v1")passed = [GateReport(gate="scene_placement", passed=True, findings=[GateFinding(diff --git a/harness/tests/judges/test_defects.py b/harness/tests/judges/test_defects.pyindex a1e13ca1..15b13e31 100644--- a/harness/tests/judges/test_defects.py+++ b/harness/tests/judges/test_defects.py@@ -5,10 +5,16 @@ import jsonimport pytestfrom codeverse.contracts.artifacts import RenderSet, RenderView-from codeverse.judges.caps import apply_caps-from codeverse.judges.output_schema import JudgeParseError, parse_judge_output, wire_schema-from codeverse.judges.rubrics import RubricError, load_rubric, rubric_from_dict-from codeverse.judges.scoring import aggregate_samples+from codeverse.judges.rubrics import (+ JudgeParseError,+ RubricError,+ aggregate_samples,+ apply_caps,+ load_rubric,+ parse_judge_output,+ rubric_from_dict,+ wire_schema,+)from codeverse.judges.vlm_judge import VlmJudgefrom tests.judges.conftest import FakeChatModel, good_replydiff --git a/harness/tests/judges/test_likeness.py b/harness/tests/judges/test_likeness.pyindex 7017bce5..a26a8202 100644--- a/harness/tests/judges/test_likeness.py+++ b/harness/tests/judges/test_likeness.py@@ -4,8 +4,8 @@ from __future__ import annotationsfrom codeverse.contracts.common import Language, Trackfrom codeverse.contracts.spec import ReferenceImage-from codeverse.judges.reference import LIKENESS_NOTE, LikenessJudgefrom codeverse.judges.rubrics import load_rubric+from codeverse.judges.vlm_judge import LIKENESS_NOTE, LikenessJudgefrom tests.judges.conftest import FakeChatModel, draw_chair, good_reply, image_parts, make_specdiff --git a/harness/tests/judges/test_live.py b/harness/tests/judges/test_live.pyindex 7263ce82..266cf942 100644--- a/harness/tests/judges/test_live.py+++ b/harness/tests/judges/test_live.py@@ -13,7 +13,7 @@ from codeverse.config import get_settingsfrom codeverse.contracts.artifacts import RenderSetfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.judges.base import JudgeInput-from codeverse.judges.scoring import is_degraded+from codeverse.judges.rubrics import is_degradedfrom codeverse.judges.vlm_judge import VlmJudgefrom tests.judges.conftest import make_measurement, make_renders, make_spec@@ -130,7 +130,7 @@ def test_live_compare_many_ranks_stool_over_crude(tmp_path):def test_live_reference_judge(tmp_path):from codeverse.contracts.spec import ReferenceImage- from codeverse.judges.reference import ReferenceJudge+ from codeverse.judges.vlm_judge import ReferenceJudgerenders, meas, source = _renders_for_test(tmp_path / "stool")if "stool" not in source:diff --git a/harness/tests/judges/test_montage.py b/harness/tests/judges/test_montage.pyindex 9f594a83..84998bae 100644--- a/harness/tests/judges/test_montage.py+++ b/harness/tests/judges/test_montage.py@@ -4,8 +4,8 @@ import pytestfrom PIL import Imagefrom codeverse.contracts.artifacts import RenderSet, RenderView-from codeverse.judges.images import JudgeImageError-from codeverse.judges.montage import (+from codeverse.judges.prompt_builder import (+ JudgeImageError,describe_montages,montage_label,montage_strip,diff --git a/harness/tests/judges/test_output_schema.py b/harness/tests/judges/test_output_schema.pyindex cc331f61..0e437bdf 100644--- a/harness/tests/judges/test_output_schema.py+++ b/harness/tests/judges/test_output_schema.py@@ -2,12 +2,12 @@ import jsonimport pytest-from codeverse.judges.output_schema import (+from codeverse.judges.rubrics import (JudgeParseError,+ load_rubric,parse_judge_output,wire_schema,)-from codeverse.judges.rubrics import load_rubricfrom tests.judges.conftest import good_replyR = load_rubric("static_object_v1")diff --git a/harness/tests/judges/test_prompt_builder.py b/harness/tests/judges/test_prompt_builder.pyindex fe4a5387..f6b40ab9 100644--- a/harness/tests/judges/test_prompt_builder.py+++ b/harness/tests/judges/test_prompt_builder.py@@ -5,12 +5,13 @@ from PIL import Imagefrom codeverse.contracts.artifacts import GateFinding, GateReport, RenderSet, RenderView, Severityfrom codeverse.contracts.chat import ImagePart, TextPartfrom codeverse.contracts.judgment import ImprovementItem, Judgment-from codeverse.judges.images import prepare_image, view_az_elfrom codeverse.judges.prompt_builder import (TEXT_BUDGET_CHARS,build_judge_messages,build_system_prompt,montage_image_parts,+ prepare_image,+ view_az_el,)from codeverse.judges.rubrics import load_rubricfrom tests.judges.conftest import draw_chair, make_rendersdiff --git a/harness/tests/judges/test_reference.py b/harness/tests/judges/test_reference.pyindex e971a9db..9b3792a7 100644--- a/harness/tests/judges/test_reference.py+++ b/harness/tests/judges/test_reference.py@@ -1,8 +1,8 @@import pytestfrom codeverse.contracts.spec import ReferenceImage-from codeverse.judges.reference import ReferenceJudge, ReferenceJudgeError, iou_to_scorefrom codeverse.judges.rubrics import load_rubric+from codeverse.judges.vlm_judge import ReferenceJudge, ReferenceJudgeError, iou_to_scorefrom tests.judges.conftest import FakeChatModel, draw_chair, good_reply, image_partsREF = load_rubric("reference_v1")diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.pyindex b1e7a677..420dae8e 100644--- a/harness/tests/judges/test_vlm_judge.py+++ b/harness/tests/judges/test_vlm_judge.py@@ -3,8 +3,7 @@ import jsonimport pytestfrom codeverse.contracts.artifacts import GateFinding, GateReport, Severity-from codeverse.judges.rubrics import load_rubric-from codeverse.judges.scoring import is_degraded+from codeverse.judges.rubrics import is_degraded, load_rubricfrom codeverse.judges.vlm_judge import VlmJudgefrom codeverse.models.base import ModelErrorfrom tests.judges.conftest import FakeChatModel, good_reply, image_parts@@ -151,7 +150,7 @@ def test_base_judge_rejects_measured_rubric(judge_input, cache_dir):def test_missing_render_is_loud(judge_input, cache_dir):- from codeverse.judges.images import JudgeImageError+ from codeverse.judges.prompt_builder import JudgeImageErrorjudge_input.renders.views[0].path = "/nonexistent/x.png"with pytest.raises(JudgeImageError):_judge(FakeChatModel([good_reply(R, IDS)]), cache_dir=cache_dir).judge(judge_input)@@ -205,8 +204,8 @@ def test_judge_prompt_hash_is_recorded_in_every_verdict(judge_input, cache_dir):def test_judge_prompt_hash_tracks_the_protocol_not_the_run(monkeypatch):from codeverse.judges import prompt_builder as pb- from codeverse.judges.output_schema import wire_schemafrom codeverse.judges.prompt_builder import judge_prompt_hash+ from codeverse.judges.rubrics import wire_schemabase = judge_prompt_hash(R)assert judge_prompt_hash(R) == base # deterministicdiff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.pyindex 53b90539..3bbcb169 100644--- a/harness/tests/orchestrator_tracks/test_refactor_batch2.py+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py@@ -123,7 +123,7 @@ def test_scene_judge_views_prefers_stamped_flags():def test_cli_judge_reconstructs_the_judged_subset(tmp_path):- from codeverse.judges.replay_input import judged_subset, resolve_paths+ from codeverse.judges.base import judged_subset, resolve_pathsrs = _rs([True, False, None])sub = judged_subset(rs)diff --git a/harness/tests/reference/test_judge_diff.py b/harness/tests/reference/test_judge_diff.pyindex 4c7e130b..7b3f5454 100644--- a/harness/tests/reference/test_judge_diff.py+++ b/harness/tests/reference/test_judge_diff.py@@ -7,8 +7,8 @@ import pytestfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.spec import ReferenceImagefrom codeverse.judges.base import JudgeInput-from codeverse.judges.reference import ReferenceJudgefrom codeverse.judges.rubrics import load_rubric+from codeverse.judges.vlm_judge import ReferenceJudgefrom codeverse.reference import SYNTH_NOTEfrom tests.judges.conftest import (FakeChatModel,@@ -101,7 +101,7 @@ def test_reference_that_contradicts_the_brief_scores_neutral(tmp_path, cache_dir"""A picture whose proportions disagree with the stated dimensions must not dragthe score down: the brief wins and the measured criterion goes neutral."""from codeverse.contracts.spec import Constraints- from codeverse.judges.reference import NEUTRAL_SCORE+ from codeverse.judges.vlm_judge import NEUTRAL_SCOREpng = draw_chair(tmp_path / "wide.png", legs=4) # ~2:1 wide silhouettespec = make_spec(references=[ReferenceImage(path=str(png), role="target", note=f"{SYNTH_NOTE} [front, x]")],@@ -118,7 +118,7 @@ def test_reference_that_contradicts_the_brief_scores_neutral(tmp_path, cache_dirdef test_part_names_are_read_from_both_plan_digest_shapes():- from codeverse.judges.reference import _plan_part_names+ from codeverse.judges.vlm_judge import _plan_part_namesclass One:plan_summary = ("Chair: a chair. Overall 0.5x0.5x0.9 m. "diff --git a/harness/tests/scene_gates/test_frame_metrics.py b/harness/tests/scene_gates/test_frame_metrics.pyindex cd8e2459..dd748c79 100644--- a/harness/tests/scene_gates/test_frame_metrics.py+++ b/harness/tests/scene_gates/test_frame_metrics.py@@ -6,8 +6,7 @@ import jsonfrom pathlib import Pathfrom codeverse.contracts.artifacts import RenderSet, RenderView, Severity-from codeverse.judges.caps import apply_caps-from codeverse.judges.rubrics import load_rubric+from codeverse.judges.rubrics import apply_caps, load_rubricfrom codeverse.spatial.frame_metrics import (FRAME_GATE,frame_findings,diff --git a/harness/tests/scene_gates/test_placement.py b/harness/tests/scene_gates/test_placement.pyindex cb73650b..967f74d9 100644--- a/harness/tests/scene_gates/test_placement.py+++ b/harness/tests/scene_gates/test_placement.py@@ -12,8 +12,7 @@ from types import SimpleNamespacefrom codeverse.contracts.artifacts import BuildResult, Severityfrom codeverse.contracts.common import Languagefrom codeverse.contracts.plan import BBox, CameraPlan, ScenePlan, ZonePlan-from codeverse.judges.caps import apply_caps-from codeverse.judges.rubrics import load_rubric+from codeverse.judges.rubrics import apply_caps, load_rubricfrom codeverse.orchestrator import build_refine_instructionsfrom codeverse.spatial.scene_placement import (GATE,