Skip to content

Official Codebase of 3DCodeVerse

Python

judges: thirteen modules become six, and the high-traffic names keep their addresses

gaoypenga82f941f6ee5
diff --git a/harness/codeverse/cli/_judge.py b/harness/codeverse/cli/_judge.py
index e8294ea6..6c891585 100644
--- a/harness/codeverse/cli/_judge.py
+++ b/harness/codeverse/cli/_judge.py
@@ -21,7 +21,7 @@ from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderView
from codeverse.contracts.common import TRACK_INFO
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.run import RoundRecord, RunRecord
-from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_paths
+from codeverse.judges.base import judged_subset, plan_digest, resolve_paths
from codeverse.proc import read_json_or_none
from codeverse.workspace import Workspace
@@ -115,7 +115,7 @@ def make_judge(rec: RunRecord, rubric_name: str, model_id: str, n: int) -> Any:
raise C.CliError(f"cannot load rubric {rubric_name!r}: {e}") from e
measured = bool(rubric.measured_criteria())
if measured or rec.spec.references:
- ReferenceJudge = C.lazy("codeverse.judges.reference", "ReferenceJudge")
+ ReferenceJudge = C.lazy("codeverse.judges.vlm_judge", "ReferenceJudge")
return ReferenceJudge(model_id=model_id, n_samples=n, rubric=rubric_name)
VlmJudge = C.lazy("codeverse.judges.vlm_judge", "VlmJudge")
return VlmJudge(rubric=rubric_name, model_id=model_id, n_samples=n)
diff --git a/harness/codeverse/cli/inspect_cmd.py b/harness/codeverse/cli/inspect_cmd.py
index 89261ab5..4f15a319 100644
--- a/harness/codeverse/cli/inspect_cmd.py
+++ b/harness/codeverse/cli/inspect_cmd.py
@@ -228,7 +228,7 @@ def judge(
except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothing
raise C.CliError(f"judge failed: {e}") from e
except Exception as e:
- ReferenceJudgeError = C.lazy("codeverse.judges.reference", "ReferenceJudgeError")
+ ReferenceJudgeError = C.lazy("codeverse.judges.vlm_judge", "ReferenceJudgeError")
if isinstance(e, ReferenceJudgeError):
raise C.CliError(f"judge failed: {e}") from e
raise
diff --git a/harness/codeverse/flywheel/record.py b/harness/codeverse/flywheel/record.py
index a0eb02e5..92857812 100644
--- a/harness/codeverse/flywheel/record.py
+++ b/harness/codeverse/flywheel/record.py
@@ -128,7 +128,7 @@ def effective_judgment(r: RoundRecord):
j = r.judgment
if j is None:
return None
- from codeverse.judges.scoring import is_degraded
+ from codeverse.judges.rubrics import is_degraded
return None if is_degraded(j) else j
diff --git a/harness/codeverse/judges/base.py b/harness/codeverse/judges/base.py
index 6d4edbd4..4f703bcc 100644
--- a/harness/codeverse/judges/base.py
+++ b/harness/codeverse/judges/base.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-from typing import Protocol
+from typing import Any, Protocol
from pydantic import BaseModel, Field
@@ -10,6 +10,7 @@ from codeverse.contracts.artifacts import GateReport, Measurement, RenderSet
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.spec import Spec
+from codeverse.workspace import Workspace
class JudgeInput(BaseModel):
@@ -31,3 +32,58 @@ class Judge(Protocol):
name: str
def judge(self, inp: JudgeInput) -> Judgment: ...
+
+
+# ===================================================================== replay_input
+# (merged from codeverse/judges/replay_input.py, 2026-08-28)
+def plan_digest(plan: dict[str, Any]) -> str:
+ """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""
+ bits: list[str] = []
+ title = plan.get("object_name") or plan.get("title") or ""
+ if title:
+ bits.append(f"{title}: {plan.get('summary', '')}".strip())
+ bbox = plan.get("overall_bbox") or {}
+ if isinstance(bbox, dict) and bbox.get("extents"):
+ e = bbox["extents"]
+ bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")
+ parts = plan.get("parts") or []
+ if parts:
+ bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")
+ if plan.get("root_link"):
+ bits.append(f"Root link {plan['root_link']}.")
+ joints = plan.get("joints") or []
+ if joints:
+ bits.append("Joints: " + "; ".join(
+ f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"
+ for j in joints) + ".")
+ for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):
+ items = plan.get(key) or []
+ if items:
+ bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")
+ if plan.get("setting"):
+ bits.append(f"Setting: {plan['setting']}.")
+ return " ".join(bits)
+
+
+def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:
+ """Render paths out of a round record, resolved against THIS workspace.
+
+ Delegates to ``Workspace.rebase``: this used to rebase only paths for which
+ ``is_absolute()`` was False, which made it a no-op against every record the harness
+ itself writes (they are all absolute) — so a moved or archived run kept pointing at
+ the host it was produced on.
+ """
+ if rs is None:
+ return None
+ fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]
+ sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet
+ out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir
+ return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})
+
+
+def judged_subset(rs: RenderSet | None) -> RenderSet | None:
+ """The views the in-run judge actually saw: the per-view ``judge`` flags
+ stamped at render time; legacy rounds (no flags) keep every stored view."""
+ if rs is None or not any(v.judge is not None for v in rs.views):
+ return rs
+ return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})
diff --git a/harness/codeverse/judges/calibration.py b/harness/codeverse/judges/calibration.py
index 78f3f87b..805f8e13 100644
--- a/harness/codeverse/judges/calibration.py
+++ b/harness/codeverse/judges/calibration.py
@@ -32,9 +32,8 @@ from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.run import RoundRecord
from codeverse.contracts.spec import Spec
-from codeverse.judges.base import JudgeInput
-from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_paths
-from codeverse.judges.scoring import is_degraded
+from codeverse.judges.base import JudgeInput, judged_subset, plan_digest, resolve_paths
+from codeverse.judges.rubrics import is_degraded
from codeverse.judges.vlm_judge import VlmJudge
from codeverse.workspace import Workspace
diff --git a/harness/codeverse/judges/caps.py b/harness/codeverse/judges/caps.py
deleted file mode 100644
index 91bb717d..00000000
--- a/harness/codeverse/judges/caps.py
+++ /dev/null
@@ -1,186 +0,0 @@
-"""Deterministic caps: bound the overall score from gate findings / acceptance.
-
-The VLM never sees or computes these. ``apply_caps`` returns the capped overall
-and a ledger of which rules fired and why, so the verdict is auditable.
-"""
-
-from __future__ import annotations
-
-from fnmatch import fnmatch
-
-from pydantic import BaseModel, Field
-
-from codeverse.contracts.artifacts import GateFinding, GateReport, RenderView, Severity
-from codeverse.contracts.plan import AcceptanceItem
-from codeverse.judges.rubrics import CapRule, Rubric
-
-_SEV_RANK = {Severity.INFO: 0, Severity.WARN: 1, Severity.ERROR: 2}
-
-
-class CapApplied(BaseModel):
- rule: str
- cap: float
- evidence: str = Field(description="which finding / acceptance item triggered it")
-
-
-class CapResult(BaseModel):
- overall: float
- caps_applied: list[CapApplied] = Field(default_factory=list)
-
- @property
- def capped(self) -> bool:
- return bool(self.caps_applied)
-
-
-def _finding_tokens(f: GateFinding) -> str:
- """Searchable lower-cased text for a finding: data.kind / data.code / message / target."""
- bits = [str(f.data.get("kind", "")), str(f.data.get("code", "")), f.message, f.target or ""]
- return " ".join(bits).lower()
-
-
-def _finding_matches(rule: CapRule, report: GateReport, f: GateFinding) -> bool:
- if not fnmatch(report.gate, rule.gate) and not fnmatch(f.gate, rule.gate):
- return False
- if _SEV_RANK[f.severity] < _SEV_RANK[rule.severity]:
- return False
- if rule.kinds:
- text = _finding_tokens(f)
- return any(k.lower() in text for k in rule.kinds)
- return True
-
-
-def missing_must_items(
- acceptance_items: list[AcceptanceItem], acceptance_results: dict[str, bool]
-) -> list[str]:
- """Ids of ``must`` acceptance items that are not verified (missing counts as failed)."""
- return [
- a.id
- for a in acceptance_items
- if a.priority == "must" and not acceptance_results.get(a.id, False)
- ]
-
-
-def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) -> bool:
- """True when a deterministic gate MEASURED the thing this checklist defect claims, and found nothing.
-
- A checklist defect is a VLM's reading of a picture. Some of them — a floating part, an
- interpenetration — are also what a gate measures on the exported mesh, and the rubric
- already says which: a ``when="gate"`` cap rule with the same id and a ``kinds`` list.
- When every gate that rule watches ran and passed with no ERROR finding of those kinds,
- the measurement contradicts the perception, and the perception must not cap.
-
- Measured 2026-08-26 on a plan-pinned pair (fancy_v1 gas_street_lamp): the connectivity
- gate said "all 9 parts connected, gap <= 2 mm"; the judge read the dark seam under the
- pedestal as "floating in mid-air, a clear daylight gap", marked floating_part present, and
- ``defect:floating_part`` capped the run at 0.6 (uncapped 0.72) against 0.96 for a sibling
- the eye cannot tell apart. CLAUDE.md law 3: gates decide geometry, the VLM is perception.
-
- A defect with no matching gate rule (wrong_object, missing_named_part …) is never
- overridden: nothing measured it. A gate that did not run overrides nothing either.
- """
- rule = next(
- (r for r in rubric.caps if r.when == "gate" and r.id == defect_id and r.kinds), None
- )
- if rule is None:
- return False
- watched = [g for g in gates if fnmatch(g.gate, rule.gate)]
- if not watched:
- return False
- for g in watched:
- if not g.passed:
- return False
- for f in g.findings:
- if f.severity == Severity.ERROR and any(
- k in (f.message or "").lower() for k in rule.kinds
- ):
- return False
- return True
-
-
-def veto_measured_defects(
- rubric: Rubric, defects: dict[str, bool], gates: list[GateReport]
-) -> tuple[dict[str, bool], list[str]]:
- """``defects`` with every measured-absent one switched off, plus the ids switched off."""
- overridden = [did for did, on in defects.items() if on and measured_absent(rubric, did, gates)]
- return {did: (on and did not in overridden) for did, on in defects.items()}, overridden
-
-
-def apply_caps(
- rubric: Rubric,
- overall: float,
- gates: list[GateReport],
- acceptance_results: dict[str, bool],
- acceptance_items: list[AcceptanceItem] | None = None,
- *,
- console_errors: list[str] | None = None,
- views: list[RenderView] | None = None,
- defects_present: dict[str, bool] | None = None,
-) -> CapResult:
- """Apply every rubric cap rule; the overall becomes ``min(overall, caps...)``.
-
- Only the first matching finding per rule is recorded as evidence (one ledger
- line per rule), but all rules are evaluated. ``defects_present`` (checklist
- id → True) adds one ``defect:<id>`` ledger line per present defect that
- carries a ``cap``.
- """
- applied: list[CapApplied] = []
- for rule in rubric.caps:
- evidence = _rule_evidence(
- rule,
- gates,
- acceptance_results,
- acceptance_items or [],
- console_errors or [],
- views or [],
- )
- if evidence is not None:
- applied.append(CapApplied(rule=rule.id, cap=rule.cap, evidence=evidence))
- for did, present in (defects_present or {}).items():
- if not present:
- continue
- try:
- item = rubric.defect(did)
- except KeyError:
- continue
- if item.cap is not None:
- applied.append(
- CapApplied(
- rule=f"defect:{did}",
- cap=item.cap,
- evidence=f"judge checklist: {item.text[:120]}",
- )
- )
- capped = overall
- for a in applied:
- capped = min(capped, a.cap)
- return CapResult(overall=max(0.0, min(1.0, capped)), caps_applied=applied)
-
-
-def _rule_evidence(
- rule: CapRule,
- gates: list[GateReport],
- acceptance_results: dict[str, bool],
- acceptance_items: list[AcceptanceItem],
- console_errors: list[str],
- views: list[RenderView],
-) -> str | None:
- if rule.when == "missing_views":
- tokens = [k.lower() for k in rule.kinds]
- if any(any(t in f"{v.name} {v.mode}".lower() for t in tokens) for v in views):
- return None
- return f"no render view matching {rule.kinds} among {len(views)} view(s)"
- if rule.when == "acceptance":
- missing = missing_must_items(acceptance_items, acceptance_results)
- if missing:
- return f"must items not verified: {', '.join(missing[:6])}"
- return None
- if rule.when == "console":
- if console_errors:
- return f"{len(console_errors)} console error(s); first: {console_errors[0][:160]}"
- return None
- for report in gates:
- for f in report.findings:
- if _finding_matches(rule, report, f):
- target = f" [{f.target}]" if f.target else ""
- return f"{report.gate}{target}: {f.message[:200]}"
- return None
diff --git a/harness/codeverse/judges/images.py b/harness/codeverse/judges/images.py
deleted file mode 100644
index 564b51b4..00000000
--- a/harness/codeverse/judges/images.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""Judge image preparation: downscale, label strips, caching, view geometry.
-
-Every image sent to a judge goes through ``prepare_image`` so the token cost is
-bounded (≤ ``max_px`` on the long side) and every view carries a burnt-in label
-(``VIEW 3/8 — front · az 0° el 8°``) that the model can cite as evidence.
-Prepared files are cached under ``<cache_dir>/<sha>.png`` keyed by source path,
-mtime, size, max_px and label text.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import math
-import os
-import threading
-from pathlib import Path
-
-from PIL import Image, ImageDraw
-
-from codeverse.config import get_settings
-from codeverse.contracts.artifacts import RenderView
-from codeverse.contracts.chat import ImagePart
-from codeverse.conventions import OBJECT_VIEWS, SCENE_VIEWS, ViewPreset
-from codeverse.spatial.sheet import load_font
-
-_PRESETS: dict[str, ViewPreset] = {v.name: v for v in (*OBJECT_VIEWS, *SCENE_VIEWS)}
-
-
-class JudgeImageError(FileNotFoundError):
- """A render referenced by a RenderSet is missing or unreadable."""
-
-
-def default_cache_dir() -> Path:
- return get_settings().cache_dir / "judge_images"
-
-
-def view_az_el(view: RenderView) -> tuple[float, float] | None:
- """Azimuth/elevation (deg) of a view: from the preset name, else from camera geometry (Y-up)."""
- if view.name in _PRESETS:
- p = _PRESETS[view.name]
- return p.azimuth_deg, p.elevation_deg
- if view.camera_position is not None:
- cx, cy, cz = view.camera_position
- lx, ly, lz = view.look_at or (0.0, 0.0, 0.0)
- dx, dy, dz = cx - lx, cy - ly, cz - lz
- horiz = math.hypot(dx, dz)
- if horiz < 1e-9 and abs(dy) < 1e-9:
- return None
- az = math.degrees(math.atan2(dx, dz)) % 360.0 # 0 = +Z front, CCW from above
- el = math.degrees(math.atan2(dy, horiz))
- return round(az, 1), round(el, 1)
- return None
-
-
-def _cache_key(src: Path, max_px: int, label: str) -> str:
- st = src.stat()
- h = hashlib.sha1(f"{src.resolve()}|{st.st_mtime_ns}|{st.st_size}|{max_px}|{label}".encode())
- return h.hexdigest()[:20]
-
-
-def prepare_image(
- src: str | Path, *, label: str = "", max_px: int = 768, cache_dir: Path | None = None
-) -> Path:
- """Downscale ``src`` to ≤ ``max_px`` on its long side and burn ``label`` into a top strip.
-
- Returns the cached PNG path. Raises ``JudgeImageError`` if the source is missing.
- """
- src = Path(src)
- if not src.is_file():
- raise JudgeImageError(f"judge image missing: {src}")
- cache = Path(cache_dir) if cache_dir else default_cache_dir()
- cache.mkdir(parents=True, exist_ok=True)
- out = cache / f"{_cache_key(src, max_px, label)}.png"
- if out.is_file():
- return out
- with Image.open(src) as im:
- im = im.convert("RGB")
- w, h = im.size
- scale = min(1.0, max_px / max(w, h))
- if scale < 1.0:
- im = im.resize((max(1, round(w * scale)), max(1, round(h * scale))), Image.LANCZOS)
- if label:
- im = _with_label_strip(im, label)
- tmp = out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png") # unique per writer: concurrent judges share the cache
- im.save(tmp, format="PNG", optimize=True)
- tmp.replace(out) # atomic; a concurrent identical write simply wins last
- return out
-
-
-def _with_label_strip(im: Image.Image, label: str) -> Image.Image:
- w, h = im.size
- strip_h = max(22, int(h * 0.055))
- font = load_font(max(12, int(strip_h * 0.62)), bold=True)
- canvas = Image.new("RGB", (w, h + strip_h), (18, 18, 22))
- canvas.paste(im, (0, strip_h))
- draw = ImageDraw.Draw(canvas)
- text = label
- # truncate to fit
- while text and draw.textlength(text, font=font) > w - 12 and len(text) > 8:
- text = text[:-2]
- font_px = getattr(font, "size", 11)
- draw.text((6, max(2, (strip_h - font_px) // 2)), text, fill=(245, 245, 240), font=font)
- return canvas
-
-
-def image_part(path: Path, label: str) -> ImagePart:
- return ImagePart(path=str(path), mime="image/png", label=label)
diff --git a/harness/codeverse/judges/montage.py b/harness/codeverse/judges/montage.py
deleted file mode 100644
index 3e8ba01a..00000000
--- a/harness/codeverse/judges/montage.py
+++ /dev/null
@@ -1,274 +0,0 @@
-"""Judge montages: ≤ 2×2 labelled grids built from a ``RenderSet``.
-
-Research finding applied here: VLM judges get position-biased and saturate when
-shown many tiles, so every judge image carries **at most four views**. A judge
-call sees, in priority order and capped at ``max_montages``:
-
-1. ``shaded`` primary — the 4 most informative shaded views (3/4 views, top,
- underside for objects; authored cameras for scenes);
-2. ``poses`` — the articulation pose sheet (harness-made) or a 2×2 of ``pose_*``
- views, when the render set has them;
-3. ``geometry`` — a 2×2 of clay / normals renders (from ``geometry_views`` or from
- non-shaded views inside the set) so holes, inverted faces and intersections
- are visible without shading/material noise;
-4. ``shaded`` secondary — the remaining shaded views (then a 2×2 of ``pose_*``
- views when a pose sheet was already shown);
-
-plus 0–2 ``detail`` crops (centre of the best 3/4 view at 2×; the ground-contact
-band of the lowest view). ``shuffle_montages`` permutes montage order *and* tile
-order per n-sample so judge noise from position bias averages out.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import os
-import random
-import threading
-from dataclasses import dataclass, replace
-from pathlib import Path
-from typing import Literal
-
-from codeverse.contracts.artifacts import RenderSet, RenderView
-from codeverse.conventions import SCENE_VIEWS
-from codeverse.judges.images import JudgeImageError, default_cache_dir, view_az_el
-from codeverse.spatial.sheet import crop_region, montage_2x2
-
-MontageKind = Literal["shaded", "geometry", "poses", "pose_sheet", "detail"]
-
-#: modes that show geometry without material/lighting noise
-GEOMETRY_MODES = ("clay", "normals", "wire", "silhouette", "depth")
-#: most-informative-first order for the object rig (names from conventions.OBJECT_VIEWS)
-OBJECT_RANK = ("front_right_34", "back_left_34", "top", "low_front_left", "front", "right", "back", "left")
-#: scene rig: authored cameras first (graded for composition), then the overview rig.
-#: The rig names come from ``conventions.SCENE_VIEWS`` — anything else in a scene render
-#: set is a camera the SCENE authored, whatever it is called (they are PascalCase plan
-#: names such as ``Establishing``, never a fixed prefix).
-SCENE_OVERVIEW_RANK = ("overview_front_right", "overview_back_left", "overview_top", "eye_front", "eye_right", "eye_back_left")
-SCENE_RIG_NAMES = frozenset(v.name for v in SCENE_VIEWS) | frozenset(SCENE_OVERVIEW_RANK)
-POSE_PREFIX = "pose_"
-POSE_SHEET_NAMES = ("articulation_sheet", "pose_sheet")
-_POSITIONS = {
- 1: ("single",),
- 2: ("left", "right"),
- 3: ("top-left", "top-right", "bottom-left"),
- 4: ("top-left", "top-right", "bottom-left", "bottom-right"),
-}
-MAX_TILES = 4
-
-
-@dataclass(frozen=True)
-class Montage:
- """One judge image: 1–4 tiles (or a pass-through sheet / a detail crop)."""
-
- kind: MontageKind
- tiles: tuple[RenderView, ...]
- title: str
- passthrough: bool = False
- crop: tuple[float, float, float, float] | None = None
- hint: str = ""
-
- @property
- def is_detail(self) -> bool:
- return self.kind == "detail"
-
-
-# --------------------------------------------------------------------------- view grouping / ranking
-def is_pose_view(v: RenderView) -> bool:
- return v.name.startswith(POSE_PREFIX)
-
-
-def is_pose_sheet(v: RenderView) -> bool:
- return v.name in POSE_SHEET_NAMES
-
-
-def is_geometry_view(v: RenderView) -> bool:
- return v.mode in GEOMETRY_MODES
-
-
-def rank_views(views: list[RenderView], *, scene: bool) -> list[RenderView]:
- """Most informative first; unknown names keep their given order after the known ones."""
- if scene:
- def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:
- i, v = iv
- if v.name not in SCENE_RIG_NAMES:
- return (0, 0, i) # the scene's OWN cameras: the pictures being graded
- if v.name in SCENE_OVERVIEW_RANK:
- return (1, SCENE_OVERVIEW_RANK.index(v.name), i)
- return (2, 0, i)
- else:
- def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:
- i, v = iv
- return (0, OBJECT_RANK.index(v.name), i) if v.name in OBJECT_RANK else (1, 0, i)
- return [v for _, v in sorted(enumerate(views), key=key)]
-
-
-def _chunks(views: list[RenderView], n: int) -> list[list[RenderView]]:
- return [views[i : i + n] for i in range(0, len(views), n)]
-
-
-def plan_montages(
- renders: RenderSet,
- *,
- geometry_views: RenderSet | None = None,
- scene: bool = False,
- max_montages: int = 3,
- detail_crops: int = 2,
-) -> list[Montage]:
- """Decide which montages a judge call gets (see module docstring for the priority)."""
- shaded = [v for v in renders.views if not (is_pose_view(v) or is_pose_sheet(v) or is_geometry_view(v))]
- poses = [v for v in renders.views if is_pose_view(v)]
- sheets = [v for v in renders.views if is_pose_sheet(v)]
- geometry = [v for v in renders.views if is_geometry_view(v)]
- if geometry_views is not None:
- geometry = list(geometry_views.views) + geometry
- shaded = rank_views(shaded, scene=scene)
- shaded_groups = _chunks(shaded, MAX_TILES)
-
- candidates: list[Montage] = []
- if shaded_groups:
- candidates.append(Montage("shaded", tuple(shaded_groups[0]), "SHADED views"))
- if sheets:
- candidates.append(Montage("pose_sheet", (sheets[0],), "POSE SHEET (rows = poses, columns = views; tile labels name the joint and its value)", passthrough=True))
- elif poses:
- candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))
- if geometry:
- geo = _match_geometry(geometry, candidates[0].tiles if candidates else ())
- candidates.append(Montage("geometry", tuple(geo), f"GEOMETRY-ONLY views ({geo[0].mode}: no materials — look for holes, black/inverted faces, intersections, gaps)"))
- for extra in shaded_groups[1:]:
- candidates.append(Montage("shaded", tuple(extra), "SHADED views (remaining)"))
- if sheets and poses: # the sheet already shows the poses; the 2×2 is a larger-tile duplicate, lowest priority
- candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))
- chosen = candidates[: max(0, max_montages)]
- chosen.extend(plan_detail_crops(shaded, scene=scene)[: max(0, detail_crops)])
- return chosen
-
-
-def _match_geometry(geometry: list[RenderView], primary: tuple[RenderView, ...]) -> list[RenderView]:
- """Prefer geometry views whose names match the primary shaded montage, then the rest."""
- names = [v.name for v in primary]
- by_name = {v.name: v for v in geometry}
- out = [by_name[n] for n in names if n in by_name]
- out += [v for v in geometry if v not in out]
- return out[:MAX_TILES]
-
-
-def plan_detail_crops(shaded: list[RenderView], *, scene: bool) -> list[Montage]:
- """Centre crop of the best view at 2× (detail legibility) + ground band of the lowest view (objects only)."""
- if not shaded:
- return []
- out = [Montage("detail", (shaded[0],), f"DETAIL CROP — centre of {shaded[0].name} at 2×",
- crop=(0.25, 0.25, 0.75, 0.75),
- hint="look for bevels, seams, gaps and surface artefacts; judge detail at this scale")]
- if scene:
- return out
- lowest = min(shaded, key=lambda v: (view_az_el(v) or (0.0, 90.0))[1])
- if lowest is not shaded[0] or len(shaded) == 1:
- out.append(Montage("detail", (lowest,), f"DETAIL CROP — ground-contact band of {lowest.name}",
- crop=(0.0, 0.55, 1.0, 1.0),
- hint="look for feet/legs that do not reach the ground and parts hovering above their support"))
- return out
-
-
-# --------------------------------------------------------------------------- shuffling
-def shuffle_montages(montages: list[Montage], seed: int | None) -> list[Montage]:
- """Permute montage order and the tile order inside each grid; detail crops stay last."""
- if seed is None:
- return list(montages)
- rng = random.Random(seed)
- grids = [m for m in montages if not m.is_detail]
- details = [m for m in montages if m.is_detail]
- rng.shuffle(grids)
- out = []
- for m in grids:
- if m.passthrough or len(m.tiles) < 2:
- out.append(m)
- continue
- tiles = list(m.tiles)
- rng.shuffle(tiles)
- out.append(replace(m, tiles=tuple(tiles)))
- return out + details
-
-
-# --------------------------------------------------------------------------- labels
-def tile_label(v: RenderView) -> str:
- bits = [v.name]
- azel = view_az_el(v)
- if azel is not None:
- bits.append(f"az {azel[0]:.0f}° el {azel[1]:.0f}°")
- if v.mode and v.mode != "shaded":
- bits.append(v.mode)
- if v.time_s is not None:
- bits.append(f"t={v.time_s:g}s")
- return " · ".join(bits)
-
-
-def montage_strip(m: Montage, index: int, total: int) -> str:
- """Short label burnt into the image's top strip (``MONTAGE 2/3 — SHADED views``)."""
- if m.is_detail:
- return f"{m.title} [{index}/{total}]"
- title = m.title.split(" (")[0]
- return f"MONTAGE {index}/{total} — {title}"
-
-
-def montage_label(m: Montage, index: int, total: int) -> str:
- """Full label for the text part: strip + tile layout (``…: top-left = front_right_34 · az 35° el 22°, …``)."""
- head = montage_strip(m, index, total)
- if m.is_detail:
- return head + (f" — {m.hint}" if m.hint else "")
- if m.passthrough:
- return f"MONTAGE {index}/{total} — {m.title}"
- pos = _POSITIONS[len(m.tiles)]
- return f"MONTAGE {index}/{total} — {m.title}: " + ", ".join(f"{p} = {tile_label(v)}" for p, v in zip(pos, m.tiles, strict=True))
-
-
-# --------------------------------------------------------------------------- rendering
-def _key(parts: list[str]) -> str:
- return hashlib.sha1("|".join(parts).encode()).hexdigest()[:20]
-
-
-def _stat(p: str) -> str:
- path = Path(p)
- if not path.is_file():
- raise JudgeImageError(f"judge image missing: {path}")
- st = path.stat()
- return f"{path.resolve()}|{st.st_mtime_ns}|{st.st_size}"
-
-
-def _tmp(out: Path) -> Path:
- return out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png")
-
-
-def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int = 512) -> Path:
- """Build (or fetch from cache) the PNG for one montage; pass-through sheets return their source.
-
- Writes go through a per-writer temp file + atomic replace so concurrent
- judges sharing a cache directory never read a half-written PNG.
- """
- if m.passthrough:
- _stat(m.tiles[0].path)
- return Path(m.tiles[0].path)
- cache = Path(cache_dir) if cache_dir else default_cache_dir()
- cache.mkdir(parents=True, exist_ok=True)
- if m.is_detail:
- v = m.tiles[0]
- out = cache / f"crop_{_key([_stat(v.path), str(m.crop), str(tile_px)])}.png"
- if not out.is_file():
- tmp = crop_region(v.path, _tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)
- tmp.replace(out)
- return out
- images = [(tile_label(v), v.path) for v in m.tiles]
- out = cache / f"montage_{_key([_stat(p) + '|' + lbl for lbl, p in images] + [str(tile_px)])}.png"
- if not out.is_file():
- tmp = montage_2x2(images, _tmp(out), tile=tile_px)
- tmp.replace(out)
- return out
-
-
-def describe_montages(montages: list[Montage]) -> str:
- """One line per image for the VIEW RIG paragraph (what each image is, in send order)."""
- grids = [m for m in montages if not m.is_detail]
- details = [m for m in montages if m.is_detail]
- lines = [f"- image {i}: {m.title}" for i, m in enumerate(grids, 1)]
- lines += [f"- detail crop {i}: {m.title.replace('DETAIL CROP — ', '')}" for i, m in enumerate(details, 1)]
- return "\n".join(lines)
diff --git a/harness/codeverse/judges/output_schema.py b/harness/codeverse/judges/output_schema.py
deleted file mode 100644
index 79840f4a..00000000
--- a/harness/codeverse/judges/output_schema.py
+++ /dev/null
@@ -1,216 +0,0 @@
-"""The judge's wire format: a pydantic model built per rubric (+ acceptance ids).
-
-The model forces the VLM to score EVERY visual criterion and to answer EVERY
-acceptance item; ``overall`` is deliberately absent — it is computed in code.
-Parsing is lenient about shape (dict-of-records or list-of-records, fenced
-JSON in plain text) but strict about content: a missing criterion is a
-``JudgeParseError`` so the caller can retry instead of scoring a hole as 0.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-from pydantic import BaseModel, Field, ValidationError, create_model
-
-from codeverse.contracts.judgment import ImprovementItem, JudgeIssue
-from codeverse.judges.rubrics import Rubric
-from codeverse.models.schema_utils import JsonParseError, parse_json_lenient
-
-
-class JudgeParseError(ValueError):
- """The model's reply could not be turned into a complete JudgeOutput."""
-
-
-class CriterionScore(BaseModel):
- score: float = Field(ge=0.0, le=1.0, description="0..1 against the anchors")
- evidence: str = Field(description="what you saw, citing VIEW labels / measurements")
-
-
-class AcceptanceVerdict(BaseModel):
- verified: bool = Field(description="true only if the renders/measurements prove it")
- evidence: str = Field(default="", description="which view / number proves or disproves it")
-
-
-class DefectVerdict(BaseModel):
- present: bool = Field(description="true only if the defect is VISIBLE in the images or stated by a gate/measurement")
- evidence: str = Field(default="", description="which image / tile / number shows it (or why it is absent)")
-
-
-class JudgeOutput(BaseModel):
- """Provider-neutral parsed judge reply (criteria + acceptance + defect checklist as dicts)."""
-
- criteria: dict[str, CriterionScore]
- summary: str = ""
- strengths: list[str] = Field(default_factory=list)
- issues: list[JudgeIssue] = Field(default_factory=list)
- improvement_plan: list[ImprovementItem] = Field(default_factory=list)
- acceptance: dict[str, AcceptanceVerdict] = Field(default_factory=dict)
- defects: dict[str, bool] = Field(default_factory=dict, description="checklist id → defect present")
- defect_evidence: dict[str, str] = Field(default_factory=dict)
-
- @property
- def scores(self) -> dict[str, float]:
- return {k: v.score for k, v in self.criteria.items()}
-
- @property
- def acceptance_bools(self) -> dict[str, bool]:
- return {k: v.verified for k, v in self.acceptance.items()}
-
-
-def build_wire_model(rubric: Rubric, acceptance_ids: list[str]) -> type[BaseModel]:
- """A pydantic model whose ``criteria`` / ``acceptance`` objects have FIXED keys.
-
- Criterion ids are valid identifiers (enforced by the rubric). Acceptance ids
- are arbitrary strings → fields ``acc_<i>`` aliased to the id.
- """
- crit_fields: dict[str, Any] = {
- c.id: (CriterionScore, Field(description=f"{c.label}: {c.description.strip()[:200]}"))
- for c in rubric.visual_criteria()
- }
- Criteria = create_model("Criteria", **crit_fields)
- # Field order = generation order for JSON-mode models: observe (summary, strengths, issues, defect
- # checklist, acceptance) BEFORE scoring, so the scores follow the evidence rather than the reverse.
- fields: dict[str, Any] = {
- "summary": (str, Field(description="2-4 sentences: what it is, what is right, what is most wrong")),
- "strengths": (list[str], Field(default_factory=list, description="≤ 5 short bullets")),
- "issues": (list[JudgeIssue], Field(default_factory=list, description="observable defects, most severe first")),
- }
- if rubric.defects:
- def_fields: dict[str, Any] = {
- d.id: (DefectVerdict, Field(description=d.text.strip()[:200])) for d in rubric.defects
- }
- Defects = create_model("Defects", **def_fields)
- fields["defects"] = (Defects, Field(description="binary checklist: one entry per defect id, present=true ONLY when seen"))
- if acceptance_ids:
- acc_fields: dict[str, Any] = {
- f"acc_{i}": (AcceptanceVerdict, Field(alias=aid, description=f"acceptance item {aid}"))
- for i, aid in enumerate(acceptance_ids)
- }
- Acceptance = create_model("Acceptance", **acc_fields)
- fields["acceptance"] = (Acceptance, Field(description="one entry per acceptance item id"))
- fields["criteria"] = (Criteria, Field(description="one entry per criterion id, scored AFTER the observations above"))
- fields["improvement_plan"] = (
- list[ImprovementItem],
- Field(default_factory=list, description="≤ 6 concrete instructions for the builder, priority 1 first"),
- )
- return create_model("JudgeReply", **fields)
-
-
-def wire_schema(rubric: Rubric, acceptance_ids: list[str]) -> dict[str, Any]:
- return build_wire_model(rubric, acceptance_ids).model_json_schema(by_alias=True)
-
-
-# --------------------------------------------------------------------------- parsing
-def _records_to_dict(value: Any, key_names: tuple[str, ...]) -> dict[str, Any]:
- """Accept ``{id: {...}}`` or ``[{id: ..., ...}, ...]`` and return ``{id: {...}}``."""
- if isinstance(value, dict):
- return value
- if isinstance(value, list):
- out: dict[str, Any] = {}
- for rec in value:
- if not isinstance(rec, dict):
- raise JudgeParseError(f"record is not an object: {rec!r}")
- key = next((rec[k] for k in key_names if k in rec), None)
- if key is None:
- raise JudgeParseError(f"record lacks an id field {key_names}: {rec!r}")
- out[str(key)] = {k: v for k, v in rec.items() if k not in key_names}
- return out
- raise JudgeParseError(f"expected object or list, got {type(value).__name__}")
-
-
-def _coerce_acceptance(value: Any) -> dict[str, Any]:
- recs = _records_to_dict(value, ("id", "item_id", "item"))
- out: dict[str, Any] = {}
- for k, v in recs.items():
- if isinstance(v, bool):
- out[k] = {"verified": v, "evidence": ""}
- elif isinstance(v, dict):
- vv = dict(v)
- if "verified" not in vv and "passed" in vv:
- vv["verified"] = vv.pop("passed")
- out[k] = vv
- else:
- raise JudgeParseError(f"acceptance item {k!r}: unexpected value {v!r}")
- return out
-
-
-def _coerce_defects(value: Any) -> tuple[dict[str, bool], dict[str, str]]:
- """``{id: bool}`` or ``{id: {present, evidence}}`` or a list of records → (present, evidence)."""
- if value is None:
- return {}, {}
- recs = _records_to_dict(value, ("id", "defect", "item"))
- present: dict[str, bool] = {}
- evidence: dict[str, str] = {}
- for k, v in recs.items():
- if isinstance(v, bool):
- present[k] = v
- elif isinstance(v, dict):
- flag = v.get("present", v.get("value", v.get("verified")))
- if not isinstance(flag, bool):
- raise JudgeParseError(f"defect {k!r}: 'present' must be a boolean, got {flag!r}")
- present[k] = flag
- evidence[k] = str(v.get("evidence", "") or "")
- else:
- raise JudgeParseError(f"defect item {k!r}: unexpected value {v!r}")
- return present, evidence
-
-
-def parse_judge_output(
- payload: Any, rubric: Rubric, acceptance_ids: list[str], *, measured_scores: dict[str, float] | None = None
-) -> JudgeOutput:
- """Validate a parsed reply (dict) or raw text into a complete ``JudgeOutput``.
-
- ``measured_scores`` (criterion id → score) are injected for ``kind: measured``
- criteria so the output always covers the full rubric.
- """
- if isinstance(payload, str):
- try:
- payload = parse_json_lenient(payload)
- except JsonParseError as e:
- raise JudgeParseError(f"no JSON object in reply: {e}") from e
- if not isinstance(payload, dict):
- raise JudgeParseError(f"reply is not a JSON object: {type(payload).__name__}")
- data = dict(payload)
- if "criteria" not in data and "per_criterion" in data:
- data["criteria"] = data.pop("per_criterion")
- try:
- criteria = _records_to_dict(data.get("criteria", {}), ("id", "criterion", "name"))
- criteria = {
- k: ({"score": v, "evidence": ""} if isinstance(v, (int, float)) else v) for k, v in criteria.items()
- }
- acceptance = _coerce_acceptance(data.get("acceptance", {}))
- defects, defect_evidence = _coerce_defects(data.get("defects", {}))
- out = JudgeOutput(
- criteria=criteria,
- summary=str(data.get("summary", "") or ""),
- strengths=[str(s) for s in data.get("strengths", []) or []],
- issues=data.get("issues", []) or [],
- improvement_plan=data.get("improvement_plan", []) or [],
- acceptance=acceptance,
- defects=defects,
- defect_evidence=defect_evidence,
- )
- except ValidationError as e:
- raise JudgeParseError(f"judge reply failed validation: {e}") from e
- measured = measured_scores or {}
- for cid, s in measured.items():
- out.criteria[cid] = CriterionScore(score=max(0.0, min(1.0, float(s))), evidence="measured in code")
- missing = [c.id for c in rubric.criteria if c.id not in out.criteria]
- if missing:
- raise JudgeParseError(f"judge reply missing criteria {missing}")
- extra = [k for k in out.criteria if k not in rubric.weights]
- for k in extra: # unknown criteria are ignored, not scored
- out.criteria.pop(k)
- missing_acc = [a for a in acceptance_ids if a not in out.acceptance]
- for a in missing_acc: # unanswered acceptance items count as NOT verified
- out.acceptance[a] = AcceptanceVerdict(verified=False, evidence="not answered by judge")
- known = {d.id for d in rubric.defects}
- for k in [k for k in out.defects if k not in known]: # unknown checklist ids are ignored
- out.defects.pop(k)
- out.defect_evidence.pop(k, None)
- for d in rubric.defects:
- if d.id not in out.defects: # unanswered defect → absent (the conservative direction for a penalty)
- out.defects[d.id] = False
- out.defect_evidence[d.id] = "not answered by judge"
- return out
diff --git a/harness/codeverse/judges/pairwise.py b/harness/codeverse/judges/pairwise.py
index 4ee50688..3290c47a 100644
--- a/harness/codeverse/judges/pairwise.py
+++ b/harness/codeverse/judges/pairwise.py
@@ -27,11 +27,16 @@ from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart, TextPa
from codeverse.contracts.common import Usage
from codeverse.contracts.spec import Spec
from codeverse.fanout import fan_out
-from codeverse.judges.images import image_part, prepare_image
-from codeverse.judges.montage import montage_label, plan_montages, render_montage
-from codeverse.judges.output_schema import JudgeParseError
-from codeverse.judges.prompt_builder import MONTAGE_TILE_PX, brief_section
-from codeverse.judges.rubrics import Rubric, load_rubric
+from codeverse.judges.prompt_builder import (
+ MONTAGE_TILE_PX,
+ brief_section,
+ image_part,
+ montage_label,
+ plan_montages,
+ prepare_image,
+ render_montage,
+)
+from codeverse.judges.rubrics import JudgeParseError, Rubric, load_rubric
from codeverse.models.base import ChatModel, ModelError
from codeverse.models.schema_utils import JsonParseError, parse_json_lenient
diff --git a/harness/codeverse/judges/prompt_builder.py b/harness/codeverse/judges/prompt_builder.py
index 5ae63081..14f3aa3c 100644
--- a/harness/codeverse/judges/prompt_builder.py
+++ b/harness/codeverse/judges/prompt_builder.py
@@ -1,38 +1,364 @@
-"""Build the judge conversation: system (role + rubric + defect checklist +
-output rules) and ONE user message (brief, plan digest, acceptance checklist,
-measurement table, gate digest, previous-verdict framing, view-rig paragraph,
-then the labelled images).
-
-Images are ≤ 2×2 **montages** (see ``montage.py``): at most ``max_montages``
-grids (shaded primary, pose sheet, geometry-only, shaded secondary) plus 0–2
-detail crops, every tile labelled, whole-image label strips burnt in. Text is
-budgeted to ≈6k tokens. The builder's code and reasoning are never included
-(blind judge).
+"""Judge image preparation: downscale, label strips, caching, view geometry.
+
+Every image sent to a judge goes through ``prepare_image`` so the token cost is
+bounded (≤ ``max_px`` on the long side) and every view carries a burnt-in label
+(``VIEW 3/8 — front · az 0° el 8°``) that the model can cite as evidence.
+Prepared files are cached under ``<cache_dir>/<sha>.png`` keyed by source path,
+mtime, size, max_px and label text.
"""
from __future__ import annotations
+import hashlib
+import math
+import os
+import random
+import threading
+from dataclasses import dataclass, replace
from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
+
+from PIL import Image, ImageDraw
-from codeverse.contracts.artifacts import GateReport, Measurement, RenderSet, Severity
+from codeverse.config import get_settings
+from codeverse.contracts.artifacts import GateReport, Measurement, RenderSet, RenderView, Severity
from codeverse.contracts.chat import ChatMessage, ImagePart, TextPart
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.spec import Spec
-from codeverse.judges.images import image_part, prepare_image
-from codeverse.judges.montage import (
- Montage,
- describe_montages,
- montage_label,
- montage_strip,
- plan_montages,
- render_montage,
- shuffle_montages,
-)
+from codeverse.conventions import OBJECT_VIEWS, SCENE_VIEWS, ViewPreset
from codeverse.judges.rubrics import Rubric
from codeverse.spatial.measure import measure_summary_table
+from codeverse.spatial.sheet import crop_region, load_font, montage_2x2
+
+_PRESETS: dict[str, ViewPreset] = {v.name: v for v in (*OBJECT_VIEWS, *SCENE_VIEWS)}
+
+
+class JudgeImageError(FileNotFoundError):
+ """A render referenced by a RenderSet is missing or unreadable."""
+
+
+def default_cache_dir() -> Path:
+ return get_settings().cache_dir / "judge_images"
+
+
+def view_az_el(view: RenderView) -> tuple[float, float] | None:
+ """Azimuth/elevation (deg) of a view: from the preset name, else from camera geometry (Y-up)."""
+ if view.name in _PRESETS:
+ p = _PRESETS[view.name]
+ return p.azimuth_deg, p.elevation_deg
+ if view.camera_position is not None:
+ cx, cy, cz = view.camera_position
+ lx, ly, lz = view.look_at or (0.0, 0.0, 0.0)
+ dx, dy, dz = cx - lx, cy - ly, cz - lz
+ horiz = math.hypot(dx, dz)
+ if horiz < 1e-9 and abs(dy) < 1e-9:
+ return None
+ az = math.degrees(math.atan2(dx, dz)) % 360.0 # 0 = +Z front, CCW from above
+ el = math.degrees(math.atan2(dy, horiz))
+ return round(az, 1), round(el, 1)
+ return None
+
+
+def _cache_key(src: Path, max_px: int, label: str) -> str:
+ st = src.stat()
+ h = hashlib.sha1(f"{src.resolve()}|{st.st_mtime_ns}|{st.st_size}|{max_px}|{label}".encode())
+ return h.hexdigest()[:20]
+
+
+def prepare_image(
+ src: str | Path, *, label: str = "", max_px: int = 768, cache_dir: Path | None = None
+) -> Path:
+ """Downscale ``src`` to ≤ ``max_px`` on its long side and burn ``label`` into a top strip.
+
+ Returns the cached PNG path. Raises ``JudgeImageError`` if the source is missing.
+ """
+ src = Path(src)
+ if not src.is_file():
+ raise JudgeImageError(f"judge image missing: {src}")
+ cache = Path(cache_dir) if cache_dir else default_cache_dir()
+ cache.mkdir(parents=True, exist_ok=True)
+ out = cache / f"{_cache_key(src, max_px, label)}.png"
+ if out.is_file():
+ return out
+ with Image.open(src) as im:
+ im = im.convert("RGB")
+ w, h = im.size
+ scale = min(1.0, max_px / max(w, h))
+ if scale < 1.0:
+ im = im.resize((max(1, round(w * scale)), max(1, round(h * scale))), Image.LANCZOS)
+ if label:
+ im = _with_label_strip(im, label)
+ tmp = out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png") # unique per writer: concurrent judges share the cache
+ im.save(tmp, format="PNG", optimize=True)
+ tmp.replace(out) # atomic; a concurrent identical write simply wins last
+ return out
+
+
+def _with_label_strip(im: Image.Image, label: str) -> Image.Image:
+ w, h = im.size
+ strip_h = max(22, int(h * 0.055))
+ font = load_font(max(12, int(strip_h * 0.62)), bold=True)
+ canvas = Image.new("RGB", (w, h + strip_h), (18, 18, 22))
+ canvas.paste(im, (0, strip_h))
+ draw = ImageDraw.Draw(canvas)
+ text = label
+ # truncate to fit
+ while text and draw.textlength(text, font=font) > w - 12 and len(text) > 8:
+ text = text[:-2]
+ font_px = getattr(font, "size", 11)
+ draw.text((6, max(2, (strip_h - font_px) // 2)), text, fill=(245, 245, 240), font=font)
+ return canvas
+
+
+def image_part(path: Path, label: str) -> ImagePart:
+ return ImagePart(path=str(path), mime="image/png", label=label)
+
+
+# ===================================================================== montage
+# (merged from codeverse/judges/montage.py, 2026-08-28)
+MontageKind = Literal["shaded", "geometry", "poses", "pose_sheet", "detail"]
+
+#: modes that show geometry without material/lighting noise
+GEOMETRY_MODES = ("clay", "normals", "wire", "silhouette", "depth")
+#: most-informative-first order for the object rig (names from conventions.OBJECT_VIEWS)
+OBJECT_RANK = ("front_right_34", "back_left_34", "top", "low_front_left", "front", "right", "back", "left")
+#: scene rig: authored cameras first (graded for composition), then the overview rig.
+#: The rig names come from ``conventions.SCENE_VIEWS`` — anything else in a scene render
+#: set is a camera the SCENE authored, whatever it is called (they are PascalCase plan
+#: names such as ``Establishing``, never a fixed prefix).
+SCENE_OVERVIEW_RANK = ("overview_front_right", "overview_back_left", "overview_top", "eye_front", "eye_right", "eye_back_left")
+SCENE_RIG_NAMES = frozenset(v.name for v in SCENE_VIEWS) | frozenset(SCENE_OVERVIEW_RANK)
+POSE_PREFIX = "pose_"
+POSE_SHEET_NAMES = ("articulation_sheet", "pose_sheet")
+_POSITIONS = {
+ 1: ("single",),
+ 2: ("left", "right"),
+ 3: ("top-left", "top-right", "bottom-left"),
+ 4: ("top-left", "top-right", "bottom-left", "bottom-right"),
+}
+MAX_TILES = 4
+
+
+@dataclass(frozen=True)
+class Montage:
+ """One judge image: 1–4 tiles (or a pass-through sheet / a detail crop)."""
+
+ kind: MontageKind
+ tiles: tuple[RenderView, ...]
+ title: str
+ passthrough: bool = False
+ crop: tuple[float, float, float, float] | None = None
+ hint: str = ""
+
+ @property
+ def is_detail(self) -> bool:
+ return self.kind == "detail"
+
+
+# --------------------------------------------------------------------------- view grouping / ranking
+def is_pose_view(v: RenderView) -> bool:
+ return v.name.startswith(POSE_PREFIX)
+
+
+def is_pose_sheet(v: RenderView) -> bool:
+ return v.name in POSE_SHEET_NAMES
+
+
+def is_geometry_view(v: RenderView) -> bool:
+ return v.mode in GEOMETRY_MODES
+
+
+def rank_views(views: list[RenderView], *, scene: bool) -> list[RenderView]:
+ """Most informative first; unknown names keep their given order after the known ones."""
+ if scene:
+ def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:
+ i, v = iv
+ if v.name not in SCENE_RIG_NAMES:
+ return (0, 0, i) # the scene's OWN cameras: the pictures being graded
+ if v.name in SCENE_OVERVIEW_RANK:
+ return (1, SCENE_OVERVIEW_RANK.index(v.name), i)
+ return (2, 0, i)
+ else:
+ def key(iv: tuple[int, RenderView]) -> tuple[int, int, int]:
+ i, v = iv
+ return (0, OBJECT_RANK.index(v.name), i) if v.name in OBJECT_RANK else (1, 0, i)
+ return [v for _, v in sorted(enumerate(views), key=key)]
+
+
+def _chunks(views: list[RenderView], n: int) -> list[list[RenderView]]:
+ return [views[i : i + n] for i in range(0, len(views), n)]
+
+
+def plan_montages(
+ renders: RenderSet,
+ *,
+ geometry_views: RenderSet | None = None,
+ scene: bool = False,
+ max_montages: int = 3,
+ detail_crops: int = 2,
+) -> list[Montage]:
+ """Decide which montages a judge call gets (see module docstring for the priority)."""
+ shaded = [v for v in renders.views if not (is_pose_view(v) or is_pose_sheet(v) or is_geometry_view(v))]
+ poses = [v for v in renders.views if is_pose_view(v)]
+ sheets = [v for v in renders.views if is_pose_sheet(v)]
+ geometry = [v for v in renders.views if is_geometry_view(v)]
+ if geometry_views is not None:
+ geometry = list(geometry_views.views) + geometry
+ shaded = rank_views(shaded, scene=scene)
+ shaded_groups = _chunks(shaded, MAX_TILES)
+
+ candidates: list[Montage] = []
+ if shaded_groups:
+ candidates.append(Montage("shaded", tuple(shaded_groups[0]), "SHADED views"))
+ if sheets:
+ candidates.append(Montage("pose_sheet", (sheets[0],), "POSE SHEET (rows = poses, columns = views; tile labels name the joint and its value)", passthrough=True))
+ elif poses:
+ candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))
+ if geometry:
+ geo = _match_geometry(geometry, candidates[0].tiles if candidates else ())
+ candidates.append(Montage("geometry", tuple(geo), f"GEOMETRY-ONLY views ({geo[0].mode}: no materials — look for holes, black/inverted faces, intersections, gaps)"))
+ for extra in shaded_groups[1:]:
+ candidates.append(Montage("shaded", tuple(extra), "SHADED views (remaining)"))
+ if sheets and poses: # the sheet already shows the poses; the 2×2 is a larger-tile duplicate, lowest priority
+ candidates.append(Montage("poses", tuple(poses[:MAX_TILES]), "POSED views (joints moved by the harness; tile label = pose)"))
+ chosen = candidates[: max(0, max_montages)]
+ chosen.extend(plan_detail_crops(shaded, scene=scene)[: max(0, detail_crops)])
+ return chosen
+
+
+def _match_geometry(geometry: list[RenderView], primary: tuple[RenderView, ...]) -> list[RenderView]:
+ """Prefer geometry views whose names match the primary shaded montage, then the rest."""
+ names = [v.name for v in primary]
+ by_name = {v.name: v for v in geometry}
+ out = [by_name[n] for n in names if n in by_name]
+ out += [v for v in geometry if v not in out]
+ return out[:MAX_TILES]
+
+
+def plan_detail_crops(shaded: list[RenderView], *, scene: bool) -> list[Montage]:
+ """Centre crop of the best view at 2× (detail legibility) + ground band of the lowest view (objects only)."""
+ if not shaded:
+ return []
+ out = [Montage("detail", (shaded[0],), f"DETAIL CROP — centre of {shaded[0].name} at 2×",
+ crop=(0.25, 0.25, 0.75, 0.75),
+ hint="look for bevels, seams, gaps and surface artefacts; judge detail at this scale")]
+ if scene:
+ return out
+ lowest = min(shaded, key=lambda v: (view_az_el(v) or (0.0, 90.0))[1])
+ if lowest is not shaded[0] or len(shaded) == 1:
+ out.append(Montage("detail", (lowest,), f"DETAIL CROP — ground-contact band of {lowest.name}",
+ crop=(0.0, 0.55, 1.0, 1.0),
+ hint="look for feet/legs that do not reach the ground and parts hovering above their support"))
+ return out
+
+
+# --------------------------------------------------------------------------- shuffling
+def shuffle_montages(montages: list[Montage], seed: int | None) -> list[Montage]:
+ """Permute montage order and the tile order inside each grid; detail crops stay last."""
+ if seed is None:
+ return list(montages)
+ rng = random.Random(seed)
+ grids = [m for m in montages if not m.is_detail]
+ details = [m for m in montages if m.is_detail]
+ rng.shuffle(grids)
+ out = []
+ for m in grids:
+ if m.passthrough or len(m.tiles) < 2:
+ out.append(m)
+ continue
+ tiles = list(m.tiles)
+ rng.shuffle(tiles)
+ out.append(replace(m, tiles=tuple(tiles)))
+ return out + details
+
+
+# --------------------------------------------------------------------------- labels
+def tile_label(v: RenderView) -> str:
+ bits = [v.name]
+ azel = view_az_el(v)
+ if azel is not None:
+ bits.append(f"az {azel[0]:.0f}° el {azel[1]:.0f}°")
+ if v.mode and v.mode != "shaded":
+ bits.append(v.mode)
+ if v.time_s is not None:
+ bits.append(f"t={v.time_s:g}s")
+ return " · ".join(bits)
+
+
+def montage_strip(m: Montage, index: int, total: int) -> str:
+ """Short label burnt into the image's top strip (``MONTAGE 2/3 — SHADED views``)."""
+ if m.is_detail:
+ return f"{m.title} [{index}/{total}]"
+ title = m.title.split(" (")[0]
+ return f"MONTAGE {index}/{total} — {title}"
+
+
+def montage_label(m: Montage, index: int, total: int) -> str:
+ """Full label for the text part: strip + tile layout (``…: top-left = front_right_34 · az 35° el 22°, …``)."""
+ head = montage_strip(m, index, total)
+ if m.is_detail:
+ return head + (f" — {m.hint}" if m.hint else "")
+ if m.passthrough:
+ return f"MONTAGE {index}/{total} — {m.title}"
+ pos = _POSITIONS[len(m.tiles)]
+ return f"MONTAGE {index}/{total} — {m.title}: " + ", ".join(f"{p} = {tile_label(v)}" for p, v in zip(pos, m.tiles, strict=True))
+
+
+# --------------------------------------------------------------------------- rendering
+def _key(parts: list[str]) -> str:
+ return hashlib.sha1("|".join(parts).encode()).hexdigest()[:20]
+
+
+def _stat(p: str) -> str:
+ path = Path(p)
+ if not path.is_file():
+ raise JudgeImageError(f"judge image missing: {path}")
+ st = path.stat()
+ return f"{path.resolve()}|{st.st_mtime_ns}|{st.st_size}"
+
+
+def _tmp(out: Path) -> Path:
+ return out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png")
+
+
+def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int = 512) -> Path:
+ """Build (or fetch from cache) the PNG for one montage; pass-through sheets return their source.
+
+ Writes go through a per-writer temp file + atomic replace so concurrent
+ judges sharing a cache directory never read a half-written PNG.
+ """
+ if m.passthrough:
+ _stat(m.tiles[0].path)
+ return Path(m.tiles[0].path)
+ cache = Path(cache_dir) if cache_dir else default_cache_dir()
+ cache.mkdir(parents=True, exist_ok=True)
+ if m.is_detail:
+ v = m.tiles[0]
+ out = cache / f"crop_{_key([_stat(v.path), str(m.crop), str(tile_px)])}.png"
+ if not out.is_file():
+ tmp = crop_region(v.path, _tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)
+ tmp.replace(out)
+ return out
+ images = [(tile_label(v), v.path) for v in m.tiles]
+ out = cache / f"montage_{_key([_stat(p) + '|' + lbl for lbl, p in images] + [str(tile_px)])}.png"
+ if not out.is_file():
+ tmp = montage_2x2(images, _tmp(out), tile=tile_px)
+ tmp.replace(out)
+ return out
+
+
+def describe_montages(montages: list[Montage]) -> str:
+ """One line per image for the VIEW RIG paragraph (what each image is, in send order)."""
+ grids = [m for m in montages if not m.is_detail]
+ details = [m for m in montages if m.is_detail]
+ lines = [f"- image {i}: {m.title}" for i, m in enumerate(grids, 1)]
+ lines += [f"- detail crop {i}: {m.title.replace('DETAIL CROP — ', '')}" for i, m in enumerate(details, 1)]
+ return "\n".join(lines)
+
+# ===================================================================== prompt_builder
+# (merged from codeverse/judges/prompt_builder.py, 2026-08-28)
if TYPE_CHECKING: # pragma: no cover
from codeverse.judges.base import JudgeInput
@@ -248,7 +574,7 @@ def judge_prompt_hash(rubric: Rubric) -> str:
"""
import json
- from codeverse.judges.output_schema import wire_schema
+ from codeverse.judges.rubrics import wire_schema
from codeverse.prompts import prompt_hash
payload = "\n".join([
diff --git a/harness/codeverse/judges/reference.py b/harness/codeverse/judges/reference.py
deleted file mode 100644
index 8c41bb51..00000000
--- a/harness/codeverse/judges/reference.py
+++ /dev/null
@@ -1,358 +0,0 @@
-"""``ReferenceJudge``: image-conditioned judging (rubric ``reference_v1``).
-
-Three things the base judge does not do:
-
-* the spec's reference images go in BEFORE the renders, labelled ``REFERENCE k``
- (and labelled **SYNTHESIZED** when they were generated by
- :mod:`codeverse.reference`, so the judge knows the target is not ground truth);
-* ``silhouette_match`` is scored IN CODE from the front-render-vs-reference
- outline IoU (:func:`codeverse.spatial.silhouette.compare_silhouette`) — the VLM
- only scores the perceptual criteria;
-* one extra vision pass (:func:`codeverse.reference.mismatch.compare`) names the
- CONCRETE differences — missing feature, wrong part count, wrong proportion —
- and puts them in front of the scoring call, so the verdict's issues and
- improvement plan (and therefore the next round's refine tasks) are specific
- instead of "looks blocky". Set ``diff=False`` (or ``CV3D_REFERENCE_DIFF=off``)
- to skip that call; it is fail-soft either way.
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import re
-from collections.abc import Callable
-from pathlib import Path
-from typing import Any
-
-from codeverse.contracts.artifacts import RenderView
-from codeverse.judges.base import JudgeInput
-from codeverse.judges.rubrics import Rubric
-from codeverse.judges.vlm_judge import JudgeContext, VlmJudge
-
-log = logging.getLogger(__name__)
-
-SilhouetteFn = Callable[[str, str], dict[str, Any]]
-
-#: IoU → score mapping: ≤ 0.25 → 0, ≥ 0.85 → 1, linear in between (matches the rubric anchors).
-IOU_LOW, IOU_HIGH = 0.25, 0.85
-
-#: score the measured criterion gets when the measurement cannot be trusted
-#: (unreliable mask, or a reference whose proportions contradict the brief)
-NEUTRAL_SCORE = 0.5
-
-#: views handed to the mismatch pass (front-ish first, then a 3/4 and a side)
-DIFF_VIEW_NAMES: tuple[str, ...] = ("front", "front_right_34", "front_left_34", "right", "left")
-_ENV_DIFF = "CV3D_REFERENCE_DIFF"
-
-
-def iou_to_score(iou: float) -> float:
- return max(0.0, min(1.0, (float(iou) - IOU_LOW) / (IOU_HIGH - IOU_LOW)))
-
-
-def _default_silhouette_fn() -> SilhouetteFn | None:
- try:
- from codeverse.spatial.silhouette import compare_silhouette # type: ignore[attr-defined]
- except (ImportError, AttributeError):
- return None
- return compare_silhouette
-
-
-def _diff_enabled(flag: bool) -> bool:
- return flag and os.environ.get(_ENV_DIFF, "on").strip().lower() not in ("0", "off", "false", "no")
-
-
-def _conflict_note(info: dict[str, Any]) -> str:
- try:
- from codeverse.reference import conflict_note
-
- return conflict_note(info)
- except ImportError: # pragma: no cover
- return ""
-
-
-def _is_synth(note: str) -> bool:
- return note.startswith("SYNTHESIZED")
-
-
-class ReferenceJudge(VlmJudge):
- """VlmJudge + reference images + measured silhouette + named mismatches."""
-
- name = "reference"
-
- def __init__(
- self,
- model_id: str | None = None,
- n_samples: int = 1,
- temperature: float = 0.2,
- *,
- rubric: str | Rubric = "reference_v1",
- silhouette_fn: SilhouetteFn | None = None,
- front_view_names: tuple[str, ...] = ("front", "front_right_34"),
- best_view: bool = True,
- diff: bool = True,
- diff_model: Any | None = None,
- diff_model_id: str = "",
- **kwargs: Any,
- ):
- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)
- self.silhouette_fn = silhouette_fn or _default_silhouette_fn()
- self.front_view_names = front_view_names
- self.best_view = best_view
- self.diff = diff
- self._diff_model = diff_model
- self.diff_model_id = diff_model_id
- #: the last diff computed (tests / callers that want the mismatches themselves)
- self.last_diff: Any | None = None
-
- # ------------------------------------------------------------------ API
- def judge(self, inp: JudgeInput, **kwargs: Any) -> Any:
- """``VlmJudge.judge`` plus the mismatch pass's own spend.
-
- The diff is a real model call made from :meth:`context`; folding its
- ``Usage`` into the verdict keeps ``BudgetGuard`` and ``record.total_usage``
- honest (the ledger already sees it through the metered chat model)."""
- verdict = super().judge(inp, **kwargs)
- diff = self.last_diff
- if diff is not None and getattr(diff, "usage", None) is not None:
- verdict.usage = verdict.usage + diff.usage
- return verdict
-
- # ------------------------------------------------------------------ hook
- def context(self, inp: JudgeInput) -> JudgeContext:
- refs = [r for r in inp.spec.references if Path(r.path).is_file()]
- synth = bool(refs) and all(_is_synth(r.note) for r in refs)
- images: list[tuple[str, str]] = []
- for i, r in enumerate(refs[:3], 1):
- tag = "SYNTHESIZED " if _is_synth(r.note) else ""
- note = f" — {r.note}" if r.note else ""
- images.append((f"{tag}REFERENCE {i}/{min(len(refs), 3)} ({r.role}){note}", r.path))
- blocks: list[str] = []
- if synth:
- blocks.append(
- "NOTE ON THE REFERENCE: it was SYNTHESIZED from the brief by an image model, not photographed. "
- "It is a shape/part-inventory target only. Where it disagrees with the brief or the stated "
- "dimensions, the BRIEF is correct and the render must follow the brief.")
- conflict = self.dimension_conflict(inp, refs)
- note = _conflict_note(conflict)
- if note:
- blocks.append(note)
- measured_ids = [c.id for c in self.rubric.measured_criteria()]
- info: dict[str, Any] = {}
- measured_scores: dict[str, float] = {}
- if measured_ids:
- info = self.measure_silhouette(inp)
- if "iou" not in info:
- raise ReferenceJudgeError(
- f"cannot score measured criteria {measured_ids}: {info.get('error', 'no iou')}")
- score, why = self.silhouette_score(info, conflict)
- measured_scores = {cid: score for cid in measured_ids}
- blocks.append(
- f"MEASURED SILHOUETTE (harness): {info['render']} render vs reference {info['reference']}: "
- f"IoU {info['iou']:.3f} → silhouette_match score {score:.2f}{why}."
- + (f" extra: {info['extra']}" if info.get("extra") else ""))
- elif refs:
- info = self.measure_silhouette(inp)
- diff = self.reference_diff(inp, refs, info, synthesized=synth) if refs else None
- self.last_diff = diff
- if diff is not None and diff.as_text():
- blocks.append(diff.as_text())
- return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images)
-
- # ------------------------------------------------------------------ proportion guard
- def dimension_conflict(self, inp: JudgeInput, refs: list[Any]) -> dict[str, Any]:
- """Does the target reference's own outline contradict the brief's dimensions?
-
- When it does, no numeric proportion signal derived from that picture may be
- used against the object — the brief wins (see
- :mod:`codeverse.reference.proportions`).
- """
- targets = [r for r in refs if r.role == "target"] or refs
- if not targets:
- return {"conflict": False}
- try:
- from codeverse.reference import dimension_conflict
-
- return dimension_conflict(inp.spec, targets[0].path)
- except Exception as e: # noqa: BLE001 — advisory
- log.warning("dimension-conflict check failed: %s", e)
- return {"conflict": False, "error": str(e)}
-
- def silhouette_score(self, info: dict[str, Any], conflict: dict[str, Any]) -> tuple[float, str]:
- """IoU → measured score, NEUTRAL when the reference contradicts the brief or the
- mask is unreliable. Returns ``(score, ' (why)')``."""
- if conflict.get("conflict"):
- return NEUTRAL_SCORE, " (NEUTRAL: the reference's proportions contradict the brief's dimensions)"
- if info.get("reliable") is False:
- return NEUTRAL_SCORE, " (NEUTRAL: the background mask was unreliable)"
- return iou_to_score(info["iou"]), ""
-
- # ------------------------------------------------------------------ mismatch pass
- def reference_diff(self, inp: JudgeInput, refs: list[Any], info: dict[str, Any], *, synthesized: bool) -> Any | None:
- """One extra vision call naming concrete mismatches. ``None`` when disabled;
- never raises (a failed diff simply contributes no text)."""
- if not _diff_enabled(self.diff):
- return None
- try:
- from codeverse.reference import compare
- except ImportError: # pragma: no cover - the package is part of the wheel
- return None
- targets = [r.path for r in refs if r.role == "target"] or [r.path for r in refs]
- renders = self.diff_views(list(inp.renders.views))
- measured = dict(info) if "iou" in info else None
- if measured is not None:
- measured.setdefault("view", info.get("render", ""))
- try:
- return compare(inp.spec, targets, renders, model=self._pick_diff_model(),
- part_names=_plan_part_names(inp), measured=measured, synthesized=synthesized)
- except Exception as e: # noqa: BLE001 — an advisory pass must never fail a verdict
- log.warning("reference diff pass failed: %s", e)
- return None
-
- def _pick_diff_model(self) -> Any:
- """``diff_model`` > ``diff_model_id`` (a cheaper perception model) > the judge's own model."""
- if self._diff_model is not None:
- return self._diff_model
- if self.diff_model_id:
- from codeverse.models import get_chat_model
-
- self._diff_model = get_chat_model(self.diff_model_id)
- return self._diff_model
- return self.model
-
- def diff_views(self, views: list[RenderView]) -> list[str]:
- """Up to 3 shaded views for the diff, front-ish first."""
- by_name = {v.name: v for v in views}
- picked = [by_name[n].path for n in DIFF_VIEW_NAMES if n in by_name]
- for v in views:
- if len(picked) >= 3:
- break
- if v.path not in picked:
- picked.append(v.path)
- return picked[:3]
-
- # ------------------------------------------------------------------ silhouette
- def pick_front_view(self, views: list[RenderView]) -> RenderView | None:
- for name in self.front_view_names:
- for v in views:
- if v.name == name:
- return v
- return views[0] if views else None
-
- def measure_silhouette(self, inp: JudgeInput) -> dict[str, Any]:
- """{iou, render, reference, extra} or {error}.
-
- With ``best_view=True`` (default) the IoU is taken from the render view that
- best matches the reference's camera instead of a hardcoded ``front``: a
- reference photograph has ONE camera, and comparing a three-quarter product
- shot to a straight-on elevation measures camera agreement, not shape. The
- winning view name travels in ``render`` so the number stays auditable.
- """
- targets = [r for r in inp.spec.references if r.role == "target" and Path(r.path).is_file()] or [
- r for r in inp.spec.references if Path(r.path).is_file()]
- if not targets:
- return {"error": "spec has no readable reference images"}
- views = list(inp.renders.views)
- if not views:
- return {"error": "render set has no views"}
- if self.silhouette_fn is None:
- return {"error": "compare_silhouette unavailable (codeverse.spatial.silhouette not importable)"}
- res, name = self._measure(views, targets[0].path)
- if not isinstance(res, dict) or "iou" not in res:
- return {"error": f"compare_silhouette returned no iou: {res!r}"}
- extra = {k: v for k, v in res.items() if k not in ("iou", "per_view") and isinstance(v, (int, float, str, bool))}
- out = {"iou": float(res["iou"]), "render": name, "reference": Path(targets[0].path).name,
- "reliable": bool(res.get("reliable", True)),
- "aspect_ratio_err": res.get("aspect_ratio_err"), "extra": extra}
- if res.get("per_view"):
- out["per_view"] = res["per_view"]
- return out
-
- def _measure(self, views: list[RenderView], reference: str) -> tuple[dict[str, Any], str]:
- """(result, view name). Best-matching view when enabled and available."""
- if self.best_view and self.silhouette_fn is _default_silhouette_fn():
- try:
- from codeverse.spatial.silhouette import best_view_match
- except ImportError: # pragma: no cover
- best_view_match = None # type: ignore[assignment]
- if best_view_match is not None:
- res = best_view_match(views, reference)
- if "iou" in res:
- return res, str(res.get("view", ""))
- view = self.pick_front_view(views)
- if view is None: # pragma: no cover - guarded by the caller
- return {}, ""
- return self.silhouette_fn(view.path, reference), view.name # type: ignore[misc]
-
-
-def _plan_part_names(inp: JudgeInput) -> list[str]:
- """Part names from the plan digest the judge already receives (best effort).
-
- ``judges.replay_input.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;
- a multi-line digest lists one ``- Name · role · …`` per line. Both are handled, and
- an unrecognised digest simply yields no names (the diff prompt then says so).
- """
- text = getattr(inp, "plan_summary", "") or ""
- names: list[str] = []
- for chunk in re.split(r"(?i)\bparts\s*:", text)[1:]:
- for raw in chunk.split("\n")[0].split(","):
- names.append(raw.split("\u00d7")[0].split("(")[0].strip().rstrip("."))
- for line in text.splitlines():
- line = line.strip()
- if not line.startswith(("-", "*", "\u00b7")):
- continue
- head = line.lstrip("-*\u00b7 ").split("\u00b7")[0].split("(")[0].split(":")[0].strip()
- names.append(head)
- seen: list[str] = []
- for n in names:
- if n and " " not in n and 2 < len(n) <= 40 and n[:1].isupper() and n not in seen:
- seen.append(n)
- return seen[:40]
-
-
-
-LIKENESS_NOTE = (
- "REAL-WORLD REFERENCE PHOTOS are attached (labelled REAL-WORLD REFERENCE n/N). They are not a "
- "composition to copy; they are what the thing the brief names ACTUALLY looks like. Score "
- "brief fidelity, colour/light and richness against them: dominant colour and where the "
- "secondary colours sit, how the structure folds / layers / thins out, where the brightness "
- "concentrates and how much of the frame stays dark, what the real texture is at fine scale. A "
- "frame that ticks every noun in the brief but would never be mistaken for the photographed "
- "thing (a row of evenly spaced bars standing in for curtains, a flat band standing in for a "
- "gradient of light, cartoon-saturated colour where the photo is subtle) is a 0.4-0.5 on brief "
- "fidelity, not a 1.0. Foreground, framing and landscape in the photos are incidental unless "
- "the brief asks for them."
-)
-
-
-class LikenessJudge(VlmJudge):
- """``VlmJudge`` + the reference photos of the REAL thing — for tracks with no silhouette.
-
- ``ReferenceJudge`` is built for objects: it measures a front-view silhouette IoU against
- the target photo and runs a part-inventory mismatch pass. Neither means anything for a
- fragment shader or a scene, where a reference photo answers a different question — does
- this LOOK like the thing? Measured 2026-08-26 (teaser aurora, three versions, flash and
- codex): every version scored 0.78-0.94 with an empty issues list while none resembled an
- aurora (a comb of straight teal bars; pink cotton-wool lobes), because the brief is a
- checklist of nouns and the rubric had nothing to say about likeness. This judge attaches
- up to three photos beside the frames with :data:`LIKENESS_NOTE` and leaves the rubric alone.
- """
-
- name = "likeness"
-
- def __init__(self, model_id: str | None = None, n_samples: int = 1, temperature: float = 0.2, *,
- rubric: str | Rubric = "shader_v2", max_refs: int = 3, **kwargs: Any):
- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)
- self.max_refs = max(1, int(max_refs))
-
- def context(self, inp: JudgeInput) -> JudgeContext:
- if self.rubric.measured_criteria():
- raise ValueError(f"rubric {self.rubric.name} has measured criteria; LikenessJudge measures nothing")
- refs = [r for r in inp.spec.references if Path(r.path).is_file()][: self.max_refs]
- images = [(f"REAL-WORLD REFERENCE {i}/{len(refs)}" + (f" — {r.note}" if r.note else ""), r.path)
- for i, r in enumerate(refs, 1)]
- return JudgeContext(extra_text=LIKENESS_NOTE if refs else "", extra_images=images)
-
-
-class ReferenceJudgeError(RuntimeError):
- """The reference judge cannot compute its measured criteria."""
diff --git a/harness/codeverse/judges/replay_input.py b/harness/codeverse/judges/replay_input.py
deleted file mode 100644
index cdf4cdd0..00000000
--- a/harness/codeverse/judges/replay_input.py
+++ /dev/null
@@ -1,82 +0,0 @@
-"""Judge-replay inputs shared by ``3dcv judge`` (``cli/_judge.py``) and
-``judges/calibration.py``.
-
-Both re-judge a stored round from ``rounds/rNN.json`` + ``plan.json`` and must hand
-the judge what the in-run judge saw. The pure pieces of that reconstruction live
-here so the two replayers cannot drift (no cli / tracks / flywheel imports —
-``judges/`` stays below all of them):
-
-* ``plan_digest`` — was ``judges/calibration.py:plan_digest``;
- ``cli/_judge.py:plan_summary_for`` lazily imported it from there, a cli→calibration
- edge for a pure function.
-* ``resolve_paths`` — was ``cli/_judge.py:resolve_paths``; calibration fed the stored
- render paths straight in, so a moved or archived run replayed against the host it
- was produced on.
-* ``judged_subset`` — was ``cli/_judge.py:judged_subset``; calibration judged every
- stored view, so whenever a track stamps ``RenderView.judge`` its numbers were not
- those of the in-run judge or ``3dcv judge``.
-
-The pieces that need ``RunRecord`` / ``tracks.get_track`` / ``flywheel.record``
-(``previous``, ``extra_context``, clay views) stay in ``cli/_judge.py``: nothing in
-``judges/`` imports those packages and they all import ``judges/``.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-from codeverse.contracts.artifacts import RenderSet
-from codeverse.workspace import Workspace
-
-
-def plan_digest(plan: dict[str, Any]) -> str:
- """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""
- bits: list[str] = []
- title = plan.get("object_name") or plan.get("title") or ""
- if title:
- bits.append(f"{title}: {plan.get('summary', '')}".strip())
- bbox = plan.get("overall_bbox") or {}
- if isinstance(bbox, dict) and bbox.get("extents"):
- e = bbox["extents"]
- bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")
- parts = plan.get("parts") or []
- if parts:
- bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")
- if plan.get("root_link"):
- bits.append(f"Root link {plan['root_link']}.")
- joints = plan.get("joints") or []
- if joints:
- bits.append("Joints: " + "; ".join(
- f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"
- for j in joints) + ".")
- for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):
- items = plan.get(key) or []
- if items:
- bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")
- if plan.get("setting"):
- bits.append(f"Setting: {plan['setting']}.")
- return " ".join(bits)
-
-
-def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:
- """Render paths out of a round record, resolved against THIS workspace.
-
- Delegates to ``Workspace.rebase``: this used to rebase only paths for which
- ``is_absolute()`` was False, which made it a no-op against every record the harness
- itself writes (they are all absolute) — so a moved or archived run kept pointing at
- the host it was produced on.
- """
- if rs is None:
- return None
- fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]
- sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet
- out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir
- return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})
-
-
-def judged_subset(rs: RenderSet | None) -> RenderSet | None:
- """The views the in-run judge actually saw: the per-view ``judge`` flags
- stamped at render time; legacy rounds (no flags) keep every stored view."""
- if rs is None or not any(v.judge is not None for v in rs.views):
- return rs
- return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})
diff --git a/harness/codeverse/judges/rubrics.py b/harness/codeverse/judges/rubrics.py
index 491c5008..2767fade 100644
--- a/harness/codeverse/judges/rubrics.py
+++ b/harness/codeverse/judges/rubrics.py
@@ -21,14 +21,27 @@ from __future__ import annotations
import hashlib
import json
import re
+import statistics
+from fnmatch import fnmatch
from functools import lru_cache
from pathlib import Path
-from typing import Literal
+from typing import Any, Literal
import yaml
-from pydantic import BaseModel, Field, field_validator, model_validator
-
-from codeverse.contracts.artifacts import Severity
+from pydantic import (
+ BaseModel,
+ Field,
+ ValidationError,
+ create_model,
+ field_validator,
+ model_validator,
+)
+
+from codeverse.contracts.artifacts import GateFinding, GateReport, RenderView, Severity
+from codeverse.contracts.common import Usage
+from codeverse.contracts.judgment import ImprovementItem, JudgeIssue, Judgment
+from codeverse.contracts.plan import AcceptanceItem
+from codeverse.models.schema_utils import JsonParseError, parse_json_lenient
RUBRICS_DIR = Path(__file__).resolve().parent / "rubrics"
ANCHOR_LEVELS = ("1.0", "0.7", "0.4", "0.1")
@@ -211,3 +224,638 @@ def load_rubric(name: str) -> Rubric:
raise RubricError(f"rubric file {path} must be a mapping")
data.setdefault("name", path.stem)
return rubric_from_dict(data, name_hint=path.stem)
+
+
+# ===================================================================== output_schema
+# (merged from codeverse/judges/output_schema.py, 2026-08-28)
+class JudgeParseError(ValueError):
+ """The model's reply could not be turned into a complete JudgeOutput."""
+
+
+class CriterionScore(BaseModel):
+ score: float = Field(ge=0.0, le=1.0, description="0..1 against the anchors")
+ evidence: str = Field(description="what you saw, citing VIEW labels / measurements")
+
+
+class AcceptanceVerdict(BaseModel):
+ verified: bool = Field(description="true only if the renders/measurements prove it")
+ evidence: str = Field(default="", description="which view / number proves or disproves it")
+
+
+class DefectVerdict(BaseModel):
+ present: bool = Field(description="true only if the defect is VISIBLE in the images or stated by a gate/measurement")
+ evidence: str = Field(default="", description="which image / tile / number shows it (or why it is absent)")
+
+
+class JudgeOutput(BaseModel):
+ """Provider-neutral parsed judge reply (criteria + acceptance + defect checklist as dicts)."""
+
+ criteria: dict[str, CriterionScore]
+ summary: str = ""
+ strengths: list[str] = Field(default_factory=list)
+ issues: list[JudgeIssue] = Field(default_factory=list)
+ improvement_plan: list[ImprovementItem] = Field(default_factory=list)
+ acceptance: dict[str, AcceptanceVerdict] = Field(default_factory=dict)
+ defects: dict[str, bool] = Field(default_factory=dict, description="checklist id → defect present")
+ defect_evidence: dict[str, str] = Field(default_factory=dict)
+
+ @property
+ def scores(self) -> dict[str, float]:
+ return {k: v.score for k, v in self.criteria.items()}
+
+ @property
+ def acceptance_bools(self) -> dict[str, bool]:
+ return {k: v.verified for k, v in self.acceptance.items()}
+
+
+def build_wire_model(rubric: Rubric, acceptance_ids: list[str]) -> type[BaseModel]:
+ """A pydantic model whose ``criteria`` / ``acceptance`` objects have FIXED keys.
+
+ Criterion ids are valid identifiers (enforced by the rubric). Acceptance ids
+ are arbitrary strings → fields ``acc_<i>`` aliased to the id.
+ """
+ crit_fields: dict[str, Any] = {
+ c.id: (CriterionScore, Field(description=f"{c.label}: {c.description.strip()[:200]}"))
+ for c in rubric.visual_criteria()
+ }
+ Criteria = create_model("Criteria", **crit_fields)
+ # Field order = generation order for JSON-mode models: observe (summary, strengths, issues, defect
+ # checklist, acceptance) BEFORE scoring, so the scores follow the evidence rather than the reverse.
+ fields: dict[str, Any] = {
+ "summary": (str, Field(description="2-4 sentences: what it is, what is right, what is most wrong")),
+ "strengths": (list[str], Field(default_factory=list, description="≤ 5 short bullets")),
+ "issues": (list[JudgeIssue], Field(default_factory=list, description="observable defects, most severe first")),
+ }
+ if rubric.defects:
+ def_fields: dict[str, Any] = {
+ d.id: (DefectVerdict, Field(description=d.text.strip()[:200])) for d in rubric.defects
+ }
+ Defects = create_model("Defects", **def_fields)
+ fields["defects"] = (Defects, Field(description="binary checklist: one entry per defect id, present=true ONLY when seen"))
+ if acceptance_ids:
+ acc_fields: dict[str, Any] = {
+ f"acc_{i}": (AcceptanceVerdict, Field(alias=aid, description=f"acceptance item {aid}"))
+ for i, aid in enumerate(acceptance_ids)
+ }
+ Acceptance = create_model("Acceptance", **acc_fields)
+ fields["acceptance"] = (Acceptance, Field(description="one entry per acceptance item id"))
+ fields["criteria"] = (Criteria, Field(description="one entry per criterion id, scored AFTER the observations above"))
+ fields["improvement_plan"] = (
+ list[ImprovementItem],
+ Field(default_factory=list, description="≤ 6 concrete instructions for the builder, priority 1 first"),
+ )
+ return create_model("JudgeReply", **fields)
+
+
+def wire_schema(rubric: Rubric, acceptance_ids: list[str]) -> dict[str, Any]:
+ return build_wire_model(rubric, acceptance_ids).model_json_schema(by_alias=True)
+
+
+# --------------------------------------------------------------------------- parsing
+def _records_to_dict(value: Any, key_names: tuple[str, ...]) -> dict[str, Any]:
+ """Accept ``{id: {...}}`` or ``[{id: ..., ...}, ...]`` and return ``{id: {...}}``."""
+ if isinstance(value, dict):
+ return value
+ if isinstance(value, list):
+ out: dict[str, Any] = {}
+ for rec in value:
+ if not isinstance(rec, dict):
+ raise JudgeParseError(f"record is not an object: {rec!r}")
+ key = next((rec[k] for k in key_names if k in rec), None)
+ if key is None:
+ raise JudgeParseError(f"record lacks an id field {key_names}: {rec!r}")
+ out[str(key)] = {k: v for k, v in rec.items() if k not in key_names}
+ return out
+ raise JudgeParseError(f"expected object or list, got {type(value).__name__}")
+
+
+def _coerce_acceptance(value: Any) -> dict[str, Any]:
+ recs = _records_to_dict(value, ("id", "item_id", "item"))
+ out: dict[str, Any] = {}
+ for k, v in recs.items():
+ if isinstance(v, bool):
+ out[k] = {"verified": v, "evidence": ""}
+ elif isinstance(v, dict):
+ vv = dict(v)
+ if "verified" not in vv and "passed" in vv:
+ vv["verified"] = vv.pop("passed")
+ out[k] = vv
+ else:
+ raise JudgeParseError(f"acceptance item {k!r}: unexpected value {v!r}")
+ return out
+
+
+def _coerce_defects(value: Any) -> tuple[dict[str, bool], dict[str, str]]:
+ """``{id: bool}`` or ``{id: {present, evidence}}`` or a list of records → (present, evidence)."""
+ if value is None:
+ return {}, {}
+ recs = _records_to_dict(value, ("id", "defect", "item"))
+ present: dict[str, bool] = {}
+ evidence: dict[str, str] = {}
+ for k, v in recs.items():
+ if isinstance(v, bool):
+ present[k] = v
+ elif isinstance(v, dict):
+ flag = v.get("present", v.get("value", v.get("verified")))
+ if not isinstance(flag, bool):
+ raise JudgeParseError(f"defect {k!r}: 'present' must be a boolean, got {flag!r}")
+ present[k] = flag
+ evidence[k] = str(v.get("evidence", "") or "")
+ else:
+ raise JudgeParseError(f"defect item {k!r}: unexpected value {v!r}")
+ return present, evidence
+
+
+def parse_judge_output(
+ payload: Any, rubric: Rubric, acceptance_ids: list[str], *, measured_scores: dict[str, float] | None = None
+) -> JudgeOutput:
+ """Validate a parsed reply (dict) or raw text into a complete ``JudgeOutput``.
+
+ ``measured_scores`` (criterion id → score) are injected for ``kind: measured``
+ criteria so the output always covers the full rubric.
+ """
+ if isinstance(payload, str):
+ try:
+ payload = parse_json_lenient(payload)
+ except JsonParseError as e:
+ raise JudgeParseError(f"no JSON object in reply: {e}") from e
+ if not isinstance(payload, dict):
+ raise JudgeParseError(f"reply is not a JSON object: {type(payload).__name__}")
+ data = dict(payload)
+ if "criteria" not in data and "per_criterion" in data:
+ data["criteria"] = data.pop("per_criterion")
+ try:
+ criteria = _records_to_dict(data.get("criteria", {}), ("id", "criterion", "name"))
+ criteria = {
+ k: ({"score": v, "evidence": ""} if isinstance(v, (int, float)) else v) for k, v in criteria.items()
+ }
+ acceptance = _coerce_acceptance(data.get("acceptance", {}))
+ defects, defect_evidence = _coerce_defects(data.get("defects", {}))
+ out = JudgeOutput(
+ criteria=criteria,
+ summary=str(data.get("summary", "") or ""),
+ strengths=[str(s) for s in data.get("strengths", []) or []],
+ issues=data.get("issues", []) or [],
+ improvement_plan=data.get("improvement_plan", []) or [],
+ acceptance=acceptance,
+ defects=defects,
+ defect_evidence=defect_evidence,
+ )
+ except ValidationError as e:
+ raise JudgeParseError(f"judge reply failed validation: {e}") from e
+ measured = measured_scores or {}
+ for cid, s in measured.items():
+ out.criteria[cid] = CriterionScore(score=max(0.0, min(1.0, float(s))), evidence="measured in code")
+ missing = [c.id for c in rubric.criteria if c.id not in out.criteria]
+ if missing:
+ raise JudgeParseError(f"judge reply missing criteria {missing}")
+ extra = [k for k in out.criteria if k not in rubric.weights]
+ for k in extra: # unknown criteria are ignored, not scored
+ out.criteria.pop(k)
+ missing_acc = [a for a in acceptance_ids if a not in out.acceptance]
+ for a in missing_acc: # unanswered acceptance items count as NOT verified
+ out.acceptance[a] = AcceptanceVerdict(verified=False, evidence="not answered by judge")
+ known = {d.id for d in rubric.defects}
+ for k in [k for k in out.defects if k not in known]: # unknown checklist ids are ignored
+ out.defects.pop(k)
+ out.defect_evidence.pop(k, None)
+ for d in rubric.defects:
+ if d.id not in out.defects: # unanswered defect → absent (the conservative direction for a penalty)
+ out.defects[d.id] = False
+ out.defect_evidence[d.id] = "not answered by judge"
+ return out
+
+
+# ===================================================================== caps
+# (merged from codeverse/judges/caps.py, 2026-08-28)
+_SEV_RANK = {Severity.INFO: 0, Severity.WARN: 1, Severity.ERROR: 2}
+
+
+class CapApplied(BaseModel):
+ rule: str
+ cap: float
+ evidence: str = Field(description="which finding / acceptance item triggered it")
+
+
+class CapResult(BaseModel):
+ overall: float
+ caps_applied: list[CapApplied] = Field(default_factory=list)
+
+ @property
+ def capped(self) -> bool:
+ return bool(self.caps_applied)
+
+
+def _finding_tokens(f: GateFinding) -> str:
+ """Searchable lower-cased text for a finding: data.kind / data.code / message / target."""
+ bits = [str(f.data.get("kind", "")), str(f.data.get("code", "")), f.message, f.target or ""]
+ return " ".join(bits).lower()
+
+
+def _finding_matches(rule: CapRule, report: GateReport, f: GateFinding) -> bool:
+ if not fnmatch(report.gate, rule.gate) and not fnmatch(f.gate, rule.gate):
+ return False
+ if _SEV_RANK[f.severity] < _SEV_RANK[rule.severity]:
+ return False
+ if rule.kinds:
+ text = _finding_tokens(f)
+ return any(k.lower() in text for k in rule.kinds)
+ return True
+
+
+def missing_must_items(
+ acceptance_items: list[AcceptanceItem], acceptance_results: dict[str, bool]
+) -> list[str]:
+ """Ids of ``must`` acceptance items that are not verified (missing counts as failed)."""
+ return [
+ a.id
+ for a in acceptance_items
+ if a.priority == "must" and not acceptance_results.get(a.id, False)
+ ]
+
+
+def measured_absent(rubric: Rubric, defect_id: str, gates: list[GateReport]) -> bool:
+ """True when a deterministic gate MEASURED the thing this checklist defect claims, and found nothing.
+
+ A checklist defect is a VLM's reading of a picture. Some of them — a floating part, an
+ interpenetration — are also what a gate measures on the exported mesh, and the rubric
+ already says which: a ``when="gate"`` cap rule with the same id and a ``kinds`` list.
+ When every gate that rule watches ran and passed with no ERROR finding of those kinds,
+ the measurement contradicts the perception, and the perception must not cap.
+
+ Measured 2026-08-26 on a plan-pinned pair (fancy_v1 gas_street_lamp): the connectivity
+ gate said "all 9 parts connected, gap <= 2 mm"; the judge read the dark seam under the
+ pedestal as "floating in mid-air, a clear daylight gap", marked floating_part present, and
+ ``defect:floating_part`` capped the run at 0.6 (uncapped 0.72) against 0.96 for a sibling
+ the eye cannot tell apart. CLAUDE.md law 3: gates decide geometry, the VLM is perception.
+
+ A defect with no matching gate rule (wrong_object, missing_named_part …) is never
+ overridden: nothing measured it. A gate that did not run overrides nothing either.
+ """
+ rule = next(
+ (r for r in rubric.caps if r.when == "gate" and r.id == defect_id and r.kinds), None
+ )
+ if rule is None:
+ return False
+ watched = [g for g in gates if fnmatch(g.gate, rule.gate)]
+ if not watched:
+ return False
+ for g in watched:
+ if not g.passed:
+ return False
+ for f in g.findings:
+ if f.severity == Severity.ERROR and any(
+ k in (f.message or "").lower() for k in rule.kinds
+ ):
+ return False
+ return True
+
+
+def veto_measured_defects(
+ rubric: Rubric, defects: dict[str, bool], gates: list[GateReport]
+) -> tuple[dict[str, bool], list[str]]:
+ """``defects`` with every measured-absent one switched off, plus the ids switched off."""
+ overridden = [did for did, on in defects.items() if on and measured_absent(rubric, did, gates)]
+ return {did: (on and did not in overridden) for did, on in defects.items()}, overridden
+
+
+def apply_caps(
+ rubric: Rubric,
+ overall: float,
+ gates: list[GateReport],
+ acceptance_results: dict[str, bool],
+ acceptance_items: list[AcceptanceItem] | None = None,
+ *,
+ console_errors: list[str] | None = None,
+ views: list[RenderView] | None = None,
+ defects_present: dict[str, bool] | None = None,
+) -> CapResult:
+ """Apply every rubric cap rule; the overall becomes ``min(overall, caps...)``.
+
+ Only the first matching finding per rule is recorded as evidence (one ledger
+ line per rule), but all rules are evaluated. ``defects_present`` (checklist
+ id → True) adds one ``defect:<id>`` ledger line per present defect that
+ carries a ``cap``.
+ """
+ applied: list[CapApplied] = []
+ for rule in rubric.caps:
+ evidence = _rule_evidence(
+ rule,
+ gates,
+ acceptance_results,
+ acceptance_items or [],
+ console_errors or [],
+ views or [],
+ )
+ if evidence is not None:
+ applied.append(CapApplied(rule=rule.id, cap=rule.cap, evidence=evidence))
+ for did, present in (defects_present or {}).items():
+ if not present:
+ continue
+ try:
+ item = rubric.defect(did)
+ except KeyError:
+ continue
+ if item.cap is not None:
+ applied.append(
+ CapApplied(
+ rule=f"defect:{did}",
+ cap=item.cap,
+ evidence=f"judge checklist: {item.text[:120]}",
+ )
+ )
+ capped = overall
+ for a in applied:
+ capped = min(capped, a.cap)
+ return CapResult(overall=max(0.0, min(1.0, capped)), caps_applied=applied)
+
+
+def _rule_evidence(
+ rule: CapRule,
+ gates: list[GateReport],
+ acceptance_results: dict[str, bool],
+ acceptance_items: list[AcceptanceItem],
+ console_errors: list[str],
+ views: list[RenderView],
+) -> str | None:
+ if rule.when == "missing_views":
+ tokens = [k.lower() for k in rule.kinds]
+ if any(any(t in f"{v.name} {v.mode}".lower() for t in tokens) for v in views):
+ return None
+ return f"no render view matching {rule.kinds} among {len(views)} view(s)"
+ if rule.when == "acceptance":
+ missing = missing_must_items(acceptance_items, acceptance_results)
+ if missing:
+ return f"must items not verified: {', '.join(missing[:6])}"
+ return None
+ if rule.when == "console":
+ if console_errors:
+ return f"{len(console_errors)} console error(s); first: {console_errors[0][:160]}"
+ return None
+ for report in gates:
+ for f in report.findings:
+ if _finding_matches(rule, report, f):
+ target = f" [{f.target}]" if f.target else ""
+ return f"{report.gate}{target}: {f.message[:200]}"
+ return None
+
+
+# ===================================================================== scoring
+# (merged from codeverse/judges/scoring.py, 2026-08-28)
+class ScoreBreakdown(BaseModel):
+ """Everything the pass/fail decision used (serialised into ``Judgment.raw``)."""
+
+ status: str = Field(default="ok", description="ok | degraded")
+ per_criterion_mean: dict[str, float]
+ per_criterion_std: dict[str, float]
+ per_sample_overall: list[float]
+ overall_uncapped: float = Field(
+ description="rubric-weighted mean of the per-criterion means, before defects/caps"
+ )
+ overall_after_defects: float = Field(
+ default=0.0, description="overall_uncapped minus defect penalties"
+ )
+ overall: float
+ floors_hit: list[dict[str, Any]] = Field(default_factory=list)
+ caps: CapResult
+ must_missing: list[str] = Field(default_factory=list)
+ acceptance_votes: dict[str, list[bool]] = Field(default_factory=dict)
+ defects: dict[str, bool] = Field(
+ default_factory=dict, description="checklist id → present (majority vote)"
+ )
+ defect_votes: dict[str, list[bool]] = Field(default_factory=dict)
+ defect_penalty: float = 0.0
+ n_requested: int
+ n_used: int
+ sample_errors: list[str] = Field(default_factory=list)
+ tie_broken: list[str] = Field(default_factory=list,
+ description="defect / acceptance ids an exact vote tie handed to the representative sample")
+ rubric_hash: str = ""
+ judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "
+ "role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")
+ samples: list[dict[str, Any]] = Field(default_factory=list)
+
+
+def _std(vals: list[float]) -> float:
+ return float(statistics.pstdev(vals)) if len(vals) > 1 else 0.0
+
+
+def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeOutput:
+ mean = statistics.fmean(overalls)
+ return min(zip(samples, overalls, strict=True), key=lambda so: abs(so[1] - mean))[0]
+
+
+def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:
+ """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie)."""
+ if not votes:
+ return False, False
+ yes = sum(1 for v in votes if v)
+ if yes * 2 == len(votes):
+ return bool(tie_break), True
+ return yes * 2 > len(votes), False
+
+
+def aggregate_samples(
+ rubric: Rubric,
+ samples: list[JudgeOutput],
+ *,
+ gates: list[GateReport],
+ acceptance_items: list[AcceptanceItem],
+ console_errors: list[str] | None = None,
+ views: list[RenderView] | None = None,
+ usage: Usage | None = None,
+ judge_backend: str = "",
+ n_requested: int | None = None,
+ sample_errors: list[str] | None = None,
+ judge_prompt_hash: str = "",
+) -> Judgment:
+ """Combine ≥ 1 parsed samples into a Judgment (raises ValueError on zero samples).
+
+ ``views`` feeds ``missing_views`` cap rules (e.g. the articulated rubric's
+ required pose sheet).
+ """
+ if not samples:
+ raise ValueError("aggregate_samples needs at least one sample")
+ per_crit: dict[str, list[float]] = {c.id: [] for c in rubric.criteria}
+ overalls: list[float] = []
+ for s in samples:
+ sc = s.scores
+ for cid in per_crit:
+ per_crit[cid].append(float(sc[cid]))
+ overalls.append(rubric.weighted_overall(sc))
+ mean_scores = {cid: round(statistics.fmean(v), 4) for cid, v in per_crit.items()}
+ std_scores = {cid: round(_std(v), 4) for cid, v in per_crit.items()}
+ overall_uncapped = rubric.weighted_overall(mean_scores)
+
+ floors = [
+ {"criterion": cid, "score": round(sc, 4), "floor": fl}
+ for cid, sc, fl in rubric.floors_hit(mean_scores)
+ ]
+ rep = _representative(samples, overalls)
+ tie_broken: list[str] = []
+ votes: dict[str, list[bool]] = {}
+ for s in samples:
+ for aid, ok in s.acceptance_bools.items():
+ votes.setdefault(aid, []).append(bool(ok))
+ for a in acceptance_items:
+ votes.setdefault(a.id, [False])
+ acceptance: dict[str, bool] = {}
+ for aid, v in votes.items():
+ acceptance[aid], tied = _vote(v, bool(rep.acceptance_bools.get(aid, False)))
+ if tied:
+ tie_broken.append(aid)
+ must_missing = missing_must_items(acceptance_items, acceptance)
+
+ d_votes: dict[str, list[bool]] = {d.id: [] for d in rubric.defects}
+ for s in samples:
+ for did, flag in s.defects.items():
+ d_votes.setdefault(did, []).append(bool(flag))
+ defects: dict[str, bool] = {}
+ for did, v in d_votes.items():
+ defects[did], tied = _vote(v, bool(rep.defects.get(did, False)))
+ if tied:
+ tie_broken.append(did)
+ # a checklist defect that a passed gate measured as absent neither penalises nor caps
+ # (judges/caps.measured_absent); it is still named in the verdict tail
+ defects, overridden = veto_measured_defects(rubric, defects, gates)
+ penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)
+ after_defects = max(0.0, overall_uncapped - penalty)
+
+ caps = apply_caps(
+ rubric,
+ after_defects,
+ gates,
+ acceptance,
+ acceptance_items,
+ console_errors=console_errors,
+ views=views,
+ defects_present=defects,
+ )
+ overall = caps.overall
+ passed = overall >= rubric.pass_threshold and not floors and not must_missing
+
+ summary = rep.summary.strip()
+ tail = _verdict_tail(
+ rubric,
+ overall,
+ passed,
+ floors,
+ caps,
+ must_missing,
+ defects=defects,
+ penalty=penalty,
+ overridden=overridden,
+ )
+ if tail:
+ summary = (summary + " " if summary else "") + tail
+
+ breakdown = ScoreBreakdown(
+ per_criterion_mean=mean_scores,
+ per_criterion_std=std_scores,
+ per_sample_overall=[round(o, 4) for o in overalls],
+ overall_uncapped=round(overall_uncapped, 4),
+ overall_after_defects=round(after_defects, 4),
+ overall=round(overall, 4),
+ floors_hit=floors,
+ caps=caps,
+ must_missing=must_missing,
+ acceptance_votes=votes,
+ defects=defects,
+ defect_votes=d_votes,
+ defect_penalty=penalty,
+ n_requested=n_requested or len(samples),
+ n_used=len(samples),
+ sample_errors=sample_errors or [],
+ tie_broken=tie_broken,
+ rubric_hash=rubric.content_hash(),
+ judge_prompt_hash=judge_prompt_hash,
+ samples=[s.model_dump(mode="json") for s in samples],
+ )
+ return Judgment(
+ rubric=rubric.name,
+ judge_backend=judge_backend,
+ scores=mean_scores,
+ overall=round(overall, 4),
+ passed=passed,
+ summary=summary,
+ strengths=list(rep.strengths),
+ issues=list(rep.issues),
+ improvement_plan=sorted(rep.improvement_plan, key=lambda it: it.priority),
+ acceptance_results=acceptance,
+ n_samples=len(samples),
+ score_std=round(_std(overalls), 4),
+ usage=usage or Usage(),
+ raw=json.dumps(breakdown.model_dump(mode="json"), ensure_ascii=False),
+ )
+
+
+def _verdict_tail(
+ rubric: Rubric,
+ overall: float,
+ passed: bool,
+ floors: list[dict[str, Any]],
+ caps: CapResult,
+ must_missing: list[str],
+ *,
+ defects: dict[str, bool] | None = None,
+ penalty: float = 0.0,
+ overridden: list[str] | None = None,
+) -> str:
+ bits = [
+ f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"
+ ]
+ present = [d for d, on in (defects or {}).items() if on]
+ if present:
+ bits.append(f"defects (-{penalty:.2f}): " + ", ".join(present))
+ if overridden:
+ bits.append(
+ "checklist claims contradicted by a passed gate (no penalty, no cap): "
+ + ", ".join(overridden)
+ )
+ if caps.caps_applied:
+ bits.append(
+ "caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied)
+ )
+ if floors:
+ bits.append(
+ "floors: "
+ + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors)
+ )
+ if must_missing:
+ bits.append("must items unverified: " + ", ".join(must_missing))
+ return " | ".join(bits) + "]"
+
+
+def degraded_judgment(
+ rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int, judge_prompt_hash: str = ""
+) -> Judgment:
+ """A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""
+ breakdown = {
+ "status": "degraded",
+ "error": error,
+ "n_requested": n_requested,
+ "n_used": 0,
+ "rubric_hash": rubric.content_hash(),
+ "judge_prompt_hash": judge_prompt_hash,
+ }
+ return Judgment(
+ rubric=rubric.name,
+ judge_backend=judge_backend,
+ scores={c.id: 0.0 for c in rubric.criteria},
+ overall=0.0,
+ passed=False,
+ summary=f"judge_error: {error}",
+ n_samples=0,
+ usage=usage,
+ raw=json.dumps(breakdown, ensure_ascii=False),
+ )
+
+
+def is_degraded(j: Judgment) -> bool:
+ """True when a Judgment is a judge glitch rather than a score."""
+ if j.summary.startswith("judge_error:"):
+ return True
+ try:
+ return json.loads(j.raw).get("status") == "degraded"
+ except (json.JSONDecodeError, AttributeError, TypeError):
+ return False
diff --git a/harness/codeverse/judges/scoring.py b/harness/codeverse/judges/scoring.py
deleted file mode 100644
index 3a8389bd..00000000
--- a/harness/codeverse/judges/scoring.py
+++ /dev/null
@@ -1,297 +0,0 @@
-"""Turn N parsed judge samples into ONE ``Judgment`` — all arithmetic in code.
-
-* per-criterion score = mean over samples; overall = rubric-weighted mean;
- ``score_std`` = std of the per-sample overalls;
-* floors: any criterion mean below its floor → verdict fails;
-* defects: the rubric's binary checklist, majority vote over samples; each
- present item subtracts its ``penalty`` from the overall and its ``cap`` (if
- any) bounds it — arithmetic the VLM never does;
-* caps: deterministic bounds from gate findings / acceptance / missing views
- (``caps.py``) plus the defect caps;
-* acceptance: majority vote over samples;
-* an EXACT tie on a binary item (only possible with an even ``n_samples``) follows
- the *representative* sample (below) instead of a fixed direction. The old rule
- (defect ties → present, acceptance ties → False) made ``n_samples=2`` strictly
- harsher than both n=1 and n=3 — one dissenting sample applied every penalty and
- failed every must item — so scores from the ``economy`` profile (n=2) were not
- comparable with the others. With the representative rule the probability that
- an item is flagged at n=2 equals n=1's, while the continuous scores still average
- over both samples; ids decided this way are listed in ``tie_broken``;
-* narrative (summary / issues / plan / strengths) is taken from the sample whose
- overall is closest to the mean (the *representative* sample) so it is coherent.
-"""
-
-from __future__ import annotations
-
-import json
-import statistics
-from typing import Any
-
-from pydantic import BaseModel, Field
-
-from codeverse.contracts.artifacts import GateReport, RenderView
-from codeverse.contracts.common import Usage
-from codeverse.contracts.judgment import Judgment
-from codeverse.contracts.plan import AcceptanceItem
-from codeverse.judges.caps import CapResult, apply_caps, missing_must_items, veto_measured_defects
-from codeverse.judges.output_schema import JudgeOutput
-from codeverse.judges.rubrics import Rubric
-
-
-class ScoreBreakdown(BaseModel):
- """Everything the pass/fail decision used (serialised into ``Judgment.raw``)."""
-
- status: str = Field(default="ok", description="ok | degraded")
- per_criterion_mean: dict[str, float]
- per_criterion_std: dict[str, float]
- per_sample_overall: list[float]
- overall_uncapped: float = Field(
- description="rubric-weighted mean of the per-criterion means, before defects/caps"
- )
- overall_after_defects: float = Field(
- default=0.0, description="overall_uncapped minus defect penalties"
- )
- overall: float
- floors_hit: list[dict[str, Any]] = Field(default_factory=list)
- caps: CapResult
- must_missing: list[str] = Field(default_factory=list)
- acceptance_votes: dict[str, list[bool]] = Field(default_factory=dict)
- defects: dict[str, bool] = Field(
- default_factory=dict, description="checklist id → present (majority vote)"
- )
- defect_votes: dict[str, list[bool]] = Field(default_factory=dict)
- defect_penalty: float = 0.0
- n_requested: int
- n_used: int
- sample_errors: list[str] = Field(default_factory=list)
- tie_broken: list[str] = Field(default_factory=list,
- description="defect / acceptance ids an exact vote tie handed to the representative sample")
- rubric_hash: str = ""
- judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "
- "role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")
- samples: list[dict[str, Any]] = Field(default_factory=list)
-
-
-def _std(vals: list[float]) -> float:
- return float(statistics.pstdev(vals)) if len(vals) > 1 else 0.0
-
-
-def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeOutput:
- mean = statistics.fmean(overalls)
- return min(zip(samples, overalls, strict=True), key=lambda so: abs(so[1] - mean))[0]
-
-
-def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:
- """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie)."""
- if not votes:
- return False, False
- yes = sum(1 for v in votes if v)
- if yes * 2 == len(votes):
- return bool(tie_break), True
- return yes * 2 > len(votes), False
-
-
-def aggregate_samples(
- rubric: Rubric,
- samples: list[JudgeOutput],
- *,
- gates: list[GateReport],
- acceptance_items: list[AcceptanceItem],
- console_errors: list[str] | None = None,
- views: list[RenderView] | None = None,
- usage: Usage | None = None,
- judge_backend: str = "",
- n_requested: int | None = None,
- sample_errors: list[str] | None = None,
- judge_prompt_hash: str = "",
-) -> Judgment:
- """Combine ≥ 1 parsed samples into a Judgment (raises ValueError on zero samples).
-
- ``views`` feeds ``missing_views`` cap rules (e.g. the articulated rubric's
- required pose sheet).
- """
- if not samples:
- raise ValueError("aggregate_samples needs at least one sample")
- per_crit: dict[str, list[float]] = {c.id: [] for c in rubric.criteria}
- overalls: list[float] = []
- for s in samples:
- sc = s.scores
- for cid in per_crit:
- per_crit[cid].append(float(sc[cid]))
- overalls.append(rubric.weighted_overall(sc))
- mean_scores = {cid: round(statistics.fmean(v), 4) for cid, v in per_crit.items()}
- std_scores = {cid: round(_std(v), 4) for cid, v in per_crit.items()}
- overall_uncapped = rubric.weighted_overall(mean_scores)
-
- floors = [
- {"criterion": cid, "score": round(sc, 4), "floor": fl}
- for cid, sc, fl in rubric.floors_hit(mean_scores)
- ]
- rep = _representative(samples, overalls)
- tie_broken: list[str] = []
- votes: dict[str, list[bool]] = {}
- for s in samples:
- for aid, ok in s.acceptance_bools.items():
- votes.setdefault(aid, []).append(bool(ok))
- for a in acceptance_items:
- votes.setdefault(a.id, [False])
- acceptance: dict[str, bool] = {}
- for aid, v in votes.items():
- acceptance[aid], tied = _vote(v, bool(rep.acceptance_bools.get(aid, False)))
- if tied:
- tie_broken.append(aid)
- must_missing = missing_must_items(acceptance_items, acceptance)
-
- d_votes: dict[str, list[bool]] = {d.id: [] for d in rubric.defects}
- for s in samples:
- for did, flag in s.defects.items():
- d_votes.setdefault(did, []).append(bool(flag))
- defects: dict[str, bool] = {}
- for did, v in d_votes.items():
- defects[did], tied = _vote(v, bool(rep.defects.get(did, False)))
- if tied:
- tie_broken.append(did)
- # a checklist defect that a passed gate measured as absent neither penalises nor caps
- # (judges/caps.measured_absent); it is still named in the verdict tail
- defects, overridden = veto_measured_defects(rubric, defects, gates)
- penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)
- after_defects = max(0.0, overall_uncapped - penalty)
-
- caps = apply_caps(
- rubric,
- after_defects,
- gates,
- acceptance,
- acceptance_items,
- console_errors=console_errors,
- views=views,
- defects_present=defects,
- )
- overall = caps.overall
- passed = overall >= rubric.pass_threshold and not floors and not must_missing
-
- summary = rep.summary.strip()
- tail = _verdict_tail(
- rubric,
- overall,
- passed,
- floors,
- caps,
- must_missing,
- defects=defects,
- penalty=penalty,
- overridden=overridden,
- )
- if tail:
- summary = (summary + " " if summary else "") + tail
-
- breakdown = ScoreBreakdown(
- per_criterion_mean=mean_scores,
- per_criterion_std=std_scores,
- per_sample_overall=[round(o, 4) for o in overalls],
- overall_uncapped=round(overall_uncapped, 4),
- overall_after_defects=round(after_defects, 4),
- overall=round(overall, 4),
- floors_hit=floors,
- caps=caps,
- must_missing=must_missing,
- acceptance_votes=votes,
- defects=defects,
- defect_votes=d_votes,
- defect_penalty=penalty,
- n_requested=n_requested or len(samples),
- n_used=len(samples),
- sample_errors=sample_errors or [],
- tie_broken=tie_broken,
- rubric_hash=rubric.content_hash(),
- judge_prompt_hash=judge_prompt_hash,
- samples=[s.model_dump(mode="json") for s in samples],
- )
- return Judgment(
- rubric=rubric.name,
- judge_backend=judge_backend,
- scores=mean_scores,
- overall=round(overall, 4),
- passed=passed,
- summary=summary,
- strengths=list(rep.strengths),
- issues=list(rep.issues),
- improvement_plan=sorted(rep.improvement_plan, key=lambda it: it.priority),
- acceptance_results=acceptance,
- n_samples=len(samples),
- score_std=round(_std(overalls), 4),
- usage=usage or Usage(),
- raw=json.dumps(breakdown.model_dump(mode="json"), ensure_ascii=False),
- )
-
-
-def _verdict_tail(
- rubric: Rubric,
- overall: float,
- passed: bool,
- floors: list[dict[str, Any]],
- caps: CapResult,
- must_missing: list[str],
- *,
- defects: dict[str, bool] | None = None,
- penalty: float = 0.0,
- overridden: list[str] | None = None,
-) -> str:
- bits = [
- f"[verdict: overall {overall:.2f} vs threshold {rubric.pass_threshold:.2f} → {'PASS' if passed else 'FAIL'}"
- ]
- present = [d for d, on in (defects or {}).items() if on]
- if present:
- bits.append(f"defects (-{penalty:.2f}): " + ", ".join(present))
- if overridden:
- bits.append(
- "checklist claims contradicted by a passed gate (no penalty, no cap): "
- + ", ".join(overridden)
- )
- if caps.caps_applied:
- bits.append(
- "caps: " + "; ".join(f"{c.rule}≤{c.cap:.2f} ({c.evidence})" for c in caps.caps_applied)
- )
- if floors:
- bits.append(
- "floors: "
- + "; ".join(f"{f['criterion']} {f['score']:.2f}<{f['floor']:.2f}" for f in floors)
- )
- if must_missing:
- bits.append("must items unverified: " + ", ".join(must_missing))
- return " | ".join(bits) + "]"
-
-
-def degraded_judgment(
- rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int, judge_prompt_hash: str = ""
-) -> Judgment:
- """A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""
- breakdown = {
- "status": "degraded",
- "error": error,
- "n_requested": n_requested,
- "n_used": 0,
- "rubric_hash": rubric.content_hash(),
- "judge_prompt_hash": judge_prompt_hash,
- }
- return Judgment(
- rubric=rubric.name,
- judge_backend=judge_backend,
- scores={c.id: 0.0 for c in rubric.criteria},
- overall=0.0,
- passed=False,
- summary=f"judge_error: {error}",
- n_samples=0,
- usage=usage,
- raw=json.dumps(breakdown, ensure_ascii=False),
- )
-
-
-def is_degraded(j: Judgment) -> bool:
- """True when a Judgment is a judge glitch rather than a score."""
- if j.summary.startswith("judge_error:"):
- return True
- try:
- return json.loads(j.raw).get("status") == "degraded"
- except (json.JSONDecodeError, AttributeError, TypeError):
- return False
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index dc36e8fc..70bd8e53 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -18,27 +18,32 @@ to treat as a glitch, never as a score.
from __future__ import annotations
import logging
+import os
+import re
import time
+from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
-from typing import Literal
+from typing import Any, Literal
from codeverse.config import get_settings
-from codeverse.contracts.artifacts import RenderSet
+from codeverse.contracts.artifacts import RenderSet, RenderView
from codeverse.contracts.chat import ChatRequest, ChatResponse
from codeverse.contracts.common import Usage
from codeverse.contracts.judgment import Judgment
from codeverse.fanout import fan_out
from codeverse.judges.base import JudgeInput
-from codeverse.judges.output_schema import (
+from codeverse.judges.prompt_builder import build_judge_messages, judge_prompt_hash
+from codeverse.judges.rubrics import (
JudgeOutput,
JudgeParseError,
+ Rubric,
+ aggregate_samples,
+ degraded_judgment,
+ load_rubric,
parse_judge_output,
wire_schema,
)
-from codeverse.judges.prompt_builder import build_judge_messages, judge_prompt_hash
-from codeverse.judges.rubrics import Rubric, load_rubric
-from codeverse.judges.scoring import aggregate_samples, degraded_judgment
from codeverse.models.base import ChatModel, ModelError
log = logging.getLogger(__name__)
@@ -233,3 +238,333 @@ class VlmJudge:
log.warning("judge %s: %s", req.label, last[:300])
req = req.model_copy(update={"temperature": min(1.0, self.temperature + 0.1 * attempt)})
return None, usage, last or "unknown judge failure"
+
+
+# ===================================================================== reference
+# (merged from codeverse/judges/reference.py, 2026-08-28)
+log = logging.getLogger(__name__)
+
+SilhouetteFn = Callable[[str, str], dict[str, Any]]
+
+#: IoU → score mapping: ≤ 0.25 → 0, ≥ 0.85 → 1, linear in between (matches the rubric anchors).
+IOU_LOW, IOU_HIGH = 0.25, 0.85
+
+#: score the measured criterion gets when the measurement cannot be trusted
+#: (unreliable mask, or a reference whose proportions contradict the brief)
+NEUTRAL_SCORE = 0.5
+
+#: views handed to the mismatch pass (front-ish first, then a 3/4 and a side)
+DIFF_VIEW_NAMES: tuple[str, ...] = ("front", "front_right_34", "front_left_34", "right", "left")
+_ENV_DIFF = "CV3D_REFERENCE_DIFF"
+
+
+def iou_to_score(iou: float) -> float:
+ return max(0.0, min(1.0, (float(iou) - IOU_LOW) / (IOU_HIGH - IOU_LOW)))
+
+
+def _default_silhouette_fn() -> SilhouetteFn | None:
+ try:
+ from codeverse.spatial.silhouette import compare_silhouette # type: ignore[attr-defined]
+ except (ImportError, AttributeError):
+ return None
+ return compare_silhouette
+
+
+def _diff_enabled(flag: bool) -> bool:
+ return flag and os.environ.get(_ENV_DIFF, "on").strip().lower() not in ("0", "off", "false", "no")
+
+
+def _conflict_note(info: dict[str, Any]) -> str:
+ try:
+ from codeverse.reference import conflict_note
+
+ return conflict_note(info)
+ except ImportError: # pragma: no cover
+ return ""
+
+
+def _is_synth(note: str) -> bool:
+ return note.startswith("SYNTHESIZED")
+
+
+class ReferenceJudge(VlmJudge):
+ """VlmJudge + reference images + measured silhouette + named mismatches."""
+
+ name = "reference"
+
+ def __init__(
+ self,
+ model_id: str | None = None,
+ n_samples: int = 1,
+ temperature: float = 0.2,
+ *,
+ rubric: str | Rubric = "reference_v1",
+ silhouette_fn: SilhouetteFn | None = None,
+ front_view_names: tuple[str, ...] = ("front", "front_right_34"),
+ best_view: bool = True,
+ diff: bool = True,
+ diff_model: Any | None = None,
+ diff_model_id: str = "",
+ **kwargs: Any,
+ ):
+ super().__init__(rubric, model_id, n_samples, temperature, **kwargs)
+ self.silhouette_fn = silhouette_fn or _default_silhouette_fn()
+ self.front_view_names = front_view_names
+ self.best_view = best_view
+ self.diff = diff
+ self._diff_model = diff_model
+ self.diff_model_id = diff_model_id
+ #: the last diff computed (tests / callers that want the mismatches themselves)
+ self.last_diff: Any | None = None
+
+ # ------------------------------------------------------------------ API
+ def judge(self, inp: JudgeInput, **kwargs: Any) -> Any:
+ """``VlmJudge.judge`` plus the mismatch pass's own spend.
+
+ The diff is a real model call made from :meth:`context`; folding its
+ ``Usage`` into the verdict keeps ``BudgetGuard`` and ``record.total_usage``
+ honest (the ledger already sees it through the metered chat model)."""
+ verdict = super().judge(inp, **kwargs)
+ diff = self.last_diff
+ if diff is not None and getattr(diff, "usage", None) is not None:
+ verdict.usage = verdict.usage + diff.usage
+ return verdict
+
+ # ------------------------------------------------------------------ hook
+ def context(self, inp: JudgeInput) -> JudgeContext:
+ refs = [r for r in inp.spec.references if Path(r.path).is_file()]
+ synth = bool(refs) and all(_is_synth(r.note) for r in refs)
+ images: list[tuple[str, str]] = []
+ for i, r in enumerate(refs[:3], 1):
+ tag = "SYNTHESIZED " if _is_synth(r.note) else ""
+ note = f" — {r.note}" if r.note else ""
+ images.append((f"{tag}REFERENCE {i}/{min(len(refs), 3)} ({r.role}){note}", r.path))
+ blocks: list[str] = []
+ if synth:
+ blocks.append(
+ "NOTE ON THE REFERENCE: it was SYNTHESIZED from the brief by an image model, not photographed. "
+ "It is a shape/part-inventory target only. Where it disagrees with the brief or the stated "
+ "dimensions, the BRIEF is correct and the render must follow the brief.")
+ conflict = self.dimension_conflict(inp, refs)
+ note = _conflict_note(conflict)
+ if note:
+ blocks.append(note)
+ measured_ids = [c.id for c in self.rubric.measured_criteria()]
+ info: dict[str, Any] = {}
+ measured_scores: dict[str, float] = {}
+ if measured_ids:
+ info = self.measure_silhouette(inp)
+ if "iou" not in info:
+ raise ReferenceJudgeError(
+ f"cannot score measured criteria {measured_ids}: {info.get('error', 'no iou')}")
+ score, why = self.silhouette_score(info, conflict)
+ measured_scores = {cid: score for cid in measured_ids}
+ blocks.append(
+ f"MEASURED SILHOUETTE (harness): {info['render']} render vs reference {info['reference']}: "
+ f"IoU {info['iou']:.3f} → silhouette_match score {score:.2f}{why}."
+ + (f" extra: {info['extra']}" if info.get("extra") else ""))
+ elif refs:
+ info = self.measure_silhouette(inp)
+ diff = self.reference_diff(inp, refs, info, synthesized=synth) if refs else None
+ self.last_diff = diff
+ if diff is not None and diff.as_text():
+ blocks.append(diff.as_text())
+ return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images)
+
+ # ------------------------------------------------------------------ proportion guard
+ def dimension_conflict(self, inp: JudgeInput, refs: list[Any]) -> dict[str, Any]:
+ """Does the target reference's own outline contradict the brief's dimensions?
+
+ When it does, no numeric proportion signal derived from that picture may be
+ used against the object — the brief wins (see
+ :mod:`codeverse.reference.proportions`).
+ """
+ targets = [r for r in refs if r.role == "target"] or refs
+ if not targets:
+ return {"conflict": False}
+ try:
+ from codeverse.reference import dimension_conflict
+
+ return dimension_conflict(inp.spec, targets[0].path)
+ except Exception as e: # noqa: BLE001 — advisory
+ log.warning("dimension-conflict check failed: %s", e)
+ return {"conflict": False, "error": str(e)}
+
+ def silhouette_score(self, info: dict[str, Any], conflict: dict[str, Any]) -> tuple[float, str]:
+ """IoU → measured score, NEUTRAL when the reference contradicts the brief or the
+ mask is unreliable. Returns ``(score, ' (why)')``."""
+ if conflict.get("conflict"):
+ return NEUTRAL_SCORE, " (NEUTRAL: the reference's proportions contradict the brief's dimensions)"
+ if info.get("reliable") is False:
+ return NEUTRAL_SCORE, " (NEUTRAL: the background mask was unreliable)"
+ return iou_to_score(info["iou"]), ""
+
+ # ------------------------------------------------------------------ mismatch pass
+ def reference_diff(self, inp: JudgeInput, refs: list[Any], info: dict[str, Any], *, synthesized: bool) -> Any | None:
+ """One extra vision call naming concrete mismatches. ``None`` when disabled;
+ never raises (a failed diff simply contributes no text)."""
+ if not _diff_enabled(self.diff):
+ return None
+ try:
+ from codeverse.reference import compare
+ except ImportError: # pragma: no cover - the package is part of the wheel
+ return None
+ targets = [r.path for r in refs if r.role == "target"] or [r.path for r in refs]
+ renders = self.diff_views(list(inp.renders.views))
+ measured = dict(info) if "iou" in info else None
+ if measured is not None:
+ measured.setdefault("view", info.get("render", ""))
+ try:
+ return compare(inp.spec, targets, renders, model=self._pick_diff_model(),
+ part_names=_plan_part_names(inp), measured=measured, synthesized=synthesized)
+ except Exception as e: # noqa: BLE001 — an advisory pass must never fail a verdict
+ log.warning("reference diff pass failed: %s", e)
+ return None
+
+ def _pick_diff_model(self) -> Any:
+ """``diff_model`` > ``diff_model_id`` (a cheaper perception model) > the judge's own model."""
+ if self._diff_model is not None:
+ return self._diff_model
+ if self.diff_model_id:
+ from codeverse.models import get_chat_model
+
+ self._diff_model = get_chat_model(self.diff_model_id)
+ return self._diff_model
+ return self.model
+
+ def diff_views(self, views: list[RenderView]) -> list[str]:
+ """Up to 3 shaded views for the diff, front-ish first."""
+ by_name = {v.name: v for v in views}
+ picked = [by_name[n].path for n in DIFF_VIEW_NAMES if n in by_name]
+ for v in views:
+ if len(picked) >= 3:
+ break
+ if v.path not in picked:
+ picked.append(v.path)
+ return picked[:3]
+
+ # ------------------------------------------------------------------ silhouette
+ def pick_front_view(self, views: list[RenderView]) -> RenderView | None:
+ for name in self.front_view_names:
+ for v in views:
+ if v.name == name:
+ return v
+ return views[0] if views else None
+
+ def measure_silhouette(self, inp: JudgeInput) -> dict[str, Any]:
+ """{iou, render, reference, extra} or {error}.
+
+ With ``best_view=True`` (default) the IoU is taken from the render view that
+ best matches the reference's camera instead of a hardcoded ``front``: a
+ reference photograph has ONE camera, and comparing a three-quarter product
+ shot to a straight-on elevation measures camera agreement, not shape. The
+ winning view name travels in ``render`` so the number stays auditable.
+ """
+ targets = [r for r in inp.spec.references if r.role == "target" and Path(r.path).is_file()] or [
+ r for r in inp.spec.references if Path(r.path).is_file()]
+ if not targets:
+ return {"error": "spec has no readable reference images"}
+ views = list(inp.renders.views)
+ if not views:
+ return {"error": "render set has no views"}
+ if self.silhouette_fn is None:
+ return {"error": "compare_silhouette unavailable (codeverse.spatial.silhouette not importable)"}
+ res, name = self._measure(views, targets[0].path)
+ if not isinstance(res, dict) or "iou" not in res:
+ return {"error": f"compare_silhouette returned no iou: {res!r}"}
+ extra = {k: v for k, v in res.items() if k not in ("iou", "per_view") and isinstance(v, (int, float, str, bool))}
+ out = {"iou": float(res["iou"]), "render": name, "reference": Path(targets[0].path).name,
+ "reliable": bool(res.get("reliable", True)),
+ "aspect_ratio_err": res.get("aspect_ratio_err"), "extra": extra}
+ if res.get("per_view"):
+ out["per_view"] = res["per_view"]
+ return out
+
+ def _measure(self, views: list[RenderView], reference: str) -> tuple[dict[str, Any], str]:
+ """(result, view name). Best-matching view when enabled and available."""
+ if self.best_view and self.silhouette_fn is _default_silhouette_fn():
+ try:
+ from codeverse.spatial.silhouette import best_view_match
+ except ImportError: # pragma: no cover
+ best_view_match = None # type: ignore[assignment]
+ if best_view_match is not None:
+ res = best_view_match(views, reference)
+ if "iou" in res:
+ return res, str(res.get("view", ""))
+ view = self.pick_front_view(views)
+ if view is None: # pragma: no cover - guarded by the caller
+ return {}, ""
+ return self.silhouette_fn(view.path, reference), view.name # type: ignore[misc]
+
+
+def _plan_part_names(inp: JudgeInput) -> list[str]:
+ """Part names from the plan digest the judge already receives (best effort).
+
+ ``judges.replay_input.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;
+ a multi-line digest lists one ``- Name · role · …`` per line. Both are handled, and
+ an unrecognised digest simply yields no names (the diff prompt then says so).
+ """
+ text = getattr(inp, "plan_summary", "") or ""
+ names: list[str] = []
+ for chunk in re.split(r"(?i)\bparts\s*:", text)[1:]:
+ for raw in chunk.split("\n")[0].split(","):
+ names.append(raw.split("\u00d7")[0].split("(")[0].strip().rstrip("."))
+ for line in text.splitlines():
+ line = line.strip()
+ if not line.startswith(("-", "*", "\u00b7")):
+ continue
+ head = line.lstrip("-*\u00b7 ").split("\u00b7")[0].split("(")[0].split(":")[0].strip()
+ names.append(head)
+ seen: list[str] = []
+ for n in names:
+ if n and " " not in n and 2 < len(n) <= 40 and n[:1].isupper() and n not in seen:
+ seen.append(n)
+ return seen[:40]
+
+
+
+LIKENESS_NOTE = (
+ "REAL-WORLD REFERENCE PHOTOS are attached (labelled REAL-WORLD REFERENCE n/N). They are not a "
+ "composition to copy; they are what the thing the brief names ACTUALLY looks like. Score "
+ "brief fidelity, colour/light and richness against them: dominant colour and where the "
+ "secondary colours sit, how the structure folds / layers / thins out, where the brightness "
+ "concentrates and how much of the frame stays dark, what the real texture is at fine scale. A "
+ "frame that ticks every noun in the brief but would never be mistaken for the photographed "
+ "thing (a row of evenly spaced bars standing in for curtains, a flat band standing in for a "
+ "gradient of light, cartoon-saturated colour where the photo is subtle) is a 0.4-0.5 on brief "
+ "fidelity, not a 1.0. Foreground, framing and landscape in the photos are incidental unless "
+ "the brief asks for them."
+)
+
+
+class LikenessJudge(VlmJudge):
+ """``VlmJudge`` + the reference photos of the REAL thing — for tracks with no silhouette.
+
+ ``ReferenceJudge`` is built for objects: it measures a front-view silhouette IoU against
+ the target photo and runs a part-inventory mismatch pass. Neither means anything for a
+ fragment shader or a scene, where a reference photo answers a different question — does
+ this LOOK like the thing? Measured 2026-08-26 (teaser aurora, three versions, flash and
+ codex): every version scored 0.78-0.94 with an empty issues list while none resembled an
+ aurora (a comb of straight teal bars; pink cotton-wool lobes), because the brief is a
+ checklist of nouns and the rubric had nothing to say about likeness. This judge attaches
+ up to three photos beside the frames with :data:`LIKENESS_NOTE` and leaves the rubric alone.
+ """
+
+ name = "likeness"
+
+ def __init__(self, model_id: str | None = None, n_samples: int = 1, temperature: float = 0.2, *,
+ rubric: str | Rubric = "shader_v2", max_refs: int = 3, **kwargs: Any):
+ super().__init__(rubric, model_id, n_samples, temperature, **kwargs)
+ self.max_refs = max(1, int(max_refs))
+
+ def context(self, inp: JudgeInput) -> JudgeContext:
+ if self.rubric.measured_criteria():
+ raise ValueError(f"rubric {self.rubric.name} has measured criteria; LikenessJudge measures nothing")
+ refs = [r for r in inp.spec.references if Path(r.path).is_file()][: self.max_refs]
+ images = [(f"REAL-WORLD REFERENCE {i}/{len(refs)}" + (f" — {r.note}" if r.note else ""), r.path)
+ for i, r in enumerate(refs, 1)]
+ return JudgeContext(extra_text=LIKENESS_NOTE if refs else "", extra_images=images)
+
+
+class ReferenceJudgeError(RuntimeError):
+ """The reference judge cannot compute its measured criteria."""
diff --git a/harness/codeverse/reference.py b/harness/codeverse/reference.py
index 07feec45..30bfa47a 100644
--- a/harness/codeverse/reference.py
+++ b/harness/codeverse/reference.py
@@ -5,7 +5,7 @@ synthesizes a neutral studio product shot of the brief with the image model,
**validates** it (:mod:`.gate`), and hands it to the rest of the harness as the
fidelity anchor: the planner sees it, the generator's ``compare_reference`` tool
sees it, the silhouette gate measures against it and the judge scores against it
-(:class:`codeverse.judges.reference.ReferenceJudge`).
+(:class:`codeverse.judges.vlm_judge.ReferenceJudge`).
A synthesized reference is never ground truth and is marked as such everywhere
(:mod:`.attach`): the user's own ``--image`` wins, the brief's dimensions win,
diff --git a/harness/codeverse/spatial/silhouette.py b/harness/codeverse/spatial/silhouette.py
index 5401c979..1ad9dc95 100644
--- a/harness/codeverse/spatial/silhouette.py
+++ b/harness/codeverse/spatial/silhouette.py
@@ -161,7 +161,7 @@ def compare_silhouette(render_png: Path | str, reference_png: Path | str, *, dif
reliable, diff_png_path?}``. ``reliable`` is False when either mask is empty, covers
more than 95 % of the pixels, or has a bounding box spanning the whole frame — all
three mean the background estimate failed and the IoU below is arithmetic on noise.
- The caller must honour it: :class:`~codeverse.judges.reference.ReferenceJudge` scores
+ The caller must honour it: :class:`~codeverse.judges.vlm_judge.ReferenceJudge` scores
``silhouette_match`` NEUTRAL rather than low, because "we could not measure this" is
not the same as "it does not match".
"""
diff --git a/harness/codeverse/texturing/gate.py b/harness/codeverse/texturing/gate.py
index e178cf56..73af695a 100644
--- a/harness/codeverse/texturing/gate.py
+++ b/harness/codeverse/texturing/gate.py
@@ -91,7 +91,7 @@ def _plan_summary(plan: StaticPlan | None) -> str:
def _degraded(j: Judgment) -> bool:
- from codeverse.judges.scoring import is_degraded
+ from codeverse.judges.rubrics import is_degraded
return is_degraded(j)
diff --git a/harness/codeverse/tracks/common.py b/harness/codeverse/tracks/common.py
index aff03cbb..57ab927d 100644
--- a/harness/codeverse/tracks/common.py
+++ b/harness/codeverse/tracks/common.py
@@ -61,11 +61,11 @@ class Services:
def reference_judge(self, model_id: str, n_samples: int = 1, rubric: str = "reference_v1") -> Any:
"""Image-conditioned judge (spec has reference images): renders + references + silhouette IoU."""
- return _import("codeverse.judges.reference", "ReferenceJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
+ return _import("codeverse.judges.vlm_judge", "ReferenceJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
def likeness_judge(self, model_id: str, n_samples: int = 1, rubric: str = "shader_v2") -> Any:
"""Reference photos beside the frames, no silhouette (graphics / scene with images)."""
- return _import("codeverse.judges.reference", "LikenessJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
+ return _import("codeverse.judges.vlm_judge", "LikenessJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
def pairwise(self, model_id: str) -> Any:
"""Position-swapped A/B judge: ``.compare(spec, renders_a, renders_b, rubric=...)``."""
diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.py
index dff22b45..84d4b242 100644
--- a/harness/codeverse/tracks/steps.py
+++ b/harness/codeverse/tracks/steps.py
@@ -470,7 +470,7 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildRes
def _is_degraded(judgment: Judgment) -> bool:
try:
- from codeverse.judges.scoring import is_degraded
+ from codeverse.judges.rubrics import is_degraded
except ImportError: # pragma: no cover — judges package always ships with tracks
return False
return is_degraded(judgment)
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index d045f7cb..8ec4fc88 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -141,11 +141,11 @@ codeverse/
billing.py (SUBSCRIPTION_BACKENDS/bills_usd — which backends take real dollars,
so max_usd guards money and not list price; docs/COST.md §25)
guard.py routing.py reconstruct.py (old runs) audit.py report.py
- judges/ base.py (Judge protocol), rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,
- prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.py
- (compare_many), reference.py, calibration.py, metrics.py, replay_input.py
- (plan_digest / resolve_paths / judged_subset — the pure round-replay pieces
- `3dcv judge` and calibration share)
+ judges/ base.py (Judge protocol + the pure round-replay pieces `3dcv judge` and
+ calibration share), rubrics.py + rubrics/*.yaml (defect checklists, the wire
+ schema, caps and scoring), prompt_builder.py (image prep, montages, the
+ judge messages), vlm_judge.py (+ the reference/likeness judges),
+ pairwise.py (compare_many), calibration.py
reference/ reference GROUNDING — give the pipeline a picture of what it is building:
one module: synthesis, THE plausibility gate that makes a synthesized
reference safe to use, Spec attachment + honesty guards, proportions
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 8fa925dc..beb7a089 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -282,19 +282,19 @@ VlmJudge(rubric="static_object_v1", model_id=None (settings.default_judge = gemi
VlmJudge.judge(inp, *, geometry_views: RenderSet | None = None) -> Judgment # also reads inp.geometry_views
from codeverse.judges.base import JudgeInput # (spec, renders, measurement=None, gates=[], acceptance=[], plan_summary="",
# round_index=0, previous=None, extra_context="", geometry_views=None)
-from codeverse.judges.montage import plan_montages, render_montage, Montage # ≤3 2×2 montages (shaded/geometry/poses) + ≤2 detail
+from codeverse.judges.prompt_builder import plan_montages, render_montage, Montage # ≤3 2×2 montages (shaded/geometry/poses) + ≤2 detail
# crops @≤1024px replace sheet+9 views; clay/normals views (RenderView.mode) auto-route to the GEOMETRY montage
-from codeverse.judges.scoring import is_degraded, aggregate_samples # ScoreBreakdown adds defects, defect_votes (majority,
+from codeverse.judges.rubrics import is_degraded, aggregate_samples # ScoreBreakdown adds defects, defect_votes (majority,
# ties→representative sample, D36), tie_broken, defect_penalty, overall_after_defects,
# judge_prompt_hash (D37); overall = caps(weighted_mean − Σpenalty)
-from codeverse.judges.caps import apply_caps # (rubric, overall, gates, acceptance_results, acceptance_items=None, *,
+from codeverse.judges.rubrics import apply_caps # (rubric, overall, gates, acceptance_results, acceptance_items=None, *,
# console_errors=None, views=None, defects_present=None) -> CapResult;
# cap rules add when="missing_views" and ledger lines "defect:<id>"
from codeverse.judges.pairwise import PairwiseJudge # .compare(spec, renders_a, renders_b, *, rubric=…) -> PairwiseResult
PairwiseJudge.compare_many(spec, candidates: list[RenderSet], *, rubric) -> RankingResult{order, points, confidence, pairs, usage, .best}
from codeverse.judges.calibration import calibrate, CalibrationTable # (run_dirs, *, model_id, n_samples=3, out_dir, geometry_mode,
# rounds, …) -> rows + pearson/spearman(errors vs score), mean_std, cost; CLI: python -m codeverse.judges.calibration RUN… --n 3
-from codeverse.judges.reference import ReferenceJudge # image-conditioned specs
+from codeverse.judges.vlm_judge import ReferenceJudge # image-conditioned specs
from codeverse.judges.metrics import judge_agreement, plateau, best_index
```
Rubrics: `static_object_v1` (0.72), `articulated_v1` (requires pose views via
diff --git a/harness/tests/compare_bench/test_fixed_eval_language.py b/harness/tests/compare_bench/test_fixed_eval_language.py
index aee4ee8c..ac9cd08f 100644
--- a/harness/tests/compare_bench/test_fixed_eval_language.py
+++ b/harness/tests/compare_bench/test_fixed_eval_language.py
@@ -67,7 +67,7 @@ def test_graphics_judge_is_the_track_rubric_with_the_photos():
from bench._fixed_eval import FixedEvaluator
from codeverse.contracts.common import Track
from codeverse.contracts.spec import ReferenceImage, Spec
- from codeverse.judges.reference import LikenessJudge
+ from codeverse.judges.vlm_judge import LikenessJudge
ev = FixedEvaluator("gemini:gemini-3.1-pro-preview", n_samples=2)
with_photos = Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="x", references=[ReferenceImage(path="/p.png")])
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 82706d09..edc488f8 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -175,7 +175,7 @@ def test_judge_rejudges_with_in_run_inputs(runs_dir: Path, monkeypatch):
def test_judge_uses_reference_judge_for_measured_rubrics(runs_dir: Path, monkeypatch):
- import codeverse.judges.reference as ref
+ import codeverse.judges.vlm_judge as ref
class _FakeRef(_FakeVlm):
pass
diff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.py
index ac3f5964..06827bde 100644
--- a/harness/tests/judges/test_calibration.py
+++ b/harness/tests/judges/test_calibration.py
@@ -5,8 +5,8 @@ import pytest
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
from codeverse.contracts.run import RoundRecord
+from codeverse.judges.base import plan_digest
from codeverse.judges.calibration import calibrate, load_run_cases, pearson, spearman
-from codeverse.judges.replay_input import plan_digest
from codeverse.judges.rubrics import load_rubric
from tests.judges.conftest import (
ACCEPTANCE,
diff --git a/harness/tests/judges/test_caps.py b/harness/tests/judges/test_caps.py
index 92d697a3..769a9a03 100644
--- a/harness/tests/judges/test_caps.py
+++ b/harness/tests/judges/test_caps.py
@@ -1,7 +1,6 @@
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
from codeverse.contracts.plan import AcceptanceItem
-from codeverse.judges.caps import apply_caps
-from codeverse.judges.rubrics import load_rubric
+from codeverse.judges.rubrics import apply_caps, load_rubric
R = load_rubric("static_object_v1")
@@ -73,8 +72,7 @@ def test_a_floating_claim_the_connectivity_gate_measured_absent_neither_penalise
'all 9 parts connected, gap <= 2 mm'; the judge marked floating_part present on a dark
seam and defect:floating_part capped the run at 0.6 (uncapped 0.72) against 0.96 for a
sibling the eye cannot tell apart. Gates decide geometry (law 3)."""
- from codeverse.judges.caps import measured_absent, veto_measured_defects
- from codeverse.judges.rubrics import load_rubric
+ from codeverse.judges.rubrics import load_rubric, measured_absent, veto_measured_defects
rubric = load_rubric("static_object_v1")
passed = [_connectivity(True, ("info", "all 9 parts are connected (9 contacts, gap <= 2 mm)"),
@@ -86,8 +84,7 @@ def test_a_floating_claim_the_connectivity_gate_measured_absent_neither_penalise
def test_a_real_floating_part_still_caps():
- from codeverse.judges.caps import measured_absent
- from codeverse.judges.rubrics import load_rubric
+ from codeverse.judges.rubrics import load_rubric, measured_absent
rubric = load_rubric("static_object_v1")
failed = [_connectivity(False, ("error", "part 'Seat' is floating 12 mm above 'Leg'"))]
@@ -100,8 +97,7 @@ def test_scene_placement_gate_measures_the_floating_asset_claim():
"""scene_v1 (2026-08-26): the placement gate and the checklist defect share an id, so a passed
gate vetoes the VLM's floating/sunken claim and an ERROR there caps the round at 0.7."""
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
- from codeverse.judges.caps import apply_caps, measured_absent
- from codeverse.judges.rubrics import load_rubric
+ from codeverse.judges.rubrics import apply_caps, load_rubric, measured_absent
rubric = load_rubric("scene_v1")
passed = [GateReport(gate="scene_placement", passed=True, findings=[GateFinding(
diff --git a/harness/tests/judges/test_defects.py b/harness/tests/judges/test_defects.py
index a1e13ca1..15b13e31 100644
--- a/harness/tests/judges/test_defects.py
+++ b/harness/tests/judges/test_defects.py
@@ -5,10 +5,16 @@ import json
import pytest
from codeverse.contracts.artifacts import RenderSet, RenderView
-from codeverse.judges.caps import apply_caps
-from codeverse.judges.output_schema import JudgeParseError, parse_judge_output, wire_schema
-from codeverse.judges.rubrics import RubricError, load_rubric, rubric_from_dict
-from codeverse.judges.scoring import aggregate_samples
+from codeverse.judges.rubrics import (
+ JudgeParseError,
+ RubricError,
+ aggregate_samples,
+ apply_caps,
+ load_rubric,
+ parse_judge_output,
+ rubric_from_dict,
+ wire_schema,
+)
from codeverse.judges.vlm_judge import VlmJudge
from tests.judges.conftest import FakeChatModel, good_reply
diff --git a/harness/tests/judges/test_likeness.py b/harness/tests/judges/test_likeness.py
index 7017bce5..a26a8202 100644
--- a/harness/tests/judges/test_likeness.py
+++ b/harness/tests/judges/test_likeness.py
@@ -4,8 +4,8 @@ from __future__ import annotations
from codeverse.contracts.common import Language, Track
from codeverse.contracts.spec import ReferenceImage
-from codeverse.judges.reference import LIKENESS_NOTE, LikenessJudge
from codeverse.judges.rubrics import load_rubric
+from codeverse.judges.vlm_judge import LIKENESS_NOTE, LikenessJudge
from tests.judges.conftest import FakeChatModel, draw_chair, good_reply, image_parts, make_spec
diff --git a/harness/tests/judges/test_live.py b/harness/tests/judges/test_live.py
index 7263ce82..266cf942 100644
--- a/harness/tests/judges/test_live.py
+++ b/harness/tests/judges/test_live.py
@@ -13,7 +13,7 @@ from codeverse.config import get_settings
from codeverse.contracts.artifacts import RenderSet
from codeverse.contracts.plan import AcceptanceItem
from codeverse.judges.base import JudgeInput
-from codeverse.judges.scoring import is_degraded
+from codeverse.judges.rubrics import is_degraded
from codeverse.judges.vlm_judge import VlmJudge
from tests.judges.conftest import make_measurement, make_renders, make_spec
@@ -130,7 +130,7 @@ def test_live_compare_many_ranks_stool_over_crude(tmp_path):
def test_live_reference_judge(tmp_path):
from codeverse.contracts.spec import ReferenceImage
- from codeverse.judges.reference import ReferenceJudge
+ from codeverse.judges.vlm_judge import ReferenceJudge
renders, meas, source = _renders_for_test(tmp_path / "stool")
if "stool" not in source:
diff --git a/harness/tests/judges/test_montage.py b/harness/tests/judges/test_montage.py
index 9f594a83..84998bae 100644
--- a/harness/tests/judges/test_montage.py
+++ b/harness/tests/judges/test_montage.py
@@ -4,8 +4,8 @@ import pytest
from PIL import Image
from codeverse.contracts.artifacts import RenderSet, RenderView
-from codeverse.judges.images import JudgeImageError
-from codeverse.judges.montage import (
+from codeverse.judges.prompt_builder import (
+ JudgeImageError,
describe_montages,
montage_label,
montage_strip,
diff --git a/harness/tests/judges/test_output_schema.py b/harness/tests/judges/test_output_schema.py
index cc331f61..0e437bdf 100644
--- a/harness/tests/judges/test_output_schema.py
+++ b/harness/tests/judges/test_output_schema.py
@@ -2,12 +2,12 @@ import json
import pytest
-from codeverse.judges.output_schema import (
+from codeverse.judges.rubrics import (
JudgeParseError,
+ load_rubric,
parse_judge_output,
wire_schema,
)
-from codeverse.judges.rubrics import load_rubric
from tests.judges.conftest import good_reply
R = load_rubric("static_object_v1")
diff --git a/harness/tests/judges/test_prompt_builder.py b/harness/tests/judges/test_prompt_builder.py
index fe4a5387..f6b40ab9 100644
--- a/harness/tests/judges/test_prompt_builder.py
+++ b/harness/tests/judges/test_prompt_builder.py
@@ -5,12 +5,13 @@ from PIL import Image
from codeverse.contracts.artifacts import GateFinding, GateReport, RenderSet, RenderView, Severity
from codeverse.contracts.chat import ImagePart, TextPart
from codeverse.contracts.judgment import ImprovementItem, Judgment
-from codeverse.judges.images import prepare_image, view_az_el
from codeverse.judges.prompt_builder import (
TEXT_BUDGET_CHARS,
build_judge_messages,
build_system_prompt,
montage_image_parts,
+ prepare_image,
+ view_az_el,
)
from codeverse.judges.rubrics import load_rubric
from tests.judges.conftest import draw_chair, make_renders
diff --git a/harness/tests/judges/test_reference.py b/harness/tests/judges/test_reference.py
index e971a9db..9b3792a7 100644
--- a/harness/tests/judges/test_reference.py
+++ b/harness/tests/judges/test_reference.py
@@ -1,8 +1,8 @@
import pytest
from codeverse.contracts.spec import ReferenceImage
-from codeverse.judges.reference import ReferenceJudge, ReferenceJudgeError, iou_to_score
from codeverse.judges.rubrics import load_rubric
+from codeverse.judges.vlm_judge import ReferenceJudge, ReferenceJudgeError, iou_to_score
from tests.judges.conftest import FakeChatModel, draw_chair, good_reply, image_parts
REF = load_rubric("reference_v1")
diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.py
index b1e7a677..420dae8e 100644
--- a/harness/tests/judges/test_vlm_judge.py
+++ b/harness/tests/judges/test_vlm_judge.py
@@ -3,8 +3,7 @@ import json
import pytest
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
-from codeverse.judges.rubrics import load_rubric
-from codeverse.judges.scoring import is_degraded
+from codeverse.judges.rubrics import is_degraded, load_rubric
from codeverse.judges.vlm_judge import VlmJudge
from codeverse.models.base import ModelError
from tests.judges.conftest import FakeChatModel, good_reply, image_parts
@@ -151,7 +150,7 @@ def test_base_judge_rejects_measured_rubric(judge_input, cache_dir):
def test_missing_render_is_loud(judge_input, cache_dir):
- from codeverse.judges.images import JudgeImageError
+ from codeverse.judges.prompt_builder import JudgeImageError
judge_input.renders.views[0].path = "/nonexistent/x.png"
with pytest.raises(JudgeImageError):
_judge(FakeChatModel([good_reply(R, IDS)]), cache_dir=cache_dir).judge(judge_input)
@@ -205,8 +204,8 @@ def test_judge_prompt_hash_is_recorded_in_every_verdict(judge_input, cache_dir):
def test_judge_prompt_hash_tracks_the_protocol_not_the_run(monkeypatch):
from codeverse.judges import prompt_builder as pb
- from codeverse.judges.output_schema import wire_schema
from codeverse.judges.prompt_builder import judge_prompt_hash
+ from codeverse.judges.rubrics import wire_schema
base = judge_prompt_hash(R)
assert judge_prompt_hash(R) == base # deterministic
diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
index 53b90539..3bbcb169 100644
--- a/harness/tests/orchestrator_tracks/test_refactor_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
@@ -123,7 +123,7 @@ def test_scene_judge_views_prefers_stamped_flags():
def test_cli_judge_reconstructs_the_judged_subset(tmp_path):
- from codeverse.judges.replay_input import judged_subset, resolve_paths
+ from codeverse.judges.base import judged_subset, resolve_paths
rs = _rs([True, False, None])
sub = judged_subset(rs)
diff --git a/harness/tests/reference/test_judge_diff.py b/harness/tests/reference/test_judge_diff.py
index 4c7e130b..7b3f5454 100644
--- a/harness/tests/reference/test_judge_diff.py
+++ b/harness/tests/reference/test_judge_diff.py
@@ -7,8 +7,8 @@ import pytest
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.spec import ReferenceImage
from codeverse.judges.base import JudgeInput
-from codeverse.judges.reference import ReferenceJudge
from codeverse.judges.rubrics import load_rubric
+from codeverse.judges.vlm_judge import ReferenceJudge
from codeverse.reference import SYNTH_NOTE
from tests.judges.conftest import (
FakeChatModel,
@@ -101,7 +101,7 @@ def test_reference_that_contradicts_the_brief_scores_neutral(tmp_path, cache_dir
"""A picture whose proportions disagree with the stated dimensions must not drag
the score down: the brief wins and the measured criterion goes neutral."""
from codeverse.contracts.spec import Constraints
- from codeverse.judges.reference import NEUTRAL_SCORE
+ from codeverse.judges.vlm_judge import NEUTRAL_SCORE
png = draw_chair(tmp_path / "wide.png", legs=4) # ~2:1 wide silhouette
spec = make_spec(references=[ReferenceImage(path=str(png), role="target", note=f"{SYNTH_NOTE} [front, x]")],
@@ -118,7 +118,7 @@ def test_reference_that_contradicts_the_brief_scores_neutral(tmp_path, cache_dir
def test_part_names_are_read_from_both_plan_digest_shapes():
- from codeverse.judges.reference import _plan_part_names
+ from codeverse.judges.vlm_judge import _plan_part_names
class One:
plan_summary = ("Chair: a chair. Overall 0.5x0.5x0.9 m. "
diff --git a/harness/tests/scene_gates/test_frame_metrics.py b/harness/tests/scene_gates/test_frame_metrics.py
index cd8e2459..dd748c79 100644
--- a/harness/tests/scene_gates/test_frame_metrics.py
+++ b/harness/tests/scene_gates/test_frame_metrics.py
@@ -6,8 +6,7 @@ import json
from pathlib import Path
from codeverse.contracts.artifacts import RenderSet, RenderView, Severity
-from codeverse.judges.caps import apply_caps
-from codeverse.judges.rubrics import load_rubric
+from codeverse.judges.rubrics import apply_caps, load_rubric
from codeverse.spatial.frame_metrics import (
FRAME_GATE,
frame_findings,
diff --git a/harness/tests/scene_gates/test_placement.py b/harness/tests/scene_gates/test_placement.py
index cb73650b..967f74d9 100644
--- a/harness/tests/scene_gates/test_placement.py
+++ b/harness/tests/scene_gates/test_placement.py
@@ -12,8 +12,7 @@ from types import SimpleNamespace
from codeverse.contracts.artifacts import BuildResult, Severity
from codeverse.contracts.common import Language
from codeverse.contracts.plan import BBox, CameraPlan, ScenePlan, ZonePlan
-from codeverse.judges.caps import apply_caps
-from codeverse.judges.rubrics import load_rubric
+from codeverse.judges.rubrics import apply_caps, load_rubric
from codeverse.orchestrator import build_refine_instructions
from codeverse.spatial.scene_placement import (
GATE,