Skip to content

Official Codebase of 3DCodeVerse

Python

judges: one cache key + tmp name, the scene camera rule the renders obey, scene-ranked pairwise, env_flag for the diff switch

gaoypengd888f0e89b02
diff --git a/harness/codeverse3d/judges/pairwise.py b/harness/codeverse3d/judges/pairwise.py
index 58f73530..4b6cadff 100644
--- a/harness/codeverse3d/judges/pairwise.py
+++ b/harness/codeverse3d/judges/pairwise.py
@@ -22,8 +22,9 @@ from pydantic import BaseModel, Field
from codeverse3d.config import get_settings
from codeverse3d.contracts.artifacts import RenderSet
from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ImagePart, TextPart
-from codeverse3d.contracts.common import Usage
+from codeverse3d.contracts.common import Track, Usage
from codeverse3d.contracts.spec import Spec
+from codeverse3d.judges.base import judged_subset
from codeverse3d.judges.prompt_builder import (
MONTAGE_TILE_PX,
brief_section,
@@ -167,7 +168,7 @@ class PairwiseJudge:
)
parts: list[TextPart | ImagePart] = [TextPart(text=text)]
for tag, rs in (("A", first), ("B", second)):
- for lbl, ip in self._side_images(tag, rs):
+ for lbl, ip in self._side_images(tag, rs, scene=spec.track is Track.SCENE):
parts.append(TextPart(text=lbl))
parts.append(ip)
parts.append(TextPart(text="Compare A and B and return the JSON object."))
@@ -177,9 +178,11 @@ class PairwiseJudge:
max_wait_s=900.0, label=label,
)
- def _side_images(self, tag: str, rs: RenderSet) -> list[tuple[str, ImagePart]]:
- """One 2×2 montage of the candidate's most informative views (plus its pose sheet / geometry montage if any)."""
- montages = plan_montages(rs, scene=False, max_montages=2, detail_crops=0)
+ def _side_images(self, tag: str, rs: RenderSet, *, scene: bool) -> list[tuple[str, ImagePart]]:
+ """One 2×2 montage of the candidate's most informative views (plus its pose sheet / geometry
+ montage if any), from the views the verdict judge saw, ranked for the track: a scene's
+ own cameras first, as the verdict judge ranks them."""
+ montages = plan_montages(judged_subset(rs), scene=scene, max_montages=2, detail_crops=0)
out: list[tuple[str, ImagePart]] = []
for i, m in enumerate(montages, 1):
if len(m.tiles) > self.views_per_side:
diff --git a/harness/codeverse3d/judges/prompt_builder.py b/harness/codeverse3d/judges/prompt_builder.py
index 10bacac7..c882cc57 100644
--- a/harness/codeverse3d/judges/prompt_builder.py
+++ b/harness/codeverse3d/judges/prompt_builder.py
@@ -15,9 +15,7 @@ from __future__ import annotations
import hashlib
import math
-import os
import random
-import threading
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Literal
@@ -40,6 +38,7 @@ from codeverse3d.contracts.plan import AcceptanceItem
from codeverse3d.contracts.spec import Spec
from codeverse3d.conventions import OBJECT_CLAY_VIEWS, OBJECT_VIEWS, SCENE_VIEWS, ViewPreset
from codeverse3d.judges.rubrics import VETO_PENETRATION_DEPTH_M, Rubric
+from codeverse3d.proc import unique_tmp
from codeverse3d.spatial.connectivity import PENETRATION_ERROR_M, PENETRATION_WARN_M
from codeverse3d.spatial.measure import measure_summary_table
from codeverse3d.spatial.sheet import crop_region, load_font, montage_2x2
@@ -81,12 +80,6 @@ def view_az_el(view: RenderView) -> tuple[float, float] | None:
return None
-def _cache_key(src: Path, max_px: int, label: str) -> str:
- st = src.stat()
- h = hashlib.sha1(f"{src.resolve()}|{st.st_mtime_ns}|{st.st_size}|{max_px}|{label}".encode())
- return h.hexdigest()[:20]
-
-
def prepare_image(
src: str | Path, *, label: str = "", max_px: int = 768, cache_dir: Path | None = None
) -> Path:
@@ -94,12 +87,9 @@ def prepare_image(
Returns the cached PNG path. Raises ``JudgeImageError`` if the source is missing.
"""
- src = Path(src)
- if not src.is_file():
- raise JudgeImageError(f"judge image missing: {src}")
cache = Path(cache_dir) if cache_dir else default_cache_dir()
cache.mkdir(parents=True, exist_ok=True)
- out = cache / f"{_cache_key(src, max_px, label)}.png"
+ out = cache / f"{_key([_stat(str(src)), str(max_px), label])}.png" # _stat raises JudgeImageError
if out.is_file():
return out
with Image.open(src) as im:
@@ -110,7 +100,7 @@ def prepare_image(
im = im.resize((max(1, round(w * scale)), max(1, round(h * scale))), Image.LANCZOS)
if label:
im = _with_label_strip(im, label)
- tmp = out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png") # unique per writer: concurrent judges share the cache
+ tmp = unique_tmp(out) # unique per writer: concurrent judges share the cache
im.save(tmp, format="PNG", optimize=True)
tmp.replace(out) # atomic; a concurrent identical write simply wins last
return out
@@ -353,10 +343,6 @@ def _stat(p: str) -> str:
return f"{path.resolve()}|{st.st_mtime_ns}|{st.st_size}"
-def _tmp(out: Path) -> Path:
- return out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png")
-
-
def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int = 512) -> Path:
"""Build (or fetch from cache) the PNG for one montage; pass-through sheets return their source.
@@ -372,13 +358,13 @@ def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int =
v = m.tiles[0]
out = cache / f"crop_{_key([_stat(v.path), str(m.crop), str(tile_px)])}.png"
if not out.is_file():
- tmp = crop_region(v.path, _tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)
+ tmp = crop_region(v.path, unique_tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)
tmp.replace(out)
return out
images = [(tile_label(v), v.path) for v in m.tiles]
out = cache / f"montage_{_key([_stat(p) + '|' + lbl for lbl, p in images] + [str(tile_px)])}.png"
if not out.is_file():
- tmp = montage_2x2(images, _tmp(out), tile=tile_px)
+ tmp = montage_2x2(images, unique_tmp(out), tile=tile_px)
tmp.replace(out)
return out
@@ -512,7 +498,7 @@ def measurement_section(m: Measurement | None) -> str:
#: MEASURED STRUCTURE clips. p90 of the 321 stored static_object rounds that report a
-#: contact count is 26 contacts (2026-08-30, scratch corpus_ledger.py over bench/out); the
+#: contact count is 26 contacts (2026-08-30, scratch corpus_ledger.py over eval/bench/out); the
#: whole block has to stay near 500 tokens on such a round, so contacts and planned joins
#: are cut with an "… n more" rather than listed to the end.
LEDGER_MAX_CONTACTS = 24
@@ -548,7 +534,7 @@ def contact_ledger(gates: list[GateReport]) -> tuple[GateReport, GateFinding] |
def gates_section(gates: list[GateReport], *, max_errors: int = 12, max_warns: int = 8) -> str:
"""The judge's gate facts. ERRORs are listed as recorded. With a contact ledger the
connectivity WARNs are not prose any more: the audit of 420 judged static rounds
- (2026-08-30, docs/EVAL.md §6) found the judge marking interpenetration on every gate
+ (2026-08-30, eval/docs/EVAL.md §6) found the judge marking interpenetration on every gate
ERROR (69/69) and on 110 rounds whose only evidence was a WARN the rubric tells it to
ignore — it read the gate's sentences, not the images (same images, gate text removed:
13 of 24 flags flipped). So the WARNs become one measured line and a MEASURED STRUCTURE
@@ -827,7 +813,7 @@ RIG_RULES: dict[str, str] = {
),
"geometry": "The GEOMETRY-ONLY montage shows the same object without materials/lighting: use it for holes, inverted (black) faces, intersections and floating parts; use the SHADED montage for materials and detail.",
"poses": "POSE tiles show the SAME object with joints moved by the harness (tile label = joint@value or rest). Judge articulation only from them and the joint table.",
- "scene_cams": "Views named overview_* are harness cameras fitted to the scene bounds (layout X-ray); views named cam_* are the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.",
+ "scene_cams": "Views named overview_* and eye_* are harness cameras fitted to the scene bounds (layout X-ray); every other view is one of the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.",
"scene_craft": "On the authored cameras also read the CRAFT of the picture, not only its contents: depth layering (is there anything within a few metres framing the shot, and anything on the horizon), ground variation (blended materials, paths, dressed edges vs one flat colour), variety among repeated natural elements, small-prop dressing, and aerial perspective (distant things hazier than near ones). The same camera at two times appears as separate tiles — compare them pixel-for-pixel before answering nothing_moves.",
"object": "All views show the same object. Use top + low views for symmetry, footprint and ground contact.",
}
diff --git a/harness/codeverse3d/judges/rubrics.py b/harness/codeverse3d/judges/rubrics.py
index 2b029289..ee7d8b6b 100644
--- a/harness/codeverse3d/judges/rubrics.py
+++ b/harness/codeverse3d/judges/rubrics.py
@@ -684,7 +684,7 @@ SCORING_VERSION = 2
"""The arithmetic downstream of the model, stamped into every ``ScoreBreakdown``.
A stored verdict is reproducible from its samples only under the version that wrote
-it (``bench/rejudge_offline.py --identity`` compares nothing else); an older stamp is
+it (``eval/bench/rejudge_offline.py --identity`` compares nothing else); an older stamp is
drift to report, not a fault.
0 — before 2026-08-26 (no stamp on disk): majority vote, penalties, the cap ladder.
diff --git a/harness/codeverse3d/judges/vlm_judge.py b/harness/codeverse3d/judges/vlm_judge.py
index 50173273..d4c3f8e7 100644
--- a/harness/codeverse3d/judges/vlm_judge.py
+++ b/harness/codeverse3d/judges/vlm_judge.py
@@ -18,24 +18,23 @@ to treat as a glitch, never as a score.
from __future__ import annotations
import contextlib
-import hashlib
import logging
-import os
import re
import shutil
-import threading
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
-from codeverse3d.config import get_settings
+from codeverse3d.config import env_flag, get_settings
from codeverse3d.contracts.artifacts import Judgment, RenderView
from codeverse3d.contracts.chat import ChatRequest, ChatResponse
from codeverse3d.contracts.common import Usage
from codeverse3d.judges.base import SLICE_TRACKS, JudgeInput
from codeverse3d.judges.prompt_builder import (
+ _key,
+ _stat,
build_judge_messages,
connectivity_error_pairs,
default_cache_dir,
@@ -52,7 +51,7 @@ from codeverse3d.judges.rubrics import (
wire_schema,
)
from codeverse3d.models.base import ChatModel, ModelError
-from codeverse3d.proc import fan_out
+from codeverse3d.proc import fan_out, unique_tmp
from codeverse3d.reference import compare, conflict_note
from codeverse3d.spatial.silhouette import best_view_match, compare_silhouette
@@ -280,13 +279,10 @@ class VlmJudge:
from codeverse3d.spatial.sections import SliceManifest, judge_slices
cache = Path(self.cache_dir) if self.cache_dir else default_cache_dir()
- st = glb.stat()
- key = hashlib.sha1(
- f"{glb.resolve()}|{st.st_mtime_ns}|{st.st_size}|{pairs}".encode()).hexdigest()[:20]
- out_dir = cache / f"slices_{key}"
+ out_dir = cache / f"slices_{_key([_stat(str(glb)), str(pairs)])}"
mpath = out_dir / "manifest.json"
if not mpath.is_file():
- tmp = cache / f"slices_{key}.{os.getpid()}-{threading.get_ident()}.tmp"
+ tmp = unique_tmp(out_dir)
shutil.rmtree(tmp, ignore_errors=True)
try:
judge_slices(glb, pairs, tmp)
@@ -359,7 +355,8 @@ def iou_to_score(iou: float) -> float:
def _diff_enabled(flag: bool) -> bool:
- return flag and os.environ.get(_ENV_DIFF, "on").strip().lower() not in ("0", "off", "false", "no")
+ """``$C3D_REFERENCE_DIFF`` at call time; a typo reads as OFF (``config.env_flag``)."""
+ return flag and env_flag(_ENV_DIFF, True)
def _is_synth(note: str) -> bool:
diff --git a/harness/tests/judges/test_pairwise.py b/harness/tests/judges/test_pairwise.py
index 67e4c7a1..64642b85 100644
--- a/harness/tests/judges/test_pairwise.py
+++ b/harness/tests/judges/test_pairwise.py
@@ -68,3 +68,27 @@ def test_parallel_orderings_accumulate_in_order(tmp_path, cache_dir):
assert res.usage.cost_usd == pytest.approx(0.002) and res.usage.input_tokens == 2000
+
+
+def test_a_scene_pair_is_ranked_as_a_scene_from_the_views_the_judge_saw(tmp_path, cache_dir):
+ """Every candidate used to be ranked as an OBJECT from its whole render set: a scene's
+ own camera sank behind the overview rig, and a view the verdict judge never saw
+ (``judge=False``) could fill a montage slot."""
+ from codeverse3d.contracts.artifacts import RenderSet, RenderView
+ from codeverse3d.contracts.common import Language, Track
+ from tests.judges.conftest import draw_chair
+
+ def scene_renders(d):
+ d.mkdir(parents=True)
+ names = [("overview_front_right", True), ("overview_back_left", True), ("overview_top", True),
+ ("eye_front", True), ("unjudged_extra", False), ("Establishing", True)]
+ return RenderSet(views=[RenderView(name=n, path=str(draw_chair(d / f"{n}.png")), judge=j)
+ for n, j in names], renderer="fake")
+
+ model = FakeChatModel(by_label={":fwd": [_reply("A")], ":swap": [_reply("B")]})
+ spec = make_spec(track=Track.SCENE, language=Language.SCENE_THREEJS)
+ PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare(
+ spec, scene_renders(tmp_path / "a"), scene_renders(tmp_path / "b"))
+ labels = [p.label for p in image_parts(next(r for r in model.requests if r.label == "pairwise:fwd"))]
+ assert "top-left = Establishing" in labels[0]
+ assert not any("unjudged_extra" in lbl for lbl in labels)
diff --git a/harness/tests/judges/test_prompt_builder.py b/harness/tests/judges/test_prompt_builder.py
index 6893d279..5065512b 100644
--- a/harness/tests/judges/test_prompt_builder.py
+++ b/harness/tests/judges/test_prompt_builder.py
@@ -34,6 +34,18 @@ def _images(msgs):
return [p for p in _parts(msgs) if isinstance(p, ImagePart)]
+def test_the_scene_rig_rule_names_every_harness_camera_and_no_authored_prefix():
+ """Authored cameras carry the plan's names (``cam_i`` is only scene_host's fallback for an
+ unnamed one) and the eye-level rig is the harness's too: the judge was told authored views
+ are ``cam_*`` and nothing about ``eye_*``."""
+ from codeverse3d.conventions import SCENE_VIEWS
+ from codeverse3d.judges.prompt_builder import RIG_RULES
+
+ rule = RIG_RULES["scene_cams"]
+ assert "cam_*" not in rule
+ assert all(f"{v.name.split('_')[0]}_*" in rule for v in SCENE_VIEWS)
+
+
def test_system_has_role_rubric_anchors_and_defects():
system = build_system_prompt(R)
assert "BLIND JUDGE" in system
diff --git a/harness/tests/reference/test_judge_diff.py b/harness/tests/reference/test_judge_diff.py
index 31615804..0c18cb9c 100644
--- a/harness/tests/reference/test_judge_diff.py
+++ b/harness/tests/reference/test_judge_diff.py
@@ -76,8 +76,10 @@ def test_diff_can_be_switched_off(ref_input, cache_dir):
assert all(not r.label.startswith("reference_diff") for r in model.requests)
-def test_diff_off_via_env(ref_input, cache_dir, monkeypatch):
- monkeypatch.setenv("C3D_REFERENCE_DIFF", "off")
[email protected]("value", ["off", "of"])
+def test_diff_off_via_env(ref_input, cache_dir, monkeypatch, value):
+ """A typo is a control run, not the variant (config.env_flag): "of" used to read as ON."""
+ monkeypatch.setenv("C3D_REFERENCE_DIFF", value)
model = FakeChatModel(by_label={"judge": [good_reply(REF, ["A1", "A2"], 0.8)]})
_judge(model, cache_dir=cache_dir).judge(ref_input)
assert all(not r.label.startswith("reference_diff") for r in model.requests)