Official Codebase of 3DCodeVerse
Python
judges: one cache key + tmp name, the scene camera rule the renders obey, scene-ranked pairwise, env_flag for the diff switch
gaoypengd888f0e89b02
diff --git a/harness/codeverse3d/judges/pairwise.py b/harness/codeverse3d/judges/pairwise.pyindex 58f73530..4b6cadff 100644--- a/harness/codeverse3d/judges/pairwise.py+++ b/harness/codeverse3d/judges/pairwise.py@@ -22,8 +22,9 @@ from pydantic import BaseModel, Fieldfrom codeverse3d.config import get_settingsfrom codeverse3d.contracts.artifacts import RenderSetfrom codeverse3d.contracts.chat import ChatMessage, ChatRequest, ImagePart, TextPart-from codeverse3d.contracts.common import Usage+from codeverse3d.contracts.common import Track, Usagefrom codeverse3d.contracts.spec import Spec+from codeverse3d.judges.base import judged_subsetfrom codeverse3d.judges.prompt_builder import (MONTAGE_TILE_PX,brief_section,@@ -167,7 +168,7 @@ class PairwiseJudge:)parts: list[TextPart | ImagePart] = [TextPart(text=text)]for tag, rs in (("A", first), ("B", second)):- for lbl, ip in self._side_images(tag, rs):+ for lbl, ip in self._side_images(tag, rs, scene=spec.track is Track.SCENE):parts.append(TextPart(text=lbl))parts.append(ip)parts.append(TextPart(text="Compare A and B and return the JSON object."))@@ -177,9 +178,11 @@ class PairwiseJudge:max_wait_s=900.0, label=label,)- def _side_images(self, tag: str, rs: RenderSet) -> list[tuple[str, ImagePart]]:- """One 2×2 montage of the candidate's most informative views (plus its pose sheet / geometry montage if any)."""- montages = plan_montages(rs, scene=False, max_montages=2, detail_crops=0)+ def _side_images(self, tag: str, rs: RenderSet, *, scene: bool) -> list[tuple[str, ImagePart]]:+ """One 2×2 montage of the candidate's most informative views (plus its pose sheet / geometry+ montage if any), from the views the verdict judge saw, ranked for the track: a scene's+ own cameras first, as the verdict judge ranks them."""+ montages = plan_montages(judged_subset(rs), scene=scene, max_montages=2, detail_crops=0)out: list[tuple[str, ImagePart]] = []for i, m in enumerate(montages, 1):if len(m.tiles) > self.views_per_side:diff --git a/harness/codeverse3d/judges/prompt_builder.py b/harness/codeverse3d/judges/prompt_builder.pyindex 10bacac7..c882cc57 100644--- a/harness/codeverse3d/judges/prompt_builder.py+++ b/harness/codeverse3d/judges/prompt_builder.py@@ -15,9 +15,7 @@ from __future__ import annotationsimport hashlibimport math-import osimport random-import threadingfrom dataclasses import dataclass, replacefrom pathlib import Pathfrom typing import TYPE_CHECKING, Literal@@ -40,6 +38,7 @@ from codeverse3d.contracts.plan import AcceptanceItemfrom codeverse3d.contracts.spec import Specfrom codeverse3d.conventions import OBJECT_CLAY_VIEWS, OBJECT_VIEWS, SCENE_VIEWS, ViewPresetfrom codeverse3d.judges.rubrics import VETO_PENETRATION_DEPTH_M, Rubric+from codeverse3d.proc import unique_tmpfrom codeverse3d.spatial.connectivity import PENETRATION_ERROR_M, PENETRATION_WARN_Mfrom codeverse3d.spatial.measure import measure_summary_tablefrom codeverse3d.spatial.sheet import crop_region, load_font, montage_2x2@@ -81,12 +80,6 @@ def view_az_el(view: RenderView) -> tuple[float, float] | None:return None-def _cache_key(src: Path, max_px: int, label: str) -> str:- st = src.stat()- h = hashlib.sha1(f"{src.resolve()}|{st.st_mtime_ns}|{st.st_size}|{max_px}|{label}".encode())- return h.hexdigest()[:20]--def prepare_image(src: str | Path, *, label: str = "", max_px: int = 768, cache_dir: Path | None = None) -> Path:@@ -94,12 +87,9 @@ def prepare_image(Returns the cached PNG path. Raises ``JudgeImageError`` if the source is missing."""- src = Path(src)- if not src.is_file():- raise JudgeImageError(f"judge image missing: {src}")cache = Path(cache_dir) if cache_dir else default_cache_dir()cache.mkdir(parents=True, exist_ok=True)- out = cache / f"{_cache_key(src, max_px, label)}.png"+ out = cache / f"{_key([_stat(str(src)), str(max_px), label])}.png" # _stat raises JudgeImageErrorif out.is_file():return outwith Image.open(src) as im:@@ -110,7 +100,7 @@ def prepare_image(im = im.resize((max(1, round(w * scale)), max(1, round(h * scale))), Image.LANCZOS)if label:im = _with_label_strip(im, label)- tmp = out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png") # unique per writer: concurrent judges share the cache+ tmp = unique_tmp(out) # unique per writer: concurrent judges share the cacheim.save(tmp, format="PNG", optimize=True)tmp.replace(out) # atomic; a concurrent identical write simply wins lastreturn out@@ -353,10 +343,6 @@ def _stat(p: str) -> str:return f"{path.resolve()}|{st.st_mtime_ns}|{st.st_size}"-def _tmp(out: Path) -> Path:- return out.with_name(f"{out.stem}.{os.getpid()}-{threading.get_ident()}.tmp.png")--def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int = 512) -> Path:"""Build (or fetch from cache) the PNG for one montage; pass-through sheets return their source.@@ -372,13 +358,13 @@ def render_montage(m: Montage, *, cache_dir: Path | None = None, tile_px: int =v = m.tiles[0]out = cache / f"crop_{_key([_stat(v.path), str(m.crop), str(tile_px)])}.png"if not out.is_file():- tmp = crop_region(v.path, _tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)+ tmp = crop_region(v.path, unique_tmp(out), m.crop or (0.0, 0.0, 1.0, 1.0), min_px=tile_px)tmp.replace(out)return outimages = [(tile_label(v), v.path) for v in m.tiles]out = cache / f"montage_{_key([_stat(p) + '|' + lbl for lbl, p in images] + [str(tile_px)])}.png"if not out.is_file():- tmp = montage_2x2(images, _tmp(out), tile=tile_px)+ tmp = montage_2x2(images, unique_tmp(out), tile=tile_px)tmp.replace(out)return out@@ -512,7 +498,7 @@ def measurement_section(m: Measurement | None) -> str:#: MEASURED STRUCTURE clips. p90 of the 321 stored static_object rounds that report a-#: contact count is 26 contacts (2026-08-30, scratch corpus_ledger.py over bench/out); the+#: contact count is 26 contacts (2026-08-30, scratch corpus_ledger.py over eval/bench/out); the#: whole block has to stay near 500 tokens on such a round, so contacts and planned joins#: are cut with an "… n more" rather than listed to the end.LEDGER_MAX_CONTACTS = 24@@ -548,7 +534,7 @@ def contact_ledger(gates: list[GateReport]) -> tuple[GateReport, GateFinding] |def gates_section(gates: list[GateReport], *, max_errors: int = 12, max_warns: int = 8) -> str:"""The judge's gate facts. ERRORs are listed as recorded. With a contact ledger theconnectivity WARNs are not prose any more: the audit of 420 judged static rounds- (2026-08-30, docs/EVAL.md §6) found the judge marking interpenetration on every gate+ (2026-08-30, eval/docs/EVAL.md §6) found the judge marking interpenetration on every gateERROR (69/69) and on 110 rounds whose only evidence was a WARN the rubric tells it toignore — it read the gate's sentences, not the images (same images, gate text removed:13 of 24 flags flipped). So the WARNs become one measured line and a MEASURED STRUCTURE@@ -827,7 +813,7 @@ RIG_RULES: dict[str, str] = {),"geometry": "The GEOMETRY-ONLY montage shows the same object without materials/lighting: use it for holes, inverted (black) faces, intersections and floating parts; use the SHADED montage for materials and detail.","poses": "POSE tiles show the SAME object with joints moved by the harness (tile label = joint@value or rest). Judge articulation only from them and the joint table.",- "scene_cams": "Views named overview_* are harness cameras fitted to the scene bounds (layout X-ray); views named cam_* are the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.",+ "scene_cams": "Views named overview_* and eye_* are harness cameras fitted to the scene bounds (layout X-ray); every other view is one of the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.","scene_craft": "On the authored cameras also read the CRAFT of the picture, not only its contents: depth layering (is there anything within a few metres framing the shot, and anything on the horizon), ground variation (blended materials, paths, dressed edges vs one flat colour), variety among repeated natural elements, small-prop dressing, and aerial perspective (distant things hazier than near ones). The same camera at two times appears as separate tiles — compare them pixel-for-pixel before answering nothing_moves.","object": "All views show the same object. Use top + low views for symmetry, footprint and ground contact.",}diff --git a/harness/codeverse3d/judges/rubrics.py b/harness/codeverse3d/judges/rubrics.pyindex 2b029289..ee7d8b6b 100644--- a/harness/codeverse3d/judges/rubrics.py+++ b/harness/codeverse3d/judges/rubrics.py@@ -684,7 +684,7 @@ SCORING_VERSION = 2"""The arithmetic downstream of the model, stamped into every ``ScoreBreakdown``.A stored verdict is reproducible from its samples only under the version that wrote-it (``bench/rejudge_offline.py --identity`` compares nothing else); an older stamp is+it (``eval/bench/rejudge_offline.py --identity`` compares nothing else); an older stamp isdrift to report, not a fault.0 — before 2026-08-26 (no stamp on disk): majority vote, penalties, the cap ladder.diff --git a/harness/codeverse3d/judges/vlm_judge.py b/harness/codeverse3d/judges/vlm_judge.pyindex 50173273..d4c3f8e7 100644--- a/harness/codeverse3d/judges/vlm_judge.py+++ b/harness/codeverse3d/judges/vlm_judge.py@@ -18,24 +18,23 @@ to treat as a glitch, never as a score.from __future__ import annotationsimport contextlib-import hashlibimport logging-import osimport reimport shutil-import threadingimport timefrom collections.abc import Callablefrom dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Literal-from codeverse3d.config import get_settings+from codeverse3d.config import env_flag, get_settingsfrom codeverse3d.contracts.artifacts import Judgment, RenderViewfrom codeverse3d.contracts.chat import ChatRequest, ChatResponsefrom codeverse3d.contracts.common import Usagefrom codeverse3d.judges.base import SLICE_TRACKS, JudgeInputfrom codeverse3d.judges.prompt_builder import (+ _key,+ _stat,build_judge_messages,connectivity_error_pairs,default_cache_dir,@@ -52,7 +51,7 @@ from codeverse3d.judges.rubrics import (wire_schema,)from codeverse3d.models.base import ChatModel, ModelError-from codeverse3d.proc import fan_out+from codeverse3d.proc import fan_out, unique_tmpfrom codeverse3d.reference import compare, conflict_notefrom codeverse3d.spatial.silhouette import best_view_match, compare_silhouette@@ -280,13 +279,10 @@ class VlmJudge:from codeverse3d.spatial.sections import SliceManifest, judge_slicescache = Path(self.cache_dir) if self.cache_dir else default_cache_dir()- st = glb.stat()- key = hashlib.sha1(- f"{glb.resolve()}|{st.st_mtime_ns}|{st.st_size}|{pairs}".encode()).hexdigest()[:20]- out_dir = cache / f"slices_{key}"+ out_dir = cache / f"slices_{_key([_stat(str(glb)), str(pairs)])}"mpath = out_dir / "manifest.json"if not mpath.is_file():- tmp = cache / f"slices_{key}.{os.getpid()}-{threading.get_ident()}.tmp"+ tmp = unique_tmp(out_dir)shutil.rmtree(tmp, ignore_errors=True)try:judge_slices(glb, pairs, tmp)@@ -359,7 +355,8 @@ def iou_to_score(iou: float) -> float:def _diff_enabled(flag: bool) -> bool:- return flag and os.environ.get(_ENV_DIFF, "on").strip().lower() not in ("0", "off", "false", "no")+ """``$C3D_REFERENCE_DIFF`` at call time; a typo reads as OFF (``config.env_flag``)."""+ return flag and env_flag(_ENV_DIFF, True)def _is_synth(note: str) -> bool:diff --git a/harness/tests/judges/test_pairwise.py b/harness/tests/judges/test_pairwise.pyindex 67e4c7a1..64642b85 100644--- a/harness/tests/judges/test_pairwise.py+++ b/harness/tests/judges/test_pairwise.py@@ -68,3 +68,27 @@ def test_parallel_orderings_accumulate_in_order(tmp_path, cache_dir):assert res.usage.cost_usd == pytest.approx(0.002) and res.usage.input_tokens == 2000+++def test_a_scene_pair_is_ranked_as_a_scene_from_the_views_the_judge_saw(tmp_path, cache_dir):+ """Every candidate used to be ranked as an OBJECT from its whole render set: a scene's+ own camera sank behind the overview rig, and a view the verdict judge never saw+ (``judge=False``) could fill a montage slot."""+ from codeverse3d.contracts.artifacts import RenderSet, RenderView+ from codeverse3d.contracts.common import Language, Track+ from tests.judges.conftest import draw_chair++ def scene_renders(d):+ d.mkdir(parents=True)+ names = [("overview_front_right", True), ("overview_back_left", True), ("overview_top", True),+ ("eye_front", True), ("unjudged_extra", False), ("Establishing", True)]+ return RenderSet(views=[RenderView(name=n, path=str(draw_chair(d / f"{n}.png")), judge=j)+ for n, j in names], renderer="fake")++ model = FakeChatModel(by_label={":fwd": [_reply("A")], ":swap": [_reply("B")]})+ spec = make_spec(track=Track.SCENE, language=Language.SCENE_THREEJS)+ PairwiseJudge("fake:fake-1", chat_model=model, cache_dir=cache_dir).compare(+ spec, scene_renders(tmp_path / "a"), scene_renders(tmp_path / "b"))+ labels = [p.label for p in image_parts(next(r for r in model.requests if r.label == "pairwise:fwd"))]+ assert "top-left = Establishing" in labels[0]+ assert not any("unjudged_extra" in lbl for lbl in labels)diff --git a/harness/tests/judges/test_prompt_builder.py b/harness/tests/judges/test_prompt_builder.pyindex 6893d279..5065512b 100644--- a/harness/tests/judges/test_prompt_builder.py+++ b/harness/tests/judges/test_prompt_builder.py@@ -34,6 +34,18 @@ def _images(msgs):return [p for p in _parts(msgs) if isinstance(p, ImagePart)]+def test_the_scene_rig_rule_names_every_harness_camera_and_no_authored_prefix():+ """Authored cameras carry the plan's names (``cam_i`` is only scene_host's fallback for an+ unnamed one) and the eye-level rig is the harness's too: the judge was told authored views+ are ``cam_*`` and nothing about ``eye_*``."""+ from codeverse3d.conventions import SCENE_VIEWS+ from codeverse3d.judges.prompt_builder import RIG_RULES++ rule = RIG_RULES["scene_cams"]+ assert "cam_*" not in rule+ assert all(f"{v.name.split('_')[0]}_*" in rule for v in SCENE_VIEWS)++def test_system_has_role_rubric_anchors_and_defects():system = build_system_prompt(R)assert "BLIND JUDGE" in systemdiff --git a/harness/tests/reference/test_judge_diff.py b/harness/tests/reference/test_judge_diff.pyindex 31615804..0c18cb9c 100644--- a/harness/tests/reference/test_judge_diff.py+++ b/harness/tests/reference/test_judge_diff.py@@ -76,8 +76,10 @@ def test_diff_can_be_switched_off(ref_input, cache_dir):assert all(not r.label.startswith("reference_diff") for r in model.requests)-def test_diff_off_via_env(ref_input, cache_dir, monkeypatch):- monkeypatch.setenv("C3D_REFERENCE_DIFF", "off")[email protected]("value", ["off", "of"])+def test_diff_off_via_env(ref_input, cache_dir, monkeypatch, value):+ """A typo is a control run, not the variant (config.env_flag): "of" used to read as ON."""+ monkeypatch.setenv("C3D_REFERENCE_DIFF", value)model = FakeChatModel(by_label={"judge": [good_reply(REF, ["A1", "A2"], 0.8)]})_judge(model, cache_dir=cache_dir).judge(ref_input)assert all(not r.label.startswith("reference_diff") for r in model.requests)