Official Codebase of 3DCodeVerse
Python
bench: the fixed evaluator judges graphics cells on their frames (shader_v2; LikenessJudge with the spec's photos)
gaoypeng06cb5d366a62
diff --git a/harness/bench/_fixed_eval.py b/harness/bench/_fixed_eval.pyindex e8767d0e..38d46b76 100644--- a/harness/bench/_fixed_eval.py+++ b/harness/bench/_fixed_eval.py@@ -16,7 +16,7 @@ from pydantic import BaseModel, Fieldfrom codeverse.config import Settings, get_settingsfrom codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet-from codeverse.contracts.common import Language+from codeverse.contracts.common import Language, Trackfrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.spec import Spec@@ -83,6 +83,28 @@ class FixedEvaluator:self._judge = VlmJudge(rubric=self.rubric, model_id=self.judge_model, n_samples=self.n_samples)return self._judge+ def judge_for(self, spec: Spec) -> Any:+ """The fixed judge for THIS cell's track.++ The evaluator judged every cell with the object rubric on a GLB. A graphics cell+ has frames, not a GLB, so ``evaluate`` returned before judging and every graphics+ cell of an ``ab_plan`` battery was recorded ``judge_error: no judgment`` — measured+ 2026-08-26 on ``seed_v1`` (both arms, three rounds of in-loop verdicts each, and+ no pair score). Graphics cells are judged on their frame sheet with the track's+ rubric (``shader_v2``), through ``LikenessJudge`` when the spec carries reference+ photos so the arms see the same photos the loop saw.+ """+ if spec.track is Track.GRAPHICS:+ from codeverse.contracts.common import TRACK_INFO+ from codeverse.judges.reference import LikenessJudge+ from codeverse.judges.vlm_judge import VlmJudge++ rubric = TRACK_INFO[Track.GRAPHICS].rubric+ if spec.references:+ return LikenessJudge(self.judge_model, n_samples=self.n_samples, rubric=rubric)+ return VlmJudge(rubric=rubric, model_id=self.judge_model, n_samples=self.n_samples)+ return self.judge+def build(self, ws: Workspace, language: Language | str) -> tuple[BuildResult, GateReport]:rt = self.runtime(language)lint = rt.lint(ws)@@ -97,6 +119,8 @@ class FixedEvaluator:build, lint = self.build(ws, spec.language)out = EvalOutcome(build=build, lint=lint, gates=[lint])+ if build.ok and spec.track is Track.GRAPHICS:+ return self._evaluate_frames(ws, spec, out)if not build.ok or not build.glb_path:return outtry:@@ -113,4 +137,23 @@ class FixedEvaluator:return out+ def _evaluate_frames(self, ws: Workspace, spec: Spec, out: EvalOutcome) -> EvalOutcome:+ """Graphics: the judged frames + the gl_frames gate + the frame metrics, as the loop does."""+ from codeverse.judges.base import JudgeInput+ from codeverse.languages._gl_common import read_metrics+ from codeverse.tracks.graphics_steps import frame_stats_text, frames_render_set++ try:+ out.renders = frames_render_set(ws, out.build, 0)+ m = read_metrics(ws)+ if m is not None:+ out.gates.append(m[1])+ inp = JudgeInput(spec=spec, renders=out.renders, gates=out.gates, acceptance=acceptance_from_spec(spec),+ round_index=0, extra_context="FRAME METRICS (harness-measured):\n" + frame_stats_text(ws))+ out.judgment = self.judge_for(spec).judge(inp)+ except Exception as e: # noqa: BLE001 — recorded per cell, never kills the matrix+ out.error = f"{type(e).__name__}: {e}"+ return out++__all__ = ["RUBRIC", "EvalOutcome", "FixedEvaluator", "acceptance_from_spec"]diff --git a/harness/tests/compare_bench/test_fixed_eval_language.py b/harness/tests/compare_bench/test_fixed_eval_language.pyindex 20d7107d..fade6860 100644--- a/harness/tests/compare_bench/test_fixed_eval_language.py+++ b/harness/tests/compare_bench/test_fixed_eval_language.py@@ -23,3 +23,57 @@ def test_the_evaluator_asks_for_the_cells_own_runtime(monkeypatch):assert ev.runtime("blender").lang is Language.BLENDERassert ev.runtime(Language.THREEJS).lang is Language.THREEJS, "cached per language, not one global"assert asked == [Language.THREEJS, Language.BLENDER], "each language's runtime is constructed once"+++def test_graphics_cells_are_judged_on_their_frames(monkeypatch, tmp_path):+ """Measured 2026-08-26 (seed_v1): a graphics cell has frames, not a GLB, so the fixed+ evaluator returned before judging and every cell was `judge_error: no judgment`."""+ from types import SimpleNamespace++ from bench._fixed_eval import FixedEvaluator+ from codeverse.contracts.artifacts import BuildResult, GateReport, RenderSet, RenderView+ from codeverse.contracts.common import Track+ from codeverse.contracts.spec import ReferenceImage, Spec+ from codeverse.workspace import Workspace++ ws = Workspace(tmp_path / "run").create()+ png = tmp_path / "f.png"+ png.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)+ build = BuildResult(ok=True, language="glsl_shader", glb_path=None)+ ev = FixedEvaluator("fake:judge", n_samples=1)+ ev._runtimes[Language.GLSL_SHADER] = SimpleNamespace(lint=lambda ws: GateReport(gate="lint", passed=True), build=lambda ws, timeout_s: build)+ import bench._fixed_eval as fe+ import codeverse.tracks.graphics_steps as gs++ monkeypatch.setattr(gs, "frames_render_set", lambda ws, b, i: RenderSet(views=[RenderView(name="t=0s", path=str(png))], renderer="fake"))+ monkeypatch.setattr(gs, "frame_stats_text", lambda ws: "frames=1")+ seen: dict[str, object] = {}++ class _J:+ def judge(self, inp):+ seen["inp"] = inp+ return SimpleNamespace(overall=0.71, passed=True)++ monkeypatch.setattr(fe.FixedEvaluator, "judge_for", lambda self, spec: (seen.__setitem__("spec", spec), _J())[1])+ spec = Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="an aurora",+ references=[ReferenceImage(path=str(png), role="likeness")])+ out = ev.evaluate(ws, spec)+ assert out.error == "", out.error+ assert out.judgment.overall == 0.71+ assert seen["inp"].renders.views[0].path == str(png) and "frames=1" in seen["inp"].extra_context+++def test_graphics_judge_is_the_track_rubric_with_the_photos():+ from bench._fixed_eval import FixedEvaluator+ from codeverse.contracts.common import Track+ from codeverse.contracts.spec import ReferenceImage, Spec+ from codeverse.judges.reference import LikenessJudge++ ev = FixedEvaluator("gemini:gemini-3.1-pro-preview", n_samples=2)+ with_photos = Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="x", references=[ReferenceImage(path="/p.png")])+ j = ev.judge_for(with_photos)+ assert isinstance(j, LikenessJudge) and j.rubric.name == "shader_v2"+ plain = ev.judge_for(Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="x"))+ assert plain.rubric.name == "shader_v2" and not isinstance(plain, LikenessJudge)+ obj = ev.judge_for(Spec(id="o", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="x"))+ assert obj.rubric.name == "static_object_v1"