Skip to content

Official Codebase of 3DCodeVerse

Python

bench: the fixed evaluator judges graphics cells on their frames (shader_v2; LikenessJudge with the spec's photos)

gaoypeng06cb5d366a62
diff --git a/harness/bench/_fixed_eval.py b/harness/bench/_fixed_eval.py
index e8767d0e..38d46b76 100644
--- a/harness/bench/_fixed_eval.py
+++ b/harness/bench/_fixed_eval.py
@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field
from codeverse.config import Settings, get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet
-from codeverse.contracts.common import Language
+from codeverse.contracts.common import Language, Track
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.spec import Spec
@@ -83,6 +83,28 @@ class FixedEvaluator:
self._judge = VlmJudge(rubric=self.rubric, model_id=self.judge_model, n_samples=self.n_samples)
return self._judge
+ def judge_for(self, spec: Spec) -> Any:
+ """The fixed judge for THIS cell's track.
+
+ The evaluator judged every cell with the object rubric on a GLB. A graphics cell
+ has frames, not a GLB, so ``evaluate`` returned before judging and every graphics
+ cell of an ``ab_plan`` battery was recorded ``judge_error: no judgment`` — measured
+ 2026-08-26 on ``seed_v1`` (both arms, three rounds of in-loop verdicts each, and
+ no pair score). Graphics cells are judged on their frame sheet with the track's
+ rubric (``shader_v2``), through ``LikenessJudge`` when the spec carries reference
+ photos so the arms see the same photos the loop saw.
+ """
+ if spec.track is Track.GRAPHICS:
+ from codeverse.contracts.common import TRACK_INFO
+ from codeverse.judges.reference import LikenessJudge
+ from codeverse.judges.vlm_judge import VlmJudge
+
+ rubric = TRACK_INFO[Track.GRAPHICS].rubric
+ if spec.references:
+ return LikenessJudge(self.judge_model, n_samples=self.n_samples, rubric=rubric)
+ return VlmJudge(rubric=rubric, model_id=self.judge_model, n_samples=self.n_samples)
+ return self.judge
+
def build(self, ws: Workspace, language: Language | str) -> tuple[BuildResult, GateReport]:
rt = self.runtime(language)
lint = rt.lint(ws)
@@ -97,6 +119,8 @@ class FixedEvaluator:
build, lint = self.build(ws, spec.language)
out = EvalOutcome(build=build, lint=lint, gates=[lint])
+ if build.ok and spec.track is Track.GRAPHICS:
+ return self._evaluate_frames(ws, spec, out)
if not build.ok or not build.glb_path:
return out
try:
@@ -113,4 +137,23 @@ class FixedEvaluator:
return out
+ def _evaluate_frames(self, ws: Workspace, spec: Spec, out: EvalOutcome) -> EvalOutcome:
+ """Graphics: the judged frames + the gl_frames gate + the frame metrics, as the loop does."""
+ from codeverse.judges.base import JudgeInput
+ from codeverse.languages._gl_common import read_metrics
+ from codeverse.tracks.graphics_steps import frame_stats_text, frames_render_set
+
+ try:
+ out.renders = frames_render_set(ws, out.build, 0)
+ m = read_metrics(ws)
+ if m is not None:
+ out.gates.append(m[1])
+ inp = JudgeInput(spec=spec, renders=out.renders, gates=out.gates, acceptance=acceptance_from_spec(spec),
+ round_index=0, extra_context="FRAME METRICS (harness-measured):\n" + frame_stats_text(ws))
+ out.judgment = self.judge_for(spec).judge(inp)
+ except Exception as e: # noqa: BLE001 — recorded per cell, never kills the matrix
+ out.error = f"{type(e).__name__}: {e}"
+ return out
+
+
__all__ = ["RUBRIC", "EvalOutcome", "FixedEvaluator", "acceptance_from_spec"]
diff --git a/harness/tests/compare_bench/test_fixed_eval_language.py b/harness/tests/compare_bench/test_fixed_eval_language.py
index 20d7107d..fade6860 100644
--- a/harness/tests/compare_bench/test_fixed_eval_language.py
+++ b/harness/tests/compare_bench/test_fixed_eval_language.py
@@ -23,3 +23,57 @@ def test_the_evaluator_asks_for_the_cells_own_runtime(monkeypatch):
assert ev.runtime("blender").lang is Language.BLENDER
assert ev.runtime(Language.THREEJS).lang is Language.THREEJS, "cached per language, not one global"
assert asked == [Language.THREEJS, Language.BLENDER], "each language's runtime is constructed once"
+
+
+def test_graphics_cells_are_judged_on_their_frames(monkeypatch, tmp_path):
+ """Measured 2026-08-26 (seed_v1): a graphics cell has frames, not a GLB, so the fixed
+ evaluator returned before judging and every cell was `judge_error: no judgment`."""
+ from types import SimpleNamespace
+
+ from bench._fixed_eval import FixedEvaluator
+ from codeverse.contracts.artifacts import BuildResult, GateReport, RenderSet, RenderView
+ from codeverse.contracts.common import Track
+ from codeverse.contracts.spec import ReferenceImage, Spec
+ from codeverse.workspace import Workspace
+
+ ws = Workspace(tmp_path / "run").create()
+ png = tmp_path / "f.png"
+ png.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 16)
+ build = BuildResult(ok=True, language="glsl_shader", glb_path=None)
+ ev = FixedEvaluator("fake:judge", n_samples=1)
+ ev._runtimes[Language.GLSL_SHADER] = SimpleNamespace(lint=lambda ws: GateReport(gate="lint", passed=True), build=lambda ws, timeout_s: build)
+ import bench._fixed_eval as fe
+ import codeverse.tracks.graphics_steps as gs
+
+ monkeypatch.setattr(gs, "frames_render_set", lambda ws, b, i: RenderSet(views=[RenderView(name="t=0s", path=str(png))], renderer="fake"))
+ monkeypatch.setattr(gs, "frame_stats_text", lambda ws: "frames=1")
+ seen: dict[str, object] = {}
+
+ class _J:
+ def judge(self, inp):
+ seen["inp"] = inp
+ return SimpleNamespace(overall=0.71, passed=True)
+
+ monkeypatch.setattr(fe.FixedEvaluator, "judge_for", lambda self, spec: (seen.__setitem__("spec", spec), _J())[1])
+ spec = Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="an aurora",
+ references=[ReferenceImage(path=str(png), role="likeness")])
+ out = ev.evaluate(ws, spec)
+ assert out.error == "", out.error
+ assert out.judgment.overall == 0.71
+ assert seen["inp"].renders.views[0].path == str(png) and "frames=1" in seen["inp"].extra_context
+
+
+def test_graphics_judge_is_the_track_rubric_with_the_photos():
+ from bench._fixed_eval import FixedEvaluator
+ from codeverse.contracts.common import Track
+ from codeverse.contracts.spec import ReferenceImage, Spec
+ from codeverse.judges.reference import LikenessJudge
+
+ ev = FixedEvaluator("gemini:gemini-3.1-pro-preview", n_samples=2)
+ with_photos = Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="x", references=[ReferenceImage(path="/p.png")])
+ j = ev.judge_for(with_photos)
+ assert isinstance(j, LikenessJudge) and j.rubric.name == "shader_v2"
+ plain = ev.judge_for(Spec(id="g", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="x"))
+ assert plain.rubric.name == "shader_v2" and not isinstance(plain, LikenessJudge)
+ obj = ev.judge_for(Spec(id="o", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="x"))
+ assert obj.rubric.name == "static_object_v1"