Skip to content

Official Codebase of 3DCodeVerse

Python

bench: one track→rubric mechanism — _fixed_eval.rubric_for feeds FixedEvaluator, judge_for and the report

gaoypeng18c8f4359090
diff --git a/harness/bench/_compare_report.py b/harness/bench/_compare_report.py
index 0ac4d68c..04729b93 100644
--- a/harness/bench/_compare_report.py
+++ b/harness/bench/_compare_report.py
@@ -17,13 +17,20 @@ from typing import Any, TypeVar
from pydantic import BaseModel, Field
+from bench._fixed_eval import RUBRIC, rubric_for
from bench._jsonl import read_jsonl
+from codeverse.contracts.common import Track
T = TypeVar("T", bound=BaseModel)
-#: The fixed judge's rubric per battery track (bench/_fixed_eval.py builds the evaluator from it).
-RUBRIC_BY_TRACK: dict[str, str] = {"static_object": "static_object_v1", "articulated_object": "articulated_v1"}
+def battery_rubric(meta: dict) -> str:
+ """The fixed judge's rubric the report header names: ``rubric_for`` on the battery's track."""
+ track = str(meta.get("battery", {}).get("track") or Track.STATIC_OBJECT.value)
+ try:
+ return rubric_for(Track(track))
+ except ValueError: # a meta.json written before batteries carried a track
+ return RUBRIC
class CellResult(BaseModel):
@@ -222,7 +229,7 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], me
prompts = _prompt_order(rows, meta)
cells = {(r.prompt_id, r.arm): r for r in rows}
md = [f"# harness vs one-shot — {meta.get('battery', {}).get('name', out.name)}", "",
- f"fixed judge: **{opts.get('judge', '?')}** (rubric {RUBRIC_BY_TRACK.get(str(meta.get('battery', {}).get('track', 'static_object')), 'static_object_v1')}, n_samples={opts.get('n_samples', 2)}, "
+ f"fixed judge: **{opts.get('judge', '?')}** (rubric {battery_rubric(meta)}, n_samples={opts.get('n_samples', 2)}, "
f"acceptance = must_have list) · harness loop judge: {opts.get('loop_judge') or 'settings default'} · "
f"harness rounds ≤ {opts.get('rounds', '?')}, ≤ ${opts.get('max_usd', '?')} · cells: {len(rows)}", "",
"Every arm's final `src/model.py` is re-built, re-rendered and judged by the same evaluator; a failed "
diff --git a/harness/bench/_fixed_eval.py b/harness/bench/_fixed_eval.py
index 375373ce..b982083d 100644
--- a/harness/bench/_fixed_eval.py
+++ b/harness/bench/_fixed_eval.py
@@ -1,8 +1,10 @@
"""The fixed evaluator shared by every arm of ``bench/compare_backends.py``.
-``FixedEvaluator.evaluate(ws, spec)``: BlenderRuntime lint + build → ``measure_glb``
-→ connectivity gate → 8-view ``render_glb`` → ``VlmJudge(static_object_v1,
-<fixed judge>, n_samples)``. The judge's acceptance checklist is the brief's
+``FixedEvaluator.evaluate(ws, spec)``: the cell's runtime lint + build → ``measure_glb``
+→ connectivity gate → 8-view ``render_glb`` (articulated: + joint sweep + pose sheet;
+graphics: the frame sheet) → ``judge_for(spec)`` — a ``VlmJudge`` (or ``LikenessJudge``
+for graphics with reference photos) on ``rubric_for(spec)``, the track's ``TRACK_INFO``
+rubric, the ONE track→rubric mapping of the compare bench. The judge's acceptance checklist is the brief's
``must_have`` list (``acceptance_from_spec``) so harness and one-shot arms are
scored against exactly the same checklist — never the harness's plan.
"""
@@ -14,10 +16,9 @@ from typing import Any
from pydantic import BaseModel, Field
-from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402
from codeverse.config import Settings, get_settings
from codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet
-from codeverse.contracts.common import Language, Track
+from codeverse.contracts.common import TRACK_INFO, Language, Track
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.spec import Spec
@@ -27,6 +28,19 @@ from codeverse.workspace import Workspace
RUBRIC = "static_object_v1"
+def rubric_for(spec: Spec | Track) -> str:
+ """The fixed judge's rubric for a cell (or for a battery's track): the track's ``TRACK_INFO``
+ row — static_object_v1 / articulated_v1 / scene_v1 / shader_v2.
+
+ The one place the compare bench maps a track to a rubric: ``FixedEvaluator`` defaults to
+ it, ``judge_for`` picks each cell's judge with it and ``_compare_report`` names it in the
+ report header. (PR #1 carried a second table, ``RUBRIC_BY_TRACK``, next to main's
+ ``judge_for``; folded here on merge, 2026-08-26.)
+ """
+ track = spec if isinstance(spec, Track) else Track(spec.track)
+ return TRACK_INFO[track].rubric
+
+
def acceptance_from_spec(spec: Spec) -> list[AcceptanceItem]:
"""The fixed judge's checklist: the brief's must_have items, identical for every arm."""
return [AcceptanceItem(id=f"must_{i + 1}", text=m, how="visual", priority="must")
@@ -58,11 +72,12 @@ class FixedEvaluator:
self.settings = settings or get_settings()
self.track = track
self.language = language # the battery's default; cells are built with THEIR spec's language (runtime())
- # articulated batteries are judged on articulated_v1 (rest views + the pose sheet the
- # joint sweep renders); the rubric follows the battery's track unless a caller pins one
- self.rubric = rubric or RUBRIC_BY_TRACK.get(track.value, RUBRIC)
+ # the rubric follows each cell's track (rubric_for, through judge_for); the battery's
+ # track sets the default and a caller may pin one rubric for every cell
+ self._pinned = rubric is not None
+ self.rubric = rubric or rubric_for(track)
self._runtimes: dict[Language, Any] = {}
- self._judge: Any = None
+ self._judges: dict[str, Any] = {}
def runtime(self, language: Language | str) -> Any:
"""The build runtime for THIS cell's language, cached per language.
@@ -83,14 +98,20 @@ class FixedEvaluator:
@property
def judge(self) -> Any:
- if self._judge is None:
+ """The judge on the evaluator's default rubric (the battery's track); cells use ``judge_for``."""
+ return self._vlm_judge(self.rubric)
+
+ def _vlm_judge(self, rubric: str) -> Any:
+ if rubric not in self._judges:
from codeverse.judges.vlm_judge import VlmJudge
- self._judge = VlmJudge(rubric=self.rubric, model_id=self.judge_model, n_samples=self.n_samples)
- return self._judge
+ self._judges[rubric] = VlmJudge(rubric=rubric, model_id=self.judge_model, n_samples=self.n_samples)
+ return self._judges[rubric]
def judge_for(self, spec: Spec) -> Any:
- """The fixed judge for THIS cell's track.
+ """The fixed judge for THIS cell: ``rubric_for(spec)`` (the pinned rubric if a caller set
+ one), through ``LikenessJudge`` for a graphics cell with reference photos, ``VlmJudge``
+ (cached per rubric) otherwise.
The evaluator judged every cell with the object rubric on a GLB. A graphics cell
has frames, not a GLB, so ``evaluate`` returned before judging and every graphics
@@ -100,16 +121,12 @@ class FixedEvaluator:
rubric (``shader_v2``), through ``LikenessJudge`` when the spec carries reference
photos so the arms see the same photos the loop saw.
"""
- if spec.track is Track.GRAPHICS:
- from codeverse.contracts.common import TRACK_INFO
+ rubric = self.rubric if self._pinned else rubric_for(spec)
+ if spec.track is Track.GRAPHICS and spec.references:
from codeverse.judges.reference import LikenessJudge
- from codeverse.judges.vlm_judge import VlmJudge
- rubric = TRACK_INFO[Track.GRAPHICS].rubric
- if spec.references:
- return LikenessJudge(self.judge_model, n_samples=self.n_samples, rubric=rubric)
- return VlmJudge(rubric=rubric, model_id=self.judge_model, n_samples=self.n_samples)
- return self.judge
+ return LikenessJudge(self.judge_model, n_samples=self.n_samples, rubric=rubric)
+ return self._vlm_judge(rubric)
def build(self, ws: Workspace, language: Language | str) -> tuple[BuildResult, GateReport]:
rt = self.runtime(language)
@@ -135,7 +152,7 @@ class FixedEvaluator:
out.gates.append(check_connectivity(glb))
r = self.settings.render
out.renders = render_glb(glb, ws.renders_dir(0), views=list(OBJECT_VIEWS), width=r.width, height=r.height, sheet=True)
- if self.track is Track.ARTICULATED_OBJECT:
+ if spec.track is Track.ARTICULATED_OBJECT:
# the same deterministic articulation evidence the track gives its judge: the
# joint sweep (collisions over every joint's range) and the pose sheet / pose
# tiles, read straight from the built URDF — no plan, so every arm is treated alike
@@ -147,7 +164,7 @@ class FixedEvaluator:
out.renders.views = list(out.renders.views) + pose_views
inp = JudgeInput(spec=spec, renders=out.renders, measurement=out.measurement, gates=out.gates,
acceptance=acceptance_from_spec(spec), round_index=0)
- out.judgment = self.judge.judge(inp)
+ out.judgment = self.judge_for(spec).judge(inp)
except Exception as e: # noqa: BLE001 — recorded per cell, never kills the matrix
out.error = f"{type(e).__name__}: {e}"
return out
@@ -172,4 +189,4 @@ class FixedEvaluator:
return out
-__all__ = ["RUBRIC", "EvalOutcome", "FixedEvaluator", "acceptance_from_spec"]
+__all__ = ["RUBRIC", "EvalOutcome", "FixedEvaluator", "acceptance_from_spec", "rubric_for"]
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index cbc18c61..717cfe1d 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -218,7 +218,9 @@ written) that were accepted because the code works that way and the tests pin it
format; "Error creating WebGL context" (a saturated shared GPU) is an infra marker.
* **D42 `compare_backends` follows the battery's track (2026-08-25).** Context: the
fixed evaluator was static_object-only. Decision: `FixedEvaluator(track=, language=)`
- picks the runtime and rubric from the battery (`RUBRIC_BY_TRACK`); articulated cells add
+ picks the runtime from the battery and the rubric per cell (`_fixed_eval.rubric_for`, the
+ `TRACK_INFO` row, through `judge_for(spec)` — the PR's `RUBRIC_BY_TRACK` table was folded
+ into it on merge); articulated cells add
the joint-sweep gate and the pose sheet read from the built URDF (no plan, every arm
alike); one-shot arms get a language-aware minimal contract (the D18 recipe as rules,
no example) and must answer with both files in the `=== FILE ===` envelope.
diff --git a/harness/tests/compare_bench/test_articulated_compare.py b/harness/tests/compare_bench/test_articulated_compare.py
index 9a1054a4..702bbe2a 100644
--- a/harness/tests/compare_bench/test_articulated_compare.py
+++ b/harness/tests/compare_bench/test_articulated_compare.py
@@ -11,8 +11,7 @@ REPO = Path(__file__).resolve().parents[2]
if str(REPO) not in sys.path:
sys.path.insert(0, str(REPO))
-from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402
-from bench._fixed_eval import FixedEvaluator # noqa: E402
+from bench._fixed_eval import FixedEvaluator, rubric_for # noqa: E402
from bench._oneshot import ( # noqa: E402
MODEL_FILE,
URDF_FILE,
@@ -94,7 +93,7 @@ def test_urdf_repair_prompt_carries_both_previous_files():
def test_fixed_evaluator_follows_the_battery_track():
ev = FixedEvaluator("gemini:x", n_samples=3, track=Track.ARTICULATED_OBJECT, language=Language.URDF_BLENDER)
- assert ev.rubric == "articulated_v1" == RUBRIC_BY_TRACK["articulated_object"]
+ assert ev.rubric == "articulated_v1" == rubric_for(Track.ARTICULATED_OBJECT)
assert ev.language is Language.URDF_BLENDER and ev.runtime(Language.URDF_BLENDER).language is Language.URDF_BLENDER
static = FixedEvaluator("gemini:x")
assert static.rubric == "static_object_v1" and static.language is Language.BLENDER
@@ -102,6 +101,72 @@ def test_fixed_evaluator_follows_the_battery_track():
assert pinned.rubric == "asset_v1"
+def test_rubric_for_is_the_one_track_to_rubric_mapping():
+ """PR #1 shipped `RUBRIC_BY_TRACK` in _compare_report beside main's `judge_for`; one mechanism now:
+ rubric_for(spec | track) = TRACK_INFO's rubric, used by the evaluator default, judge_for and the report."""
+ import bench._compare_report as report
+ from codeverse.contracts.common import TRACK_INFO
+
+ assert not hasattr(report, "RUBRIC_BY_TRACK")
+ lang = {Track.STATIC_OBJECT: Language.BLENDER, Track.ARTICULATED_OBJECT: Language.URDF_BLENDER,
+ Track.SCENE: Language.SCENE_THREEJS, Track.GRAPHICS: Language.GLSL_SHADER}
+ for t in Track:
+ assert rubric_for(t) == TRACK_INFO[t].rubric
+ assert rubric_for(Spec(id="x", track=t, language=lang[t], prompt="x")) == TRACK_INFO[t].rubric
+ assert report.battery_rubric({"battery": {"track": "articulated_object"}}) == "articulated_v1"
+ assert report.battery_rubric({}) == "static_object_v1" == report.battery_rubric({"battery": {"track": "nope"}})
+ # a static-battery evaluator still judges an articulated cell on the articulated rubric ...
+ ev = FixedEvaluator("gemini:x", n_samples=1)
+ assert ev.judge_for(_urdf_spec()).rubric.name == "articulated_v1"
+ assert ev.judge_for(_urdf_spec()) is ev.judge_for(_urdf_spec()), "one judge per rubric, cached"
+ assert ev.judge.rubric.name == "static_object_v1"
+ # ... unless the caller pinned a rubric for every cell
+ assert FixedEvaluator("gemini:x", rubric="asset_v1").judge_for(_urdf_spec()).rubric.name == "asset_v1"
+
+
+def test_articulated_cells_add_the_joint_sweep_and_the_pose_sheet(monkeypatch, tmp_path):
+ """evaluate() on an articulated spec: the sweep gate joins the gates, the pose views join the
+ renders, and the judge is the cell's (articulated_v1) — keyed on the SPEC's track."""
+ from types import SimpleNamespace
+
+ import codeverse.spatial.connectivity as conn
+ import codeverse.spatial.measure as meas
+ import codeverse.spatial.render as rend
+ import codeverse.tracks.articulated_object as art
+ from codeverse.contracts.artifacts import RenderSet, RenderView
+
+ ws = Workspace(tmp_path / "ws").create()
+ glb = tmp_path / "m.glb"
+ glb.write_bytes(b"glTF")
+ build = BuildResult(ok=True, language="urdf_blender", glb_path=str(glb))
+ ev = FixedEvaluator("fake:judge", n_samples=1) # a static-default evaluator, deliberately
+ ev._runtimes[Language.URDF_BLENDER] = SimpleNamespace(lint=lambda ws: GateReport(gate="lint", passed=True),
+ build=lambda ws, timeout_s: build)
+ monkeypatch.setattr(meas, "measure_glb", lambda p: None)
+ monkeypatch.setattr(conn, "check_connectivity", lambda p: GateReport(gate="connectivity", passed=True))
+ monkeypatch.setattr(rend, "render_glb", lambda p, d, **k: RenderSet(views=[RenderView(name="front", path=str(glb))], renderer="fake"))
+ sweep = GateReport(gate="joint_sweep", passed=True)
+ pose = RenderView(name="articulation_sheet", path=str(glb))
+ seen: dict[str, object] = {}
+ monkeypatch.setattr(art, "default_joint_sweep", lambda ws, plan, out: (seen.__setitem__("plan", plan), (sweep, [pose]))[1])
+
+ class _J:
+ rubric = SimpleNamespace(name="articulated_v1")
+
+ def judge(self, inp):
+ seen["inp"] = inp
+ return SimpleNamespace(overall=0.5, passed=False)
+
+ monkeypatch.setattr(FixedEvaluator, "judge_for", lambda self, spec: (seen.__setitem__("spec", spec), _J())[1])
+ out = ev.evaluate(ws, _urdf_spec())
+ assert out.error == "", out.error
+ assert seen["plan"] is None, "no plan: every arm is swept alike"
+ assert seen["spec"].track is Track.ARTICULATED_OBJECT and out.judgment.overall == 0.5
+ assert [g.gate for g in out.gates] == ["lint", "connectivity", "joint_sweep"]
+ assert [v.name for v in out.renders.views] == ["front", "articulation_sheet"]
+ assert [v.name for v in seen["inp"].renders.views] == ["front", "articulation_sheet"]
+
+
def test_main_builds_the_evaluator_from_the_battery(monkeypatch, tmp_path):
from bench import compare_backends as cb