Official Codebase of 3DCodeVerse
Python
bench: one track→rubric mechanism — _fixed_eval.rubric_for feeds FixedEvaluator, judge_for and the report
gaoypeng18c8f4359090
diff --git a/harness/bench/_compare_report.py b/harness/bench/_compare_report.pyindex 0ac4d68c..04729b93 100644--- a/harness/bench/_compare_report.py+++ b/harness/bench/_compare_report.py@@ -17,13 +17,20 @@ from typing import Any, TypeVarfrom pydantic import BaseModel, Field+from bench._fixed_eval import RUBRIC, rubric_forfrom bench._jsonl import read_jsonl+from codeverse.contracts.common import TrackT = TypeVar("T", bound=BaseModel)-#: The fixed judge's rubric per battery track (bench/_fixed_eval.py builds the evaluator from it).-RUBRIC_BY_TRACK: dict[str, str] = {"static_object": "static_object_v1", "articulated_object": "articulated_v1"}+def battery_rubric(meta: dict) -> str:+ """The fixed judge's rubric the report header names: ``rubric_for`` on the battery's track."""+ track = str(meta.get("battery", {}).get("track") or Track.STATIC_OBJECT.value)+ try:+ return rubric_for(Track(track))+ except ValueError: # a meta.json written before batteries carried a track+ return RUBRICclass CellResult(BaseModel):@@ -222,7 +229,7 @@ def compare_markdown(out: Path, rows: list[CellResult], pairs: list[PairRow], meprompts = _prompt_order(rows, meta)cells = {(r.prompt_id, r.arm): r for r in rows}md = [f"# harness vs one-shot — {meta.get('battery', {}).get('name', out.name)}", "",- f"fixed judge: **{opts.get('judge', '?')}** (rubric {RUBRIC_BY_TRACK.get(str(meta.get('battery', {}).get('track', 'static_object')), 'static_object_v1')}, n_samples={opts.get('n_samples', 2)}, "+ f"fixed judge: **{opts.get('judge', '?')}** (rubric {battery_rubric(meta)}, n_samples={opts.get('n_samples', 2)}, "f"acceptance = must_have list) · harness loop judge: {opts.get('loop_judge') or 'settings default'} · "f"harness rounds ≤ {opts.get('rounds', '?')}, ≤ ${opts.get('max_usd', '?')} · cells: {len(rows)}", "","Every arm's final `src/model.py` is re-built, re-rendered and judged by the same evaluator; a failed "diff --git a/harness/bench/_fixed_eval.py b/harness/bench/_fixed_eval.pyindex 375373ce..b982083d 100644--- a/harness/bench/_fixed_eval.py+++ b/harness/bench/_fixed_eval.py@@ -1,8 +1,10 @@"""The fixed evaluator shared by every arm of ``bench/compare_backends.py``.-``FixedEvaluator.evaluate(ws, spec)``: BlenderRuntime lint + build → ``measure_glb``-→ connectivity gate → 8-view ``render_glb`` → ``VlmJudge(static_object_v1,-<fixed judge>, n_samples)``. The judge's acceptance checklist is the brief's+``FixedEvaluator.evaluate(ws, spec)``: the cell's runtime lint + build → ``measure_glb``+→ connectivity gate → 8-view ``render_glb`` (articulated: + joint sweep + pose sheet;+graphics: the frame sheet) → ``judge_for(spec)`` — a ``VlmJudge`` (or ``LikenessJudge``+for graphics with reference photos) on ``rubric_for(spec)``, the track's ``TRACK_INFO``+rubric, the ONE track→rubric mapping of the compare bench. The judge's acceptance checklist is the brief's``must_have`` list (``acceptance_from_spec``) so harness and one-shot arms arescored against exactly the same checklist — never the harness's plan."""@@ -14,10 +16,9 @@ from typing import Anyfrom pydantic import BaseModel, Field-from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402from codeverse.config import Settings, get_settingsfrom codeverse.contracts.artifacts import BuildResult, GateReport, Measurement, RenderSet-from codeverse.contracts.common import Language, Track+from codeverse.contracts.common import TRACK_INFO, Language, Trackfrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.spec import Spec@@ -27,6 +28,19 @@ from codeverse.workspace import WorkspaceRUBRIC = "static_object_v1"+def rubric_for(spec: Spec | Track) -> str:+ """The fixed judge's rubric for a cell (or for a battery's track): the track's ``TRACK_INFO``+ row — static_object_v1 / articulated_v1 / scene_v1 / shader_v2.++ The one place the compare bench maps a track to a rubric: ``FixedEvaluator`` defaults to+ it, ``judge_for`` picks each cell's judge with it and ``_compare_report`` names it in the+ report header. (PR #1 carried a second table, ``RUBRIC_BY_TRACK``, next to main's+ ``judge_for``; folded here on merge, 2026-08-26.)+ """+ track = spec if isinstance(spec, Track) else Track(spec.track)+ return TRACK_INFO[track].rubric++def acceptance_from_spec(spec: Spec) -> list[AcceptanceItem]:"""The fixed judge's checklist: the brief's must_have items, identical for every arm."""return [AcceptanceItem(id=f"must_{i + 1}", text=m, how="visual", priority="must")@@ -58,11 +72,12 @@ class FixedEvaluator:self.settings = settings or get_settings()self.track = trackself.language = language # the battery's default; cells are built with THEIR spec's language (runtime())- # articulated batteries are judged on articulated_v1 (rest views + the pose sheet the- # joint sweep renders); the rubric follows the battery's track unless a caller pins one- self.rubric = rubric or RUBRIC_BY_TRACK.get(track.value, RUBRIC)+ # the rubric follows each cell's track (rubric_for, through judge_for); the battery's+ # track sets the default and a caller may pin one rubric for every cell+ self._pinned = rubric is not None+ self.rubric = rubric or rubric_for(track)self._runtimes: dict[Language, Any] = {}- self._judge: Any = None+ self._judges: dict[str, Any] = {}def runtime(self, language: Language | str) -> Any:"""The build runtime for THIS cell's language, cached per language.@@ -83,14 +98,20 @@ class FixedEvaluator:@propertydef judge(self) -> Any:- if self._judge is None:+ """The judge on the evaluator's default rubric (the battery's track); cells use ``judge_for``."""+ return self._vlm_judge(self.rubric)++ def _vlm_judge(self, rubric: str) -> Any:+ if rubric not in self._judges:from codeverse.judges.vlm_judge import VlmJudge- self._judge = VlmJudge(rubric=self.rubric, model_id=self.judge_model, n_samples=self.n_samples)- return self._judge+ self._judges[rubric] = VlmJudge(rubric=rubric, model_id=self.judge_model, n_samples=self.n_samples)+ return self._judges[rubric]def judge_for(self, spec: Spec) -> Any:- """The fixed judge for THIS cell's track.+ """The fixed judge for THIS cell: ``rubric_for(spec)`` (the pinned rubric if a caller set+ one), through ``LikenessJudge`` for a graphics cell with reference photos, ``VlmJudge``+ (cached per rubric) otherwise.The evaluator judged every cell with the object rubric on a GLB. A graphics cellhas frames, not a GLB, so ``evaluate`` returned before judging and every graphics@@ -100,16 +121,12 @@ class FixedEvaluator:rubric (``shader_v2``), through ``LikenessJudge`` when the spec carries referencephotos so the arms see the same photos the loop saw."""- if spec.track is Track.GRAPHICS:- from codeverse.contracts.common import TRACK_INFO+ rubric = self.rubric if self._pinned else rubric_for(spec)+ if spec.track is Track.GRAPHICS and spec.references:from codeverse.judges.reference import LikenessJudge- from codeverse.judges.vlm_judge import VlmJudge- rubric = TRACK_INFO[Track.GRAPHICS].rubric- if spec.references:- return LikenessJudge(self.judge_model, n_samples=self.n_samples, rubric=rubric)- return VlmJudge(rubric=rubric, model_id=self.judge_model, n_samples=self.n_samples)- return self.judge+ return LikenessJudge(self.judge_model, n_samples=self.n_samples, rubric=rubric)+ return self._vlm_judge(rubric)def build(self, ws: Workspace, language: Language | str) -> tuple[BuildResult, GateReport]:rt = self.runtime(language)@@ -135,7 +152,7 @@ class FixedEvaluator:out.gates.append(check_connectivity(glb))r = self.settings.renderout.renders = render_glb(glb, ws.renders_dir(0), views=list(OBJECT_VIEWS), width=r.width, height=r.height, sheet=True)- if self.track is Track.ARTICULATED_OBJECT:+ if spec.track is Track.ARTICULATED_OBJECT:# the same deterministic articulation evidence the track gives its judge: the# joint sweep (collisions over every joint's range) and the pose sheet / pose# tiles, read straight from the built URDF — no plan, so every arm is treated alike@@ -147,7 +164,7 @@ class FixedEvaluator:out.renders.views = list(out.renders.views) + pose_viewsinp = JudgeInput(spec=spec, renders=out.renders, measurement=out.measurement, gates=out.gates,acceptance=acceptance_from_spec(spec), round_index=0)- out.judgment = self.judge.judge(inp)+ out.judgment = self.judge_for(spec).judge(inp)except Exception as e: # noqa: BLE001 — recorded per cell, never kills the matrixout.error = f"{type(e).__name__}: {e}"return out@@ -172,4 +189,4 @@ class FixedEvaluator:return out-__all__ = ["RUBRIC", "EvalOutcome", "FixedEvaluator", "acceptance_from_spec"]+__all__ = ["RUBRIC", "EvalOutcome", "FixedEvaluator", "acceptance_from_spec", "rubric_for"]diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.mdindex cbc18c61..717cfe1d 100644--- a/harness/docs/DECISIONS.md+++ b/harness/docs/DECISIONS.md@@ -218,7 +218,9 @@ written) that were accepted because the code works that way and the tests pin itformat; "Error creating WebGL context" (a saturated shared GPU) is an infra marker.* **D42 `compare_backends` follows the battery's track (2026-08-25).** Context: thefixed evaluator was static_object-only. Decision: `FixedEvaluator(track=, language=)`- picks the runtime and rubric from the battery (`RUBRIC_BY_TRACK`); articulated cells add+ picks the runtime from the battery and the rubric per cell (`_fixed_eval.rubric_for`, the+ `TRACK_INFO` row, through `judge_for(spec)` — the PR's `RUBRIC_BY_TRACK` table was folded+ into it on merge); articulated cells addthe joint-sweep gate and the pose sheet read from the built URDF (no plan, every armalike); one-shot arms get a language-aware minimal contract (the D18 recipe as rules,no example) and must answer with both files in the `=== FILE ===` envelope.diff --git a/harness/tests/compare_bench/test_articulated_compare.py b/harness/tests/compare_bench/test_articulated_compare.pyindex 9a1054a4..702bbe2a 100644--- a/harness/tests/compare_bench/test_articulated_compare.py+++ b/harness/tests/compare_bench/test_articulated_compare.py@@ -11,8 +11,7 @@ REPO = Path(__file__).resolve().parents[2]if str(REPO) not in sys.path:sys.path.insert(0, str(REPO))-from bench._compare_report import RUBRIC_BY_TRACK # noqa: E402-from bench._fixed_eval import FixedEvaluator # noqa: E402+from bench._fixed_eval import FixedEvaluator, rubric_for # noqa: E402from bench._oneshot import ( # noqa: E402MODEL_FILE,URDF_FILE,@@ -94,7 +93,7 @@ def test_urdf_repair_prompt_carries_both_previous_files():def test_fixed_evaluator_follows_the_battery_track():ev = FixedEvaluator("gemini:x", n_samples=3, track=Track.ARTICULATED_OBJECT, language=Language.URDF_BLENDER)- assert ev.rubric == "articulated_v1" == RUBRIC_BY_TRACK["articulated_object"]+ assert ev.rubric == "articulated_v1" == rubric_for(Track.ARTICULATED_OBJECT)assert ev.language is Language.URDF_BLENDER and ev.runtime(Language.URDF_BLENDER).language is Language.URDF_BLENDERstatic = FixedEvaluator("gemini:x")assert static.rubric == "static_object_v1" and static.language is Language.BLENDER@@ -102,6 +101,72 @@ def test_fixed_evaluator_follows_the_battery_track():assert pinned.rubric == "asset_v1"+def test_rubric_for_is_the_one_track_to_rubric_mapping():+ """PR #1 shipped `RUBRIC_BY_TRACK` in _compare_report beside main's `judge_for`; one mechanism now:+ rubric_for(spec | track) = TRACK_INFO's rubric, used by the evaluator default, judge_for and the report."""+ import bench._compare_report as report+ from codeverse.contracts.common import TRACK_INFO++ assert not hasattr(report, "RUBRIC_BY_TRACK")+ lang = {Track.STATIC_OBJECT: Language.BLENDER, Track.ARTICULATED_OBJECT: Language.URDF_BLENDER,+ Track.SCENE: Language.SCENE_THREEJS, Track.GRAPHICS: Language.GLSL_SHADER}+ for t in Track:+ assert rubric_for(t) == TRACK_INFO[t].rubric+ assert rubric_for(Spec(id="x", track=t, language=lang[t], prompt="x")) == TRACK_INFO[t].rubric+ assert report.battery_rubric({"battery": {"track": "articulated_object"}}) == "articulated_v1"+ assert report.battery_rubric({}) == "static_object_v1" == report.battery_rubric({"battery": {"track": "nope"}})+ # a static-battery evaluator still judges an articulated cell on the articulated rubric ...+ ev = FixedEvaluator("gemini:x", n_samples=1)+ assert ev.judge_for(_urdf_spec()).rubric.name == "articulated_v1"+ assert ev.judge_for(_urdf_spec()) is ev.judge_for(_urdf_spec()), "one judge per rubric, cached"+ assert ev.judge.rubric.name == "static_object_v1"+ # ... unless the caller pinned a rubric for every cell+ assert FixedEvaluator("gemini:x", rubric="asset_v1").judge_for(_urdf_spec()).rubric.name == "asset_v1"+++def test_articulated_cells_add_the_joint_sweep_and_the_pose_sheet(monkeypatch, tmp_path):+ """evaluate() on an articulated spec: the sweep gate joins the gates, the pose views join the+ renders, and the judge is the cell's (articulated_v1) — keyed on the SPEC's track."""+ from types import SimpleNamespace++ import codeverse.spatial.connectivity as conn+ import codeverse.spatial.measure as meas+ import codeverse.spatial.render as rend+ import codeverse.tracks.articulated_object as art+ from codeverse.contracts.artifacts import RenderSet, RenderView++ ws = Workspace(tmp_path / "ws").create()+ glb = tmp_path / "m.glb"+ glb.write_bytes(b"glTF")+ build = BuildResult(ok=True, language="urdf_blender", glb_path=str(glb))+ ev = FixedEvaluator("fake:judge", n_samples=1) # a static-default evaluator, deliberately+ ev._runtimes[Language.URDF_BLENDER] = SimpleNamespace(lint=lambda ws: GateReport(gate="lint", passed=True),+ build=lambda ws, timeout_s: build)+ monkeypatch.setattr(meas, "measure_glb", lambda p: None)+ monkeypatch.setattr(conn, "check_connectivity", lambda p: GateReport(gate="connectivity", passed=True))+ monkeypatch.setattr(rend, "render_glb", lambda p, d, **k: RenderSet(views=[RenderView(name="front", path=str(glb))], renderer="fake"))+ sweep = GateReport(gate="joint_sweep", passed=True)+ pose = RenderView(name="articulation_sheet", path=str(glb))+ seen: dict[str, object] = {}+ monkeypatch.setattr(art, "default_joint_sweep", lambda ws, plan, out: (seen.__setitem__("plan", plan), (sweep, [pose]))[1])++ class _J:+ rubric = SimpleNamespace(name="articulated_v1")++ def judge(self, inp):+ seen["inp"] = inp+ return SimpleNamespace(overall=0.5, passed=False)++ monkeypatch.setattr(FixedEvaluator, "judge_for", lambda self, spec: (seen.__setitem__("spec", spec), _J())[1])+ out = ev.evaluate(ws, _urdf_spec())+ assert out.error == "", out.error+ assert seen["plan"] is None, "no plan: every arm is swept alike"+ assert seen["spec"].track is Track.ARTICULATED_OBJECT and out.judgment.overall == 0.5+ assert [g.gate for g in out.gates] == ["lint", "connectivity", "joint_sweep"]+ assert [v.name for v in out.renders.views] == ["front", "articulation_sheet"]+ assert [v.name for v in seen["inp"].renders.views] == ["front", "articulation_sheet"]++def test_main_builds_the_evaluator_from_the_battery(monkeypatch, tmp_path):from bench import compare_backends as cb