Skip to content

Official Codebase of 3DCodeVerse

Python

graphics: shader_v2 is the track's judge

gaoypengbf05b2db3ff3
diff --git a/harness/codeverse/contracts/common.py b/harness/codeverse/contracts/common.py
index c3348b08..2074703c 100644
--- a/harness/codeverse/contracts/common.py
+++ b/harness/codeverse/contracts/common.py
@@ -58,7 +58,7 @@ TRACK_INFO: dict[Track, TrackInfo] = {
Track.STATIC_OBJECT: TrackInfo(rubric="static_object_v1", label="3D Objects"),
Track.ARTICULATED_OBJECT: TrackInfo(rubric="articulated_v1", label="Articulated Objects"),
Track.SCENE: TrackInfo(rubric="scene_v1", label="3D Scenes"),
- Track.GRAPHICS: TrackInfo(rubric="shader_v1", label="Procedural Graphics"),
+ Track.GRAPHICS: TrackInfo(rubric="shader_v2", label="Procedural Graphics"),
}
#: entry file inside the workspace per language — the file a build starts from.
diff --git a/harness/codeverse/judges/reference.py b/harness/codeverse/judges/reference.py
index c10ffba4..3aae850a 100644
--- a/harness/codeverse/judges/reference.py
+++ b/harness/codeverse/judges/reference.py
@@ -341,7 +341,7 @@ class LikenessJudge(VlmJudge):
name = "likeness"
def __init__(self, model_id: str | None = None, n_samples: int = 1, temperature: float = 0.2, *,
- rubric: str | Rubric = "shader_v1", max_refs: int = 3, **kwargs: Any):
+ rubric: str | Rubric = "shader_v2", max_refs: int = 3, **kwargs: Any):
super().__init__(rubric, model_id, n_samples, temperature, **kwargs)
self.max_refs = max(1, int(max_refs))
diff --git a/harness/codeverse/tracks/common.py b/harness/codeverse/tracks/common.py
index 8cb17852..04dae8bc 100644
--- a/harness/codeverse/tracks/common.py
+++ b/harness/codeverse/tracks/common.py
@@ -65,7 +65,7 @@ class Services:
"""Image-conditioned judge (spec has reference images): renders + references + silhouette IoU."""
return _import("codeverse.judges.reference", "ReferenceJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
- def likeness_judge(self, model_id: str, n_samples: int = 1, rubric: str = "shader_v1") -> Any:
+ def likeness_judge(self, model_id: str, n_samples: int = 1, rubric: str = "shader_v2") -> Any:
"""Reference photos beside the frames, no silhouette (graphics / scene with images)."""
return _import("codeverse.judges.reference", "LikenessJudge")(model_id=model_id, n_samples=n_samples, rubric=rubric)
diff --git a/harness/codeverse/tracks/graphics.py b/harness/codeverse/tracks/graphics.py
index f081a2a7..9329576a 100644
--- a/harness/codeverse/tracks/graphics.py
+++ b/harness/codeverse/tracks/graphics.py
@@ -5,7 +5,7 @@ moderngl program). Reuses ``BaseTrack`` (lifecycle) and ``run_round`` (steps)
unchanged; the track-specific pieces are the planner hooks (template / example /
acceptance in ``graphics_steps``), the prompt context (no 3D frame), the
``gl_frames`` gate (frame statistics from the build) and the render step (the
-sampled frames + contact sheet as the RenderSet the ``shader_v1`` judge sees).
+sampled frames + contact sheet as the RenderSet the ``shader_v2`` judge sees).
Refinement is always one whole-program task.
"""
diff --git a/harness/tests/core/test_contract_additions.py b/harness/tests/core/test_contract_additions.py
index 8ee930f3..467e2f69 100644
--- a/harness/tests/core/test_contract_additions.py
+++ b/harness/tests/core/test_contract_additions.py
@@ -27,7 +27,7 @@ def test_track_info_covers_every_track_and_is_frozen():
assert TRACK_INFO[Track.STATIC_OBJECT].rubric == "static_object_v1"
assert TRACK_INFO[Track.ARTICULATED_OBJECT].rubric == "articulated_v1"
assert TRACK_INFO[Track.SCENE].rubric == "scene_v1"
- assert TRACK_INFO[Track.GRAPHICS].rubric == "shader_v1"
+ assert TRACK_INFO[Track.GRAPHICS].rubric == "shader_v2"
with pytest.raises(ValidationError):
TRACK_INFO[Track.SCENE].rubric = "other" # frozen
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index d07c4c3a..aa4f5037 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -174,14 +174,14 @@ def test_judge_rubric_map_includes_graphics():
from codeverse.contracts.common import TRACK_INFO, Track
from codeverse.contracts.run import RoundRecord
- assert TRACK_INFO[Track.GRAPHICS].rubric == "shader_v1"
+ assert TRACK_INFO[Track.GRAPHICS].rubric == "shader_v2"
class _R: # minimal record stub
class spec:
track = Track.GRAPHICS
rnd = RoundRecord(index=0, kind="baseline")
- assert rubric_for(_R, rnd, None) == "shader_v1"
+ assert rubric_for(_R, rnd, None) == "shader_v2"
assert rubric_for(_R, rnd, "asset_v1") == "asset_v1"
@@ -201,7 +201,7 @@ def test_calibration_rubric_map_includes_graphics(tmp_path: Path):
renders = RenderSet(views=[RenderView(name="frame_0", path=str(run / "artifacts" / "renders" / "r00" / "frame_0.png"))],
renderer="fake")
(run / "rounds" / "r00.json").write_text(RoundRecord(index=0, kind="baseline", renders=renders).model_dump_json())
- assert [c.rubric for c in load_run_cases(run)] == ["shader_v1"]
+ assert [c.rubric for c in load_run_cases(run)] == ["shader_v2"]
# --------------------------------------------------------------------------- make --texture / status extras / render graphics
diff --git a/harness/tests/graphics/test_track.py b/harness/tests/graphics/test_track.py
index 56fa12b3..42adf8ba 100644
--- a/harness/tests/graphics/test_track.py
+++ b/harness/tests/graphics/test_track.py
@@ -119,7 +119,7 @@ def test_graphics_track_end_to_end(tmp_path, settings):
rec = track.run(spec, ws)
assert rec.status is RunStatus.PASSED and [r.kind for r in rec.rounds] == ["baseline", "refine", "refine"]
assert rec.baseline_score == pytest.approx(0.55) and rec.final_score == pytest.approx(0.9) and rec.best_round == 2
- assert rec.extra["rubric"] == "shader_v1"
+ assert rec.extra["rubric"] == "shader_v2"
plan = GraphicsPlan.model_validate(json.loads(ws.plan_path.read_text()))
assert any(a.text.startswith("Includes: bokeh") for a in plan.acceptance)
# renders are the frames (views named t=<s>s) + a sheet, per round
@@ -211,8 +211,8 @@ def test_planner_reask_and_acceptance(tmp_ws):
def test_templates_render_and_rubric_loads(tmp_path, settings):
- rubric = load_rubric("shader_v1")
- assert rubric.track_hint == "graphics" and {c.id for c in rubric.criteria} >= {"brief_fidelity", "motion_quality", "technical_cleanliness"}
+ rubric = load_rubric("shader_v2")
+ assert rubric.track_hint == "graphics" and {c.id for c in rubric.criteria} >= {"likeness", "motion_quality", "technical_cleanliness"}
assert {c.id for c in rubric.caps} >= {"nan_pixels", "static_frames", "black_or_blown", "build_error"}
spec = make_spec(language=Language.OPENGL_PYTHON, generator="single-shot:fake:fake-model")
ws = Workspace(tmp_path / "runs" / "tpl").create()