Skip to content

Official Codebase of 3DCodeVerse

Python

cleanup: C3 — Judge replay: kill TRACK_RUBRIC, share plan_digest/resolve_paths/judged_subset

gaoypengf0d6b0ef1764
diff --git a/harness/codeverse/cli/_judge.py b/harness/codeverse/cli/_judge.py
index 6399a345..e8294ea6 100644
--- a/harness/codeverse/cli/_judge.py
+++ b/harness/codeverse/cli/_judge.py
@@ -21,6 +21,7 @@ from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderView
from codeverse.contracts.common import TRACK_INFO
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.run import RoundRecord, RunRecord
+from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_paths
from codeverse.proc import read_json_or_none
from codeverse.workspace import Workspace
@@ -49,7 +50,6 @@ def rubric_for(rec: RunRecord, rnd: RoundRecord, override: str | None) -> str:
def plan_summary_for(ws: Workspace) -> str:
"""Track-agnostic plan digest from plan.json (same fields the tracks summarise)."""
data = read_json_or_none(ws.plan_path)
- plan_digest = C.lazy("codeverse.judges.calibration", "plan_digest")
return plan_digest(data) if data else ""
@@ -89,30 +89,6 @@ def clay_geometry_views(ws: Workspace, index: int) -> RenderSet | None:
return RenderSet(views=views, renderer="stored") if views else None
-def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:
- """Render paths out of a round record, resolved against THIS workspace.
-
- Delegates to ``Workspace.rebase``: this used to rebase only paths for which
- ``is_absolute()`` was False, which made it a no-op against every record the harness
- itself writes (they are all absolute) — so a moved or archived run kept pointing at
- the host it was produced on.
- """
- if rs is None:
- return None
- fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]
- sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet
- out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir
- return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})
-
-
-def judged_subset(rs: RenderSet | None) -> RenderSet | None:
- """The views the in-run judge actually saw: the per-view ``judge`` flags
- stamped at render time; legacy rounds (no flags) keep every stored view."""
- if rs is None or not any(v.judge is not None for v in rs.views):
- return rs
- return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})
-
-
def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:
JudgeInput = C.lazy("codeverse.judges.base", "JudgeInput")
acceptance: list[AcceptanceItem] = list(getattr(rec.plan, "acceptance", []) or [])
diff --git a/harness/codeverse/judges/calibration.py b/harness/codeverse/judges/calibration.py
index f1f03c35..78f3f87b 100644
--- a/harness/codeverse/judges/calibration.py
+++ b/harness/codeverse/judges/calibration.py
@@ -27,17 +27,19 @@ from typing import Any
from pydantic import BaseModel, Field
from codeverse.contracts.artifacts import RenderSet
+from codeverse.contracts.common import TRACK_INFO
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem
from codeverse.contracts.run import RoundRecord
from codeverse.contracts.spec import Spec
from codeverse.judges.base import JudgeInput
+from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_paths
from codeverse.judges.scoring import is_degraded
from codeverse.judges.vlm_judge import VlmJudge
+from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
-TRACK_RUBRIC = {"static_object": "static_object_v1", "articulated_object": "articulated_v1", "scene": "scene_v1"}
GEOMETRY_VIEW_NAMES = ("front_right_34", "back_left_34", "top", "low_front_left")
@@ -124,35 +126,6 @@ class CalibrationTable(BaseModel):
# --------------------------------------------------------------------------- loading runs
-def plan_digest(plan: dict[str, Any]) -> str:
- """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""
- bits: list[str] = []
- title = plan.get("object_name") or plan.get("title") or ""
- if title:
- bits.append(f"{title}: {plan.get('summary', '')}".strip())
- bbox = plan.get("overall_bbox") or {}
- if isinstance(bbox, dict) and bbox.get("extents"):
- e = bbox["extents"]
- bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")
- parts = plan.get("parts") or []
- if parts:
- bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")
- if plan.get("root_link"):
- bits.append(f"Root link {plan['root_link']}.")
- joints = plan.get("joints") or []
- if joints:
- bits.append("Joints: " + "; ".join(
- f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"
- for j in joints) + ".")
- for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):
- items = plan.get(key) or []
- if items:
- bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")
- if plan.get("setting"):
- bits.append(f"Setting: {plan['setting']}.")
- return " ".join(bits)
-
-
def _read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text())
@@ -166,6 +139,7 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Ro
best = record.get("best_round")
acceptance = plan.get("acceptance") or []
digest = plan_digest(plan)
+ ws = Workspace(run_dir)
glb = run_dir / "artifacts" / "object.glb"
cases: list[RoundCase] = []
for path in sorted((run_dir / "rounds").glob("r*.json")):
@@ -174,9 +148,10 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Ro
continue
if rec.renders is None or not rec.renders.views:
continue
- rubric = rec.judgment.rubric if rec.judgment else TRACK_RUBRIC.get(spec.track.value, "static_object_v1")
+ rubric = rec.judgment.rubric if rec.judgment else TRACK_INFO[spec.track].rubric
+ # the same views, at the same paths, that the in-run judge and `3dcv judge` see
inp = JudgeInput(
- spec=spec, renders=rec.renders, measurement=rec.measurement, gates=rec.gates,
+ spec=spec, renders=judged_subset(resolve_paths(ws, rec.renders)), measurement=rec.measurement, gates=rec.gates,
acceptance=[AcceptanceItem.model_validate(a) for a in acceptance], plan_summary=digest,
round_index=rec.index,
)
diff --git a/harness/codeverse/judges/reference.py b/harness/codeverse/judges/reference.py
index 46c85ea3..794e12ee 100644
--- a/harness/codeverse/judges/reference.py
+++ b/harness/codeverse/judges/reference.py
@@ -288,7 +288,7 @@ class ReferenceJudge(VlmJudge):
def _plan_part_names(inp: JudgeInput) -> list[str]:
"""Part names from the plan digest the judge already receives (best effort).
- ``judges.calibration.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;
+ ``judges.replay_input.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;
a multi-line digest lists one ``- Name · role · …`` per line. Both are handled, and
an unrecognised digest simply yields no names (the diff prompt then says so).
"""
diff --git a/harness/codeverse/judges/replay_input.py b/harness/codeverse/judges/replay_input.py
new file mode 100644
index 00000000..cdf4cdd0
--- /dev/null
+++ b/harness/codeverse/judges/replay_input.py
@@ -0,0 +1,82 @@
+"""Judge-replay inputs shared by ``3dcv judge`` (``cli/_judge.py``) and
+``judges/calibration.py``.
+
+Both re-judge a stored round from ``rounds/rNN.json`` + ``plan.json`` and must hand
+the judge what the in-run judge saw. The pure pieces of that reconstruction live
+here so the two replayers cannot drift (no cli / tracks / flywheel imports —
+``judges/`` stays below all of them):
+
+* ``plan_digest`` — was ``judges/calibration.py:plan_digest``;
+ ``cli/_judge.py:plan_summary_for`` lazily imported it from there, a cli→calibration
+ edge for a pure function.
+* ``resolve_paths`` — was ``cli/_judge.py:resolve_paths``; calibration fed the stored
+ render paths straight in, so a moved or archived run replayed against the host it
+ was produced on.
+* ``judged_subset`` — was ``cli/_judge.py:judged_subset``; calibration judged every
+ stored view, so whenever a track stamps ``RenderView.judge`` its numbers were not
+ those of the in-run judge or ``3dcv judge``.
+
+The pieces that need ``RunRecord`` / ``tracks.get_track`` / ``flywheel.record``
+(``previous``, ``extra_context``, clay views) stay in ``cli/_judge.py``: nothing in
+``judges/`` imports those packages and they all import ``judges/``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from codeverse.contracts.artifacts import RenderSet
+from codeverse.workspace import Workspace
+
+
+def plan_digest(plan: dict[str, Any]) -> str:
+ """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""
+ bits: list[str] = []
+ title = plan.get("object_name") or plan.get("title") or ""
+ if title:
+ bits.append(f"{title}: {plan.get('summary', '')}".strip())
+ bbox = plan.get("overall_bbox") or {}
+ if isinstance(bbox, dict) and bbox.get("extents"):
+ e = bbox["extents"]
+ bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")
+ parts = plan.get("parts") or []
+ if parts:
+ bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")
+ if plan.get("root_link"):
+ bits.append(f"Root link {plan['root_link']}.")
+ joints = plan.get("joints") or []
+ if joints:
+ bits.append("Joints: " + "; ".join(
+ f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"
+ for j in joints) + ".")
+ for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):
+ items = plan.get(key) or []
+ if items:
+ bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")
+ if plan.get("setting"):
+ bits.append(f"Setting: {plan['setting']}.")
+ return " ".join(bits)
+
+
+def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:
+ """Render paths out of a round record, resolved against THIS workspace.
+
+ Delegates to ``Workspace.rebase``: this used to rebase only paths for which
+ ``is_absolute()`` was False, which made it a no-op against every record the harness
+ itself writes (they are all absolute) — so a moved or archived run kept pointing at
+ the host it was produced on.
+ """
+ if rs is None:
+ return None
+ fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]
+ sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet
+ out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir
+ return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})
+
+
+def judged_subset(rs: RenderSet | None) -> RenderSet | None:
+ """The views the in-run judge actually saw: the per-view ``judge`` flags
+ stamped at render time; legacy rounds (no flags) keep every stored view."""
+ if rs is None or not any(v.judge is not None for v in rs.views):
+ return rs
+ return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 2c119e12..3c95adc2 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -135,7 +135,9 @@ codeverse/
guard.py routing.py reconstruct.py (old runs) audit.py report.py
judges/ rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,
prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.py
- (compare_many), reference.py, calibration.py, metrics.py
+ (compare_many), reference.py, calibration.py, metrics.py, replay_input.py
+ (plan_digest / resolve_paths / judged_subset — the pure round-replay pieces
+ `3dcv judge` and calibration share)
reference/ reference GROUNDING — give the pipeline a picture of what it is building:
synth.py (prompt → reference image(s)), gate.py (THE plausibility gate that makes a
synthesized reference safe to use), attach.py (Spec attachment + honesty guards),
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 811cd93b..d07c4c3a 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -185,6 +185,25 @@ def test_judge_rubric_map_includes_graphics():
assert rubric_for(_R, rnd, "asset_v1") == "asset_v1"
+def test_calibration_rubric_map_includes_graphics(tmp_path: Path):
+ """Same default as `3dcv judge`: a graphics round with no stored judgment used to be
+ re-judged with static_object_v1 because calibration kept its own three-track
+ TRACK_RUBRIC instead of reading TRACK_INFO."""
+ from codeverse.contracts.artifacts import RenderSet, RenderView
+ from codeverse.contracts.common import Language, Track
+ from codeverse.contracts.run import RoundRecord
+ from codeverse.judges.calibration import load_run_cases
+
+ run = tmp_path / "shader_run"
+ (run / "rounds").mkdir(parents=True)
+ spec = Spec(id="shader_run", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="neon rain on a window")
+ (run / "spec.json").write_text(spec.model_dump_json())
+ renders = RenderSet(views=[RenderView(name="frame_0", path=str(run / "artifacts" / "renders" / "r00" / "frame_0.png"))],
+ renderer="fake")
+ (run / "rounds" / "r00.json").write_text(RoundRecord(index=0, kind="baseline", renders=renders).model_dump_json())
+ assert [c.rubric for c in load_run_cases(run)] == ["shader_v1"]
+
+
# --------------------------------------------------------------------------- make --texture / status extras / render graphics
def test_make_texture_flag_sets_spec_options(tmp_path: Path):
runs = tmp_path / "runs"
diff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.py
index 7efa5ac2..ac3f5964 100644
--- a/harness/tests/judges/test_calibration.py
+++ b/harness/tests/judges/test_calibration.py
@@ -5,7 +5,8 @@ import pytest
from codeverse.contracts.artifacts import GateFinding, GateReport, Severity
from codeverse.contracts.run import RoundRecord
-from codeverse.judges.calibration import calibrate, load_run_cases, pearson, plan_digest, spearman
+from codeverse.judges.calibration import calibrate, load_run_cases, pearson, spearman
+from codeverse.judges.replay_input import plan_digest
from codeverse.judges.rubrics import load_rubric
from tests.judges.conftest import (
ACCEPTANCE,
diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
index bad72204..d93571f7 100644
--- a/harness/tests/orchestrator_tracks/test_refactor_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
@@ -124,7 +124,7 @@ def test_scene_judge_views_prefers_stamped_flags():
def test_cli_judge_reconstructs_the_judged_subset(tmp_path):
- from codeverse.cli._judge import judged_subset, resolve_paths
+ from codeverse.judges.replay_input import judged_subset, resolve_paths
rs = _rs([True, False, None])
sub = judged_subset(rs)