Official Codebase of 3DCodeVerse
Python
cleanup: C3 — Judge replay: kill TRACK_RUBRIC, share plan_digest/resolve_paths/judged_subset
gaoypengf0d6b0ef1764
diff --git a/harness/codeverse/cli/_judge.py b/harness/codeverse/cli/_judge.pyindex 6399a345..e8294ea6 100644--- a/harness/codeverse/cli/_judge.py+++ b/harness/codeverse/cli/_judge.py@@ -21,6 +21,7 @@ from codeverse.contracts.artifacts import BuildResult, RenderSet, RenderViewfrom codeverse.contracts.common import TRACK_INFOfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.run import RoundRecord, RunRecord+from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_pathsfrom codeverse.proc import read_json_or_nonefrom codeverse.workspace import Workspace@@ -49,7 +50,6 @@ def rubric_for(rec: RunRecord, rnd: RoundRecord, override: str | None) -> str:def plan_summary_for(ws: Workspace) -> str:"""Track-agnostic plan digest from plan.json (same fields the tracks summarise)."""data = read_json_or_none(ws.plan_path)- plan_digest = C.lazy("codeverse.judges.calibration", "plan_digest")return plan_digest(data) if data else ""@@ -89,30 +89,6 @@ def clay_geometry_views(ws: Workspace, index: int) -> RenderSet | None:return RenderSet(views=views, renderer="stored") if views else None-def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:- """Render paths out of a round record, resolved against THIS workspace.-- Delegates to ``Workspace.rebase``: this used to rebase only paths for which- ``is_absolute()`` was False, which made it a no-op against every record the harness- itself writes (they are all absolute) — so a moved or archived run kept pointing at- the host it was produced on.- """- if rs is None:- return None- fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]- sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet- out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir- return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})---def judged_subset(rs: RenderSet | None) -> RenderSet | None:- """The views the in-run judge actually saw: the per-view ``judge`` flags- stamped at render time; legacy rounds (no flags) keep every stored view."""- if rs is None or not any(v.judge is not None for v in rs.views):- return rs- return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})--def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:JudgeInput = C.lazy("codeverse.judges.base", "JudgeInput")acceptance: list[AcceptanceItem] = list(getattr(rec.plan, "acceptance", []) or [])diff --git a/harness/codeverse/judges/calibration.py b/harness/codeverse/judges/calibration.pyindex f1f03c35..78f3f87b 100644--- a/harness/codeverse/judges/calibration.py+++ b/harness/codeverse/judges/calibration.py@@ -27,17 +27,19 @@ from typing import Anyfrom pydantic import BaseModel, Fieldfrom codeverse.contracts.artifacts import RenderSet+from codeverse.contracts.common import TRACK_INFOfrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItemfrom codeverse.contracts.run import RoundRecordfrom codeverse.contracts.spec import Specfrom codeverse.judges.base import JudgeInput+from codeverse.judges.replay_input import judged_subset, plan_digest, resolve_pathsfrom codeverse.judges.scoring import is_degradedfrom codeverse.judges.vlm_judge import VlmJudge+from codeverse.workspace import Workspacelog = logging.getLogger(__name__)-TRACK_RUBRIC = {"static_object": "static_object_v1", "articulated_object": "articulated_v1", "scene": "scene_v1"}GEOMETRY_VIEW_NAMES = ("front_right_34", "back_left_34", "top", "low_front_left")@@ -124,35 +126,6 @@ class CalibrationTable(BaseModel):# --------------------------------------------------------------------------- loading runs-def plan_digest(plan: dict[str, Any]) -> str:- """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""- bits: list[str] = []- title = plan.get("object_name") or plan.get("title") or ""- if title:- bits.append(f"{title}: {plan.get('summary', '')}".strip())- bbox = plan.get("overall_bbox") or {}- if isinstance(bbox, dict) and bbox.get("extents"):- e = bbox["extents"]- bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")- parts = plan.get("parts") or []- if parts:- bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")- if plan.get("root_link"):- bits.append(f"Root link {plan['root_link']}.")- joints = plan.get("joints") or []- if joints:- bits.append("Joints: " + "; ".join(- f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"- for j in joints) + ".")- for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):- items = plan.get(key) or []- if items:- bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")- if plan.get("setting"):- bits.append(f"Setting: {plan['setting']}.")- return " ".join(bits)--def _read_json(path: Path) -> dict[str, Any]:return json.loads(path.read_text())@@ -166,6 +139,7 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Robest = record.get("best_round")acceptance = plan.get("acceptance") or []digest = plan_digest(plan)+ ws = Workspace(run_dir)glb = run_dir / "artifacts" / "object.glb"cases: list[RoundCase] = []for path in sorted((run_dir / "rounds").glob("r*.json")):@@ -174,9 +148,10 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Rocontinueif rec.renders is None or not rec.renders.views:continue- rubric = rec.judgment.rubric if rec.judgment else TRACK_RUBRIC.get(spec.track.value, "static_object_v1")+ rubric = rec.judgment.rubric if rec.judgment else TRACK_INFO[spec.track].rubric+ # the same views, at the same paths, that the in-run judge and `3dcv judge` seeinp = JudgeInput(- spec=spec, renders=rec.renders, measurement=rec.measurement, gates=rec.gates,+ spec=spec, renders=judged_subset(resolve_paths(ws, rec.renders)), measurement=rec.measurement, gates=rec.gates,acceptance=[AcceptanceItem.model_validate(a) for a in acceptance], plan_summary=digest,round_index=rec.index,)diff --git a/harness/codeverse/judges/reference.py b/harness/codeverse/judges/reference.pyindex 46c85ea3..794e12ee 100644--- a/harness/codeverse/judges/reference.py+++ b/harness/codeverse/judges/reference.py@@ -288,7 +288,7 @@ class ReferenceJudge(VlmJudge):def _plan_part_names(inp: JudgeInput) -> list[str]:"""Part names from the plan digest the judge already receives (best effort).- ``judges.calibration.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;+ ``judges.replay_input.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;a multi-line digest lists one ``- Name · role · …`` per line. Both are handled, andan unrecognised digest simply yields no names (the diff prompt then says so)."""diff --git a/harness/codeverse/judges/replay_input.py b/harness/codeverse/judges/replay_input.pynew file mode 100644index 00000000..cdf4cdd0--- /dev/null+++ b/harness/codeverse/judges/replay_input.py@@ -0,0 +1,82 @@+"""Judge-replay inputs shared by ``3dcv judge`` (``cli/_judge.py``) and+``judges/calibration.py``.++Both re-judge a stored round from ``rounds/rNN.json`` + ``plan.json`` and must hand+the judge what the in-run judge saw. The pure pieces of that reconstruction live+here so the two replayers cannot drift (no cli / tracks / flywheel imports —+``judges/`` stays below all of them):++* ``plan_digest`` — was ``judges/calibration.py:plan_digest``;+ ``cli/_judge.py:plan_summary_for`` lazily imported it from there, a cli→calibration+ edge for a pure function.+* ``resolve_paths`` — was ``cli/_judge.py:resolve_paths``; calibration fed the stored+ render paths straight in, so a moved or archived run replayed against the host it+ was produced on.+* ``judged_subset`` — was ``cli/_judge.py:judged_subset``; calibration judged every+ stored view, so whenever a track stamps ``RenderView.judge`` its numbers were not+ those of the in-run judge or ``3dcv judge``.++The pieces that need ``RunRecord`` / ``tracks.get_track`` / ``flywheel.record``+(``previous``, ``extra_context``, clay views) stay in ``cli/_judge.py``: nothing in+``judges/`` imports those packages and they all import ``judges/``.+"""++from __future__ import annotations++from typing import Any++from codeverse.contracts.artifacts import RenderSet+from codeverse.workspace import Workspace+++def plan_digest(plan: dict[str, Any]) -> str:+ """A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""+ bits: list[str] = []+ title = plan.get("object_name") or plan.get("title") or ""+ if title:+ bits.append(f"{title}: {plan.get('summary', '')}".strip())+ bbox = plan.get("overall_bbox") or {}+ if isinstance(bbox, dict) and bbox.get("extents"):+ e = bbox["extents"]+ bits.append(f"Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m.")+ parts = plan.get("parts") or []+ if parts:+ bits.append("Parts: " + ", ".join(f"{p['name']}×{p['instances']}" if p.get("instances", 1) > 1 else p["name"] for p in parts) + ".")+ if plan.get("root_link"):+ bits.append(f"Root link {plan['root_link']}.")+ joints = plan.get("joints") or []+ if joints:+ bits.append("Joints: " + "; ".join(+ f"{j['name']} ({j.get('type', '?')} {j.get('parent', '?')}→{j.get('child', '?')}, [{j.get('lower', 0):.2f},{j.get('upper', 0):.2f}])"+ for j in joints) + ".")+ for key, label in (("zones", "Zones"), ("assets", "Assets"), ("cameras", "Cameras")):+ items = plan.get(key) or []+ if items:+ bits.append(f"{label}: " + ", ".join(str(i.get("name", "?")) for i in items) + ".")+ if plan.get("setting"):+ bits.append(f"Setting: {plan['setting']}.")+ return " ".join(bits)+++def resolve_paths(ws: Workspace, rs: RenderSet | None) -> RenderSet | None:+ """Render paths out of a round record, resolved against THIS workspace.++ Delegates to ``Workspace.rebase``: this used to rebase only paths for which+ ``is_absolute()`` was False, which made it a no-op against every record the harness+ itself writes (they are all absolute) — so a moved or archived run kept pointing at+ the host it was produced on.+ """+ if rs is None:+ return None+ fixed = [v.model_copy(update={"path": str(ws.rebase(v.path))}) for v in rs.views]+ sheet = str(ws.rebase(rs.contact_sheet)) if rs.contact_sheet else rs.contact_sheet+ out_dir = str(ws.rebase(rs.out_dir)) if rs.out_dir else rs.out_dir+ return rs.model_copy(update={"views": fixed, "contact_sheet": sheet, "out_dir": out_dir})+++def judged_subset(rs: RenderSet | None) -> RenderSet | None:+ """The views the in-run judge actually saw: the per-view ``judge`` flags+ stamped at render time; legacy rounds (no flags) keep every stored view."""+ if rs is None or not any(v.judge is not None for v in rs.views):+ return rs+ return rs.model_copy(update={"views": [v for v in rs.views if v.judge]})diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex 2c119e12..3c95adc2 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -135,7 +135,9 @@ codeverse/guard.py routing.py reconstruct.py (old runs) audit.py report.pyjudges/ rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.py- (compare_many), reference.py, calibration.py, metrics.py+ (compare_many), reference.py, calibration.py, metrics.py, replay_input.py+ (plan_digest / resolve_paths / judged_subset — the pure round-replay pieces+ `3dcv judge` and calibration share)reference/ reference GROUNDING — give the pipeline a picture of what it is building:synth.py (prompt → reference image(s)), gate.py (THE plausibility gate that makes asynthesized reference safe to use), attach.py (Spec attachment + honesty guards),diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.pyindex 811cd93b..d07c4c3a 100644--- a/harness/tests/flywheel_cli/test_cli.py+++ b/harness/tests/flywheel_cli/test_cli.py@@ -185,6 +185,25 @@ def test_judge_rubric_map_includes_graphics():assert rubric_for(_R, rnd, "asset_v1") == "asset_v1"+def test_calibration_rubric_map_includes_graphics(tmp_path: Path):+ """Same default as `3dcv judge`: a graphics round with no stored judgment used to be+ re-judged with static_object_v1 because calibration kept its own three-track+ TRACK_RUBRIC instead of reading TRACK_INFO."""+ from codeverse.contracts.artifacts import RenderSet, RenderView+ from codeverse.contracts.common import Language, Track+ from codeverse.contracts.run import RoundRecord+ from codeverse.judges.calibration import load_run_cases++ run = tmp_path / "shader_run"+ (run / "rounds").mkdir(parents=True)+ spec = Spec(id="shader_run", track=Track.GRAPHICS, language=Language.GLSL_SHADER, prompt="neon rain on a window")+ (run / "spec.json").write_text(spec.model_dump_json())+ renders = RenderSet(views=[RenderView(name="frame_0", path=str(run / "artifacts" / "renders" / "r00" / "frame_0.png"))],+ renderer="fake")+ (run / "rounds" / "r00.json").write_text(RoundRecord(index=0, kind="baseline", renders=renders).model_dump_json())+ assert [c.rubric for c in load_run_cases(run)] == ["shader_v1"]++# --------------------------------------------------------------------------- make --texture / status extras / render graphicsdef test_make_texture_flag_sets_spec_options(tmp_path: Path):runs = tmp_path / "runs"diff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.pyindex 7efa5ac2..ac3f5964 100644--- a/harness/tests/judges/test_calibration.py+++ b/harness/tests/judges/test_calibration.py@@ -5,7 +5,8 @@ import pytestfrom codeverse.contracts.artifacts import GateFinding, GateReport, Severityfrom codeverse.contracts.run import RoundRecord-from codeverse.judges.calibration import calibrate, load_run_cases, pearson, plan_digest, spearman+from codeverse.judges.calibration import calibrate, load_run_cases, pearson, spearman+from codeverse.judges.replay_input import plan_digestfrom codeverse.judges.rubrics import load_rubricfrom tests.judges.conftest import (ACCEPTANCE,diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.pyindex bad72204..d93571f7 100644--- a/harness/tests/orchestrator_tracks/test_refactor_batch2.py+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py@@ -124,7 +124,7 @@ def test_scene_judge_views_prefers_stamped_flags():def test_cli_judge_reconstructs_the_judged_subset(tmp_path):- from codeverse.cli._judge import judged_subset, resolve_paths+ from codeverse.judges.replay_input import judged_subset, resolve_pathsrs = _rs([True, False, None])sub = judged_subset(rs)