Skip to content

Official Codebase of 3DCodeVerse

Python

replay: a re-judge reads the round the run handed over, on the run's rubric, with the loop's previous verdict, and what it pays lands in record.total_usage

gaoypengc0edd0260360
diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.py
index 519c2041..743347f1 100644
--- a/harness/codeverse3d/addons/select.py
+++ b/harness/codeverse3d/addons/select.py
@@ -17,7 +17,8 @@ other reader asks (gallery, dataset, cost report, calibration, the CLI, eval/ben
was packaged, else :func:`pick`'s;
* :func:`round_file` a round's OWN build output (its GLB, GIF), never the last build's —
and :func:`round_complexity_block` its complexity vector;
-* :func:`package` ``deliverable/`` for one round, the texture pass on it when asked, and
+* :func:`package` ``deliverable/`` for one round, the texture pass on it when asked
+ (:func:`texture_round`, also ``3dcode texture pass``), and
``selection.json`` (round, method, scores) beside ``record.json``.
Nothing here runs during a run: ``3dcode make`` calls :func:`pick` + :func:`package` after the
@@ -34,14 +35,20 @@ from typing import Any, Literal
from pydantic import BaseModel, Field
-from codeverse3d.contracts.common import TRACK_INFO, Usage
+from codeverse3d.contracts.common import Usage
from codeverse3d.contracts.run import RoundRecord, RunRecord
from codeverse3d.cost.context import call_context
from codeverse3d.cost.types import Role, Stage
from codeverse3d.judges.base import judged_subset, resolve_paths
from codeverse3d.proc import EventLog, read_json_or_none, sha256_file
from codeverse3d.record.deliverable import build_deliverable, round_outputs, texture_report_for
-from codeverse3d.record.record import effective_judgment, load_record, package_run, round_complexity
+from codeverse3d.record.record import (
+ effective_judgment,
+ load_record,
+ repackage,
+ round_complexity,
+ rubric_of,
+)
from codeverse3d.workspace import Workspace
log = logging.getLogger(__name__)
@@ -196,7 +203,7 @@ def pairwise_verdict(ws: Workspace, rec: RunRecord, a: int, b: int, *, model: st
judge = PairwiseJudge(model)
with call_context(stage=Stage.PAIRWISE, role=Role.JUDGE, label="pick"):
res = judge.compare(rec.spec, renders[0], renders[1],
- rubric=str(rec.extra.get("rubric") or TRACK_INFO[rec.spec.track].rubric))
+ rubric=rubric_of(rec))
except Exception as e: # noqa: BLE001 — an outage must not move the pick
log.warning("pairwise pick failed: %s", e)
note.error = f"{type(e).__name__}: {e}"
@@ -243,21 +250,22 @@ def package(run_dir: Path | str, round_index: int, *, texture: bool = False, met
"""Write ``deliverable/`` for round ``round_index`` (from its commit and ``artifacts/rNN/``,
no rebuild) and ``selection.json``; returns the deliverable directory.
- ``texture=True`` runs the texture pass on that round's own GLB first, unless a pass
- already started from those exact bytes (a paid pass is never bought twice); a pack
- ships into the deliverable only when its judge gate shipped it, and the record is
- re-packaged so its total takes the pass in (the run's money is its ledger's,
- ``record.package_run``). Otherwise the record is not rewritten."""
+ ``texture=True`` runs the texture pass on that round's own GLB first (:func:`texture_round`);
+ a pack ships into the deliverable only when its judge gate shipped it. The record is
+ re-packaged (``record.repackage``), so its total takes in what the hand-over paid — the
+ texture pass, a pairwise verdict — the run's money is its ledger's."""
ws = Workspace(run_dir)
rec = load_record(ws)
- rnd = next((r for r in rec.rounds if r.index == round_index), None)
- if rnd is None:
+ if not any(r.index == round_index for r in rec.rounds):
raise ValueError(f"no round {round_index} in {ws.root} (rounds: {[r.index for r in rec.rounds]})")
if texture:
- _texture(ws, rec, round_index, image_model=image_model)
- rec = load_record(ws) # the pass wrote extra["texturing"]
- package_run(ws, rec)
- ws.write_json(ws.record_path, rec)
+ texture_glb(ws, rec, round_index) # nothing to texture is the caller's error; a failed pass is not
+ try:
+ texture_round(ws, rec, round_index, image_model=image_model)
+ except Exception as e: # noqa: BLE001 — a derived asset pack: the hand-over goes out without it
+ log.warning("texture pass failed on round %d: %s", round_index, e)
+ EventLog(ws.events_path).emit("texture.failed", round=round_index, error=f"{type(e).__name__}: {e}")
+ rec = repackage(ws) or load_record(ws) # a pass wrote extra["texturing"]
manifest = build_deliverable(ws, rec, round_index)
sel = Selection(round=round_index, method=method, scores={r.index: r.score for r in round_rows(ws.root, record=rec)},
textured=any(f.path == "deliverable/object_textured.glb" for f in manifest.files))
@@ -267,24 +275,34 @@ def package(run_dir: Path | str, round_index: int, *, texture: bool = False, met
return ws.deliverable
-def _texture(ws: Workspace, rec: RunRecord, index: int, *, image_model: Any | None) -> None:
- """The texture pass on round ``index``'s GLB (it records itself on ``record.json``)."""
- from codeverse3d.texturing.run import texture_pass, texture_supported
+def texture_glb(ws: Workspace, rec: RunRecord, index: int) -> tuple[RoundRecord, Path]:
+ """Round ``index`` and its own GLB — what a texture pass on it starts from. ValueError when
+ the track has no GLB or the round kept none."""
+ from codeverse3d.texturing.run import texture_supported
if not texture_supported(rec.spec.track):
raise ValueError(f"the texture pass is for object tracks; {rec.spec.track.value} runs have no GLB to texture")
- rnd = next(r for r in rec.rounds if r.index == index)
+ rnd = next((r for r in rec.rounds if r.index == index), None)
glb = round_file(ws, rnd)
- if glb is None:
+ if rnd is None or glb is None:
raise ValueError(f"round {index} kept no object.glb to texture (artifacts/r{index:02d}/)")
- if texture_report_for(ws, glb) is not None:
- EventLog(ws.events_path).emit("texture.skipped", reason="already_textured_this_artifact", round=index)
- return
- sheet = ws.rebase(rnd.renders.contact_sheet) if rnd.renders is not None and rnd.renders.contact_sheet else None
+ return rnd, glb
+
+
+def texture_round(ws: Workspace, rec: RunRecord, index: int, *, image_model: Any | None = None,
+ model_id: str | None = None, judge: bool = True, judge_model_id: str | None = None,
+ size: int = 1024, force: bool = False) -> Any | None:
+ """The texture pass on round ``index``'s own GLB (it records itself on ``record.json``), or None
+ when a pass already started from those exact bytes — a paid pass is never bought twice unless
+ ``force``. ValueError from :func:`texture_glb`; the pass's own failures propagate."""
+ from codeverse3d.texturing.run import texture_pass
+
+ rnd, glb = texture_glb(ws, rec, index)
events = EventLog(ws.events_path)
- try:
- texture_pass(ws, rec.spec, rec.plan, model_id=rec.spec.backends.planner, image_model=image_model, judge=True,
- judge_model_id=rec.spec.backends.judge, glb_in=glb, sheet=sheet, events=events)
- except Exception as e: # noqa: BLE001 — a derived asset pack: the hand-over goes out without it
- log.warning("texture pass failed on round %d: %s", index, e)
- events.emit("texture.failed", round=index, error=f"{type(e).__name__}: {e}")
+ if not force and texture_report_for(ws, glb) is not None:
+ events.emit("texture.skipped", reason="already_textured_this_artifact", round=index)
+ return None
+ sheet = ws.rebase(rnd.renders.contact_sheet) if rnd.renders is not None and rnd.renders.contact_sheet else None
+ return texture_pass(ws, rec.spec, rec.plan, model_id=model_id or rec.spec.backends.planner, image_model=image_model,
+ judge=judge, judge_model_id=judge_model_id or rec.spec.backends.judge, size=size, glb_in=glb,
+ sheet=sheet, events=events)
diff --git a/harness/codeverse3d/cli/_judge.py b/harness/codeverse3d/cli/_judge.py
index 72f0b8d2..24d7b8a0 100644
--- a/harness/codeverse3d/cli/_judge.py
+++ b/harness/codeverse3d/cli/_judge.py
@@ -4,8 +4,8 @@ in-run judge saw.
The payload is ``judges.base.round_input`` — the builder the in-run judge uses — fed from
disk: the round's renders / measurement / gates from ``rounds/rNN.json`` (fallback: the
round inside record.json), the typed plan from record.json (else ``plan.json``), ``previous``
-the preceding round's (non-degraded) verdict, the track's ``extra_context`` rebuilt from the
-stored gates / frame metrics, and the clay geometry views rendered for the in-run judge
+the loop's (``record.previous_verdict``: round N-1's, None when it went unjudged), the track's
+``extra_context`` rebuilt from the stored gates / frame metrics, and the clay geometry views rendered for the in-run judge
(``renders/rNN/clay/``) reattached when present. The judge class is the in-run one:
``judges.vlm_judge.judge_for``, the rule the live run uses.
"""
@@ -18,12 +18,12 @@ from pydantic import ValidationError
from codeverse3d.cli import _common as C
from codeverse3d.contracts.artifacts import BuildResult, RenderSet, RenderView
-from codeverse3d.contracts.common import TRACK_INFO
from codeverse3d.contracts.plan import Plan
from codeverse3d.contracts.run import RoundRecord, RunRecord
from codeverse3d.contracts.spec import Spec
from codeverse3d.judges.base import judged_subset, resolve_paths, round_input
from codeverse3d.proc import read_json_or_none
+from codeverse3d.record.record import previous_verdict
from codeverse3d.workspace import Workspace
@@ -39,12 +39,14 @@ def load_round(ws: Workspace, rec: RunRecord, index: int) -> RoundRecord | None:
def rubric_for(rec: RunRecord, rnd: RoundRecord, override: str | None) -> str:
- """--rubric > the rubric the round was judged with > the track default (TRACK_INFO)."""
+ """--rubric > the rubric the round was judged with > the run's (``record.rubric_of``)."""
+ from codeverse3d.record.record import rubric_of
+
if override:
return override
if rnd.judgment is not None and rnd.judgment.rubric:
return rnd.judgment.rubric
- return TRACK_INFO[rec.spec.track].rubric
+ return rubric_of(rec)
def stored_plan(ws: Workspace, rec: RunRecord) -> Plan | None:
@@ -61,20 +63,6 @@ def stored_plan(ws: Workspace, rec: RunRecord) -> Plan | None:
return None
-def previous_judgment(ws: Workspace, rec: RunRecord, index: int) -> Any:
- """The preceding round's verdict (skipping degraded ones), like the round loop."""
- from codeverse3d.record.record import effective_judgment
-
- for i in range(index - 1, -1, -1):
- rnd = load_round(ws, rec, i)
- if rnd is None:
- continue
- j = effective_judgment(rnd)
- if j is not None:
- return j
- return None
-
-
def extra_context_for(ws: Workspace, rec: RunRecord, plan: Plan | None, rnd: RoundRecord) -> str:
"""The track pipeline's OWN ``judge_context`` over the stored artifacts —
``judge_context(ws, plan, round_index, build, gates)`` needs no run context,
@@ -102,7 +90,7 @@ def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:
"""``round_input`` for a stored round (``3dcode judge``, calibration)."""
plan = stored_plan(ws, rec)
return round_input(rec.spec, plan, rnd, renders=judged_subset(resolve_paths(ws, rnd.renders)), gates=rnd.gates,
- previous=previous_judgment(ws, rec, rnd.index), extra_context=extra_context_for(ws, rec, plan, rnd),
+ previous=previous_verdict([load_round(ws, rec, rnd.index - 1)], rnd.index), extra_context=extra_context_for(ws, rec, plan, rnd),
geometry_views=clay_geometry_views(ws, rnd.index), glb_path=stored_glb_path(ws, rnd))
diff --git a/harness/codeverse3d/cli/inspect_cmd.py b/harness/codeverse3d/cli/inspect_cmd.py
index 6463fc56..a04ec949 100644
--- a/harness/codeverse3d/cli/inspect_cmd.py
+++ b/harness/codeverse3d/cli/inspect_cmd.py
@@ -103,14 +103,15 @@ def judge(
artifacts/judge/rNN_cli.json."""
from codeverse3d.addons import select
from codeverse3d.cli import _judge as J
- from codeverse3d.record.record import RecordError, load_record
+ from codeverse3d.record.record import RecordError, join_post_run, load_record
ws = C.open_workspace(slug, runs_dir)
try:
rec = load_record(ws)
except RecordError as e:
raise C.CliError(str(e), code=2) from e
- picked = select.pick(ws.root, record=rec) if round_index is None else None
+ # the round `pick` packaged (selection.json), else the best-scored one
+ picked = select.summarise(ws.root, record=rec).picked_round if round_index is None else None
idx = round_index if round_index is not None else (picked if picked is not None else max(ws.rendered_rounds(), default=0))
rnd = J.load_round(ws, rec, idx)
if rnd is None or rnd.renders is None or not rnd.renders.views:
@@ -125,14 +126,12 @@ def judge(
f"(e.g. {missing[0]}): nothing to re-judge it from", code=2)
judge_obj = J.make_judge(rec.spec, rubric_name, model or rec.spec.backends.judge, n)
n_images = J.count_prompt_images(inp, rubric_name, judge_obj)
- from codeverse3d.cost.instrument import run_ledger
-
# writes artifacts/judge/rNN_cli.json into the run: one writer per run dir
with C.mutating(ws, what=f"3dcode judge {ws.root.name}", action="judge"):
try:
- # a re-judge joins the run's ledger when it has one; otherwise the per-process
- # log (a ledger holding only this verdict would be read as the whole run's cost)
- with run_ledger(ws.root, run=ws.root.name, create=False):
+ # a re-judge joins the run's ledger (and its record's total) when it has one; otherwise
+ # the per-process log (a ledger holding only this verdict would be read as the whole run's cost)
+ with join_post_run(ws):
verdict = judge_obj.judge(inp)
except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothing
raise C.CliError(f"judge failed: {e}") from e
diff --git a/harness/codeverse3d/cli/texture_cmd.py b/harness/codeverse3d/cli/texture_cmd.py
index 850f47ce..9d4a30e8 100644
--- a/harness/codeverse3d/cli/texture_cmd.py
+++ b/harness/codeverse3d/cli/texture_cmd.py
@@ -30,38 +30,38 @@ def pass_(
judge_model: Annotated[str | None, typer.Option("--judge-model", help="judge chat model id (default: spec judge)")] = None,
image_model: Annotated[str | None, typer.Option("--image-model", help="gemini image model (default gemini-3.1-flash-image)")] = None,
size: Annotated[int, typer.Option("--size", min=256, max=2048)] = 1024,
+ force: Annotated[bool, typer.Option("--force", help="buy a new pass even if one already started from this GLB")] = False,
runs_dir: RunsDirOpt = None,
) -> None:
- """Texture the picked round's object.glb (``addons.select``) → artifacts/object_textured.glb (+ textures/)."""
+ """Texture the picked round's object.glb (``addons.select.texture_round``) → artifacts/object_textured.glb
+ (+ textures/); a packaged hand-over of that round is refreshed."""
from codeverse3d import reference
from codeverse3d.addons import select
- from codeverse3d.cost.instrument import run_ledger
- from codeverse3d.record.record import RecordError, load_record
- from codeverse3d.spatial.tool_common import load_plan
- from codeverse3d.texturing.run import texture_pass, texture_supported
+ from codeverse3d.record.record import RecordError, join_post_run, load_record
+ from codeverse3d.texturing.run import load_report
ws = C.open_workspace(slug, runs_dir)
- spec = C.load_spec(ws)
- if not texture_supported(spec.track):
- raise C.CliError(f"texture pass is for object tracks; {spec.track.value} runs use `3dcode texture scene-pack`")
try:
rec = load_record(ws)
except RecordError as e:
raise C.CliError(str(e), code=2) from e
picked = select.summarise(ws.root, record=rec).picked_round
- rnd = next((r for r in rec.rounds if r.index == picked), None)
- if rnd is None or (glb := select.round_file(ws, rnd)) is None:
- raise C.CliError(f"no picked round with its own object.glb to texture in {ws.root.name} (picked: {picked})")
- sheet = ws.rebase(rnd.renders.contact_sheet) if rnd.renders is not None and rnd.renders.contact_sheet else None
- plan = load_plan(ws.plan_path)
-
- # a post-hoc pass joins the run's ledger when it has one, else the per-process log;
- # it rewrites the run's artifacts, so it holds the run mutex (one writer per run dir)
- with (C.mutating(ws, what=f"3dcode texture pass {ws.root.name}", action="texture"),
- run_ledger(ws.root, run=ws.root.name, create=False)):
- rep = texture_pass(ws, spec, plan, model_id=model or spec.backends.planner,
- image_model=reference.get_image_model(image_model or ""), judge=judge,
- judge_model_id=judge_model, size=size, glb_in=glb, sheet=sheet)
+ if picked is None:
+ raise C.CliError(f"no picked round to texture in {ws.root.name}: `3dcode pick {ws.root.name} --round N` names one")
+ # a post-hoc pass joins the run's ledger (and its record's total) when it has one; it
+ # rewrites the run's artifacts, so it holds the run mutex (one writer per run dir)
+ with C.mutating(ws, what=f"3dcode texture pass {ws.root.name}", action="texture"), join_post_run(ws):
+ try:
+ rep = select.texture_round(ws, rec, picked, image_model=reference.get_image_model(image_model or ""),
+ model_id=model, judge=judge, judge_model_id=judge_model, size=size, force=force)
+ except ValueError as e:
+ raise C.CliError(str(e)) from e
+ sel = select.load_selection(ws)
+ if sel is not None and sel.round == picked: # the hand-over carries the pack only if it is refreshed
+ select.package(ws.root, picked, method=sel.method)
+ if rep is None:
+ warn("a pass already started from this round's GLB: its report is below (--force buys a new one)")
+ rep = load_report(ws)
_print_report(rep, ws)
@@ -110,7 +110,7 @@ def scene_pack(
) -> None:
"""Generate the scene's tileable texture pack into public/textures/ (+ manifest.json)."""
from codeverse3d import reference
- from codeverse3d.cost.instrument import run_ledger
+ from codeverse3d.record.record import join_post_run
from codeverse3d.spatial.tool_common import load_plan
from codeverse3d.texturing.plan import scene_texture_pack, texture_pack_prompt
@@ -122,8 +122,7 @@ def scene_pack(
model_id = spec.backends.planner if model is None else model
# like `pass`: the pack's plan + image spend joins the run's ledger when it has one
- with (C.mutating(ws, what=f"3dcode texture scene-pack {ws.root.name}", action="texture"),
- run_ledger(ws.root, run=ws.root.name, create=False)):
+ with C.mutating(ws, what=f"3dcode texture scene-pack {ws.root.name}", action="texture"), join_post_run(ws):
pack = scene_texture_pack(plan, out or ws.public / "textures", reference.get_image_model(image_model or ""), model_id, size=size, n_max=n)
rows = {name: f"{e.file or 'FAILED'} tile={e.tile_size_m:.2f}m {e.material_family}/{e.role} seam={e.seam_score:.3f}"
+ (f" {e.error}" if e.error else "") for name, e in pack.entries.items()}
diff --git a/harness/codeverse3d/judges/base.py b/harness/codeverse3d/judges/base.py
index 6b985688..dce99673 100644
--- a/harness/codeverse3d/judges/base.py
+++ b/harness/codeverse3d/judges/base.py
@@ -8,6 +8,7 @@ from collections.abc import Sequence
from pydantic import BaseModel, Field
from codeverse3d.contracts.artifacts import GateReport, Judgment, Measurement, RenderSet
+from codeverse3d.contracts.common import TRACK_INFO, Track
from codeverse3d.contracts.plan import (
AcceptanceItem,
ArticulatedPlan,
@@ -24,6 +25,18 @@ from codeverse3d.workspace import Workspace
#: tracks whose rounds have one canonical GLB the judge slice channel (D48) may cut;
#: scene and graphics have none and are unaffected.
SLICE_TRACKS = ("static_object", "articulated_object")
+#: the rubric of a static object judged against reference images: it adds the measured silhouette criterion
+REFERENCE_RUBRIC = "reference_v1"
+
+
+def run_rubric(spec: Spec, default: str = "") -> str:
+ """The rubric a run of ``spec`` is judged on — the ONE rule, for the run loop and every replay.
+ A static object with reference images is scored on :data:`REFERENCE_RUBRIC`; any other run on
+ ``default`` (a track's own) or its track's. Other tracks keep their rubric with references:
+ their judge still sees the images."""
+ if spec.references and spec.track is Track.STATIC_OBJECT:
+ return REFERENCE_RUBRIC
+ return default or TRACK_INFO[spec.track].rubric
class JudgeInput(BaseModel):
diff --git a/harness/codeverse3d/record/record.py b/harness/codeverse3d/record/record.py
index 7d9250e0..52545e1f 100644
--- a/harness/codeverse3d/record/record.py
+++ b/harness/codeverse3d/record/record.py
@@ -13,7 +13,8 @@ from __future__ import annotations
import logging
import platform
import subprocess
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterable, Iterator
+from contextlib import contextmanager
from datetime import UTC, datetime
from functools import lru_cache
from pathlib import Path
@@ -21,6 +22,7 @@ from typing import Any, NamedTuple
from codeverse3d import __version__
from codeverse3d.config import get_settings
+from codeverse3d.contracts.artifacts import Judgment
from codeverse3d.contracts.run import RoundRecord, RunId, RunRecord
from codeverse3d.cost.ledger import existing_ledger_path, ledger_usage, load_ledger
from codeverse3d.proc import read_json_or_none, version_line
@@ -122,6 +124,23 @@ def effective_judgment(r: RoundRecord):
return None if j is None or j.degraded else j
+def previous_verdict(rounds: Iterable[RoundRecord | None], index: int) -> Judgment | None:
+ """The verdict round ``index``'s judge is shown as the last one: round ``index - 1``'s effective
+ judgment, None when that round went unjudged — never an older round's (the judge prompt calls
+ it round N-1). The round loop's rule; ``3dcode judge`` and calibration replay it."""
+ prev = next((r for r in rounds if r is not None and r.index == index - 1), None)
+ return effective_judgment(prev) if prev is not None else None
+
+
+def rubric_of(record: RunRecord) -> str:
+ """The rubric ``record``'s run is judged on: the one it stamped (``extra["rubric"]``), else —
+ an older record — its last judged round's, else the run rule (``judges.base.run_rubric``)."""
+ from codeverse3d.judges.base import run_rubric
+
+ judged = (r.judgment.rubric for r in reversed(record.rounds) if r.judgment is not None and r.judgment.rubric)
+ return str(record.extra.get("rubric") or "") or next(judged, "") or run_rubric(record.spec)
+
+
def effective_score(r: RoundRecord) -> float | None:
"""``r.score`` with degraded judgments filtered out (None = unscored)."""
j = effective_judgment(r)
@@ -224,6 +243,35 @@ def package_run(ws: Workspace, record: RunRecord) -> None:
log.warning("telemetry not written for %s: %s", ws.root, e)
+def repackage(ws: Workspace) -> RunRecord | None:
+ """Re-package a finished run's ``record.json`` after its ledger grew (:func:`package_run`), and
+ return it; None — nothing written — when the run keeps no ledger or no readable record."""
+ if existing_ledger_path(ws.root) is None:
+ return None
+ try:
+ record = load_record(ws)
+ except RecordError:
+ return None
+ package_run(ws, record)
+ ws.write_json(ws.record_path, record)
+ return record
+
+
+@contextmanager
+def join_post_run(ws: Workspace) -> Iterator[None]:
+ """Work on a FINISHED run that may spend money (a re-judge, a texture pass or pack, a pairwise
+ pick): its rows join the run's ledger when it keeps one (``run_ledger(create=False)``), and
+ on the way out — also when the work raised after paying — the record is re-packaged, so
+ ``record.total_usage`` stays the ledger's sum (D84)."""
+ from codeverse3d.cost.instrument import run_ledger
+
+ try:
+ with run_ledger(ws.root, run=ws.root.name, create=False):
+ yield
+ finally:
+ repackage(ws)
+
+
def finalize_record(ws: Workspace, record: RunRecord) -> Path:
"""Fill derived fields + environment, write ``telemetry/`` and ``record.json``."""
record.workspace = record.workspace or str(ws.root)
diff --git a/harness/codeverse3d/record/telemetry.py b/harness/codeverse3d/record/telemetry.py
index b631e77b..f1ec2105 100644
--- a/harness/codeverse3d/record/telemetry.py
+++ b/harness/codeverse3d/record/telemetry.py
@@ -24,7 +24,6 @@ import logging
from typing import Any
from codeverse3d.config import get_settings
-from codeverse3d.contracts.common import TRACK_INFO
from codeverse3d.contracts.run import (
CostSummary,
RoleSettings,
@@ -36,7 +35,7 @@ from codeverse3d.contracts.run import (
from codeverse3d.cost.ledger import load_ledger, summarise
from codeverse3d.cost.types import STAGE_ORDER, CallCost
from codeverse3d.proc import read_json_or_none, write_json_atomic
-from codeverse3d.record.record import effective_score
+from codeverse3d.record.record import effective_score, rubric_of
from codeverse3d.workspace import Workspace
log = logging.getLogger(__name__)
@@ -63,17 +62,6 @@ def backend_kind(model_id: str) -> str:
return model_id.split(":", 1)[0] if ":" in model_id else model_id
-def _rubric_name(record: RunRecord) -> str:
- name = str(record.extra.get("rubric") or "")
- if name:
- return name
- for r in reversed(record.rounds):
- if r.judgment is not None and r.judgment.rubric:
- return r.judgment.rubric
- info = TRACK_INFO.get(record.spec.track)
- return info.rubric if info else ""
-
-
def rubric_hash(name: str) -> str:
if not name:
return ""
@@ -109,7 +97,7 @@ def _role_settings(record: RunRecord) -> list[RoleSettings]:
def settings_snapshot(record: RunRecord) -> SettingsSnapshot:
"""The resolved 'how was this run configured' block (no secrets, ever)."""
spec = record.spec
- rubric = _rubric_name(record)
+ rubric = rubric_of(record)
render: dict[str, Any] = {}
limits: dict[str, Any] = {}
keys = 0
diff --git a/harness/codeverse3d/tracks/lifecycle.py b/harness/codeverse3d/tracks/lifecycle.py
index e5c1d2c6..3befa89f 100644
--- a/harness/codeverse3d/tracks/lifecycle.py
+++ b/harness/codeverse3d/tracks/lifecycle.py
@@ -30,6 +30,7 @@ from codeverse3d.cost.context import run_binding
from codeverse3d.cost.instrument import metered_agent, metered_chat_model, run_ledger
from codeverse3d.cost.ledger import ledger_usage, load_ledger
from codeverse3d.cost.tally import timed
+from codeverse3d.judges.base import run_rubric
from codeverse3d.orchestrator import (
BudgetExceeded,
BudgetGuard,
@@ -45,6 +46,7 @@ from codeverse3d.orchestrator import (
from codeverse3d.proc import EventLog, fan_out, read_json_or_none
from codeverse3d.prompts import load_text, render
from codeverse3d.prompts.catalog import language_text
+from codeverse3d.record.record import previous_verdict
from codeverse3d.tracks.candidates import run_best_of_n
from codeverse3d.tracks.common import RunContext, Services
from codeverse3d.tracks.generation import ALLOWED_ROOTS, GenerationTask, single_shot_model_id
@@ -70,8 +72,6 @@ from codeverse3d.workspace import Workspace
log = logging.getLogger(__name__)
-REFERENCE_RUBRIC = "reference_v1"
-
def plan_stage_inputs(spec: Spec) -> dict[str, Any]:
"""Only the spec fields the PLAN depends on: hashing the whole Spec meant that
@@ -444,9 +444,7 @@ class BaseTrack:
budget.restore(BudgetSnapshot.model_validate(snap))
policy = self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds)
policy = replace(policy, n_candidates=self._resolve_candidates(spec, settings))
- # reference images: static objects are scored with reference_v1 (adds the measured silhouette
- # criterion); other tracks keep their rubric but the judge still sees the references.
- rubric = REFERENCE_RUBRIC if spec.references and self.track is Track.STATIC_OBJECT else self.rubric
+ rubric = run_rubric(spec, self.rubric)
ctx = RunContext(spec=spec, ws=ws, events=events, settings=settings, budget=budget, runtime=runtime,
services=self.services, state=state, policy=policy, track=self.track, rubric=rubric,
agent_id=spec.backends.generator,
@@ -561,8 +559,7 @@ class BaseTrack:
# last round ends the run unscored (`--rounds 0` used to stop MAX_ROUNDS with an
# unjudged baseline: the re-judge lived in the refine branch only)
rejudged.add(last.index)
- prev_j = rounds[-2].judgment if len(rounds) > 1 else None
- if rejudge_round(ctx, pipeline, last, previous=prev_j):
+ if rejudge_round(ctx, pipeline, last, previous=previous_verdict(rounds, last.index)):
self._save_budget(ctx)
if index > ctx.policy.max_rounds: # the baseline + max_rounds refine rounds all ran
return _stop(RunStatus.JUDGE_UNAVAILABLE if _unjudged(last) else RunStatus.MAX_ROUNDS)
@@ -570,7 +567,7 @@ class BaseTrack:
if not rounds: # the stages spent the clock without raising: the ONE budget path (salvage)
ctx.budget.check()
return _stop(RunStatus.BUDGET)
- previous = last.judgment if last else None
+ previous = previous_verdict(rounds, index)
if last is None:
ctx.state.status = RunStatus.GENERATING
tasks, instructions, kind = self.baseline_tasks(ctx), [], "baseline"
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index dd776c93..181f44e3 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -580,3 +580,60 @@ def test_resume_refuses_a_run_whose_clock_is_spent_even_with_force(made_run, stu
assert r.exit_code == 1 and "clock is spent" in r.output, extra
assert entered == []
assert runner.invoke(app, ["resume", run_dir.name, "--runs-dir", str(runs), "--max-minutes", "40"]).exit_code == 130
+
+
+def test_a_replay_judges_the_picked_round_on_the_runs_rubric_and_books_into_the_record(tmp_path: Path, monkeypatch):
+ """`3dcode judge` replays the run it reads. r0 0.55, r1 unjudged (a judge outage), r2 0.90, of a
+ static `--image` run:
+ * no --round after `pick --round 0` is r0, the packaged round — not the best-scored r2 (N55a);
+ * the unjudged r1 replays on the run's reference_v1, not the track's static_object_v1 (N49);
+ * r2's previous verdict is the loop's — r1's, i.e. none — not r0's (N56);
+ * what the replay (or a pick) paid joins record.total_usage: it stays the ledger's sum (N53)."""
+ import codeverse3d.judges.vlm_judge as vj
+ from codeverse3d.contracts.common import Usage
+ from codeverse3d.contracts.spec import ReferenceImage
+ from codeverse3d.cost.ledger import ledger_usage, load_ledger, record_call
+ from codeverse3d.record.record import load_record
+ from tests.flywheel_cli.conftest import _judgment, make_fake_run
+
+ runs = tmp_path / "runs"
+ ws, rec = make_fake_run(runs, "stool_img")
+ rec.spec.references = [ReferenceImage(path=str(tmp_path / "ref.png"))]
+ r2 = rec.rounds[1].model_copy(update={"index": 2, "judgment": _judgment(0.90, True, [])})
+ rec.rounds[1].judgment = None
+ rec.rounds.append(r2)
+ rec.extra["rubric"] = "reference_v1"
+ ws.write_json(ws.record_path, rec)
+ ws.write_json(ws.spec_path, rec.spec)
+ ledger = ws.telemetry / "cost.jsonl"
+
+ def paid(cost: float) -> float: # a paid call booked into the run's ledger; returns the ledger total
+ record_call(Usage(backend="gemini", model="gemini-3.7-flash", cost_usd=cost), stage="judge", label="t", ledger=ledger)
+ return ledger_usage(load_ledger(ws.root)).cost_usd
+
+ total = paid(0.03) # a pairwise verdict `pick` paid: the pick re-packages the record
+ assert runner.invoke(app, ["pick", "stool_img", "--round", "0", "--runs-dir", str(runs)]).exit_code == 0
+ assert load_record(ws).total_usage.cost_usd == pytest.approx(total)
+
+ class _Judge:
+ seen: list = []
+
+ def __init__(self, *, rubric=None, **kw):
+ self.rubric = rubric
+
+ def judge(self, inp):
+ _Judge.seen.append((inp.round_index, self.rubric, inp.previous))
+ paid(0.05)
+ return _judgment(0.7, False, [])
+
+ monkeypatch.setattr(vj, "ReferenceJudge", _Judge)
+ for args in ([], ["--round", "1"], ["--round", "2"]):
+ r = runner.invoke(app, ["judge", "stool_img", "--runs-dir", str(runs), *args])
+ assert r.exit_code == 0, r.output
+ (i0, _, _), (i1, rubric1, _), (i2, _, prev2) = _Judge.seen
+ assert (i0, i1, i2) == (0, 1, 2)
+ assert rubric1 == "reference_v1"
+ assert prev2 is None
+ assert load_record(ws).total_usage.cost_usd == pytest.approx(ledger_usage(load_ledger(ws.root)).cost_usd)
+ assert load_record(ws).total_usage.cost_usd == pytest.approx(total + 3 * 0.05)
+
diff --git a/harness/tests/flywheel_cli/test_run_layout_cli.py b/harness/tests/flywheel_cli/test_run_layout_cli.py
index 502252ba..ddb9bf4e 100644
--- a/harness/tests/flywheel_cli/test_run_layout_cli.py
+++ b/harness/tests/flywheel_cli/test_run_layout_cli.py
@@ -85,8 +85,7 @@ def test_package_run_builds_a_stable_independent_and_loadable_handover(fake_run)
assert [(f.path, f.sha256) for f in d.files] == [(f.path, f.sha256) for f in second.files]
# Package metadata can be recovered from files even before the caller saves rec.
- bare = load_record(ws)
- assert bare.telemetry is None
+ bare = load_record(ws).model_copy(update={"telemetry": None})
assert load_telemetry(ws, bare) is not None and load_deliverable(ws) is not None
# Delivered assets are copies: editing a hand-over cannot mutate evidence.
diff --git a/harness/tests/flywheel_cli/test_select.py b/harness/tests/flywheel_cli/test_select.py
index 9aa8e4f0..7ebc7daa 100644
--- a/harness/tests/flywheel_cli/test_select.py
+++ b/harness/tests/flywheel_cli/test_select.py
@@ -292,14 +292,28 @@ def test_round_outputs_are_the_rounds_own_never_the_last_builds(tmp_path: Path):
def test_cli_texture_pass_textures_the_picked_round_not_the_last_build(tmp_path: Path, monkeypatch):
- """B5: `3dcode texture pass` textures the handed-over round's GLB, not the last build's."""
+ """B5: `3dcode texture pass` textures the handed-over round's GLB, not the last build's — through
+ `select.texture_round` (N57): a pass that already started from those bytes is not bought again
+ (--force buys it), and the hand-over of that round is refreshed."""
+ from codeverse3d.texturing.run import report_path
+
ws, _ = make_fake_run(tmp_path / "runs", "chair", scores=(0.90, 0.50)) # r00 is picked, r01 built last
- seen: dict = {}
- monkeypatch.setattr("codeverse3d.texturing.run.texture_pass", lambda *a, **kw: seen.update(kw))
+ select.package(ws.root, 0, method="score")
+ seen: list[dict] = []
+ monkeypatch.setattr("codeverse3d.texturing.run.texture_pass", lambda *a, **kw: seen.append(kw) or object())
+ monkeypatch.setattr("codeverse3d.texturing.run.load_report", lambda ws: object())
monkeypatch.setattr("codeverse3d.cli.texture_cmd._print_report", lambda rep, ws: None)
monkeypatch.setattr("codeverse3d.reference.get_image_model", lambda name: None)
- monkeypatch.setattr("codeverse3d.spatial.tool_common.load_plan", lambda path: None)
- r = runner.invoke(app, ["texture", "pass", "chair", "--runs-dir", str(tmp_path / "runs"), "--no-judge"])
+ packaged: list[int] = []
+ monkeypatch.setattr(select, "package", lambda run_dir, index, **kw: packaged.append(index))
+ cmd = ["texture", "pass", "chair", "--runs-dir", str(tmp_path / "runs"), "--no-judge"]
+ r = runner.invoke(app, cmd)
assert r.exit_code == 0, r.output
- assert seen["glb_in"] == ws.round_artifacts(0) / "object.glb"
- assert seen["sheet"] is not None and seen["sheet"].is_relative_to(ws.renders_dir(0))
+ assert seen[0]["glb_in"] == ws.round_artifacts(0) / "object.glb" and seen[0]["judge"] is False
+ assert seen[0]["sheet"] is not None and seen[0]["sheet"].is_relative_to(ws.renders_dir(0))
+ assert packaged == [0] # the hand-over carries the new pack
+ ws.write_json(report_path(ws), {"glb_in": str(ws.round_artifacts(0) / "object.glb")})
+ r = runner.invoke(app, cmd)
+ assert r.exit_code == 0 and len(seen) == 1 and "already started" in r.output # never bought twice
+ r = runner.invoke(app, [*cmd, "--force"])
+ assert r.exit_code == 0 and len(seen) == 2