Skip to content

Official Codebase of 3DCodeVerse

Python

cli: plain local imports instead of lazy(); calibration re-judges with `3dcode judge`'s input and judge; the gallery finds eval/bench/out

gaoypeng455d2b4f034d
diff --git a/harness/codeverse3d/addons/calibration.py b/harness/codeverse3d/addons/calibration.py
index 7fcfe1e0..b71002cb 100644
--- a/harness/codeverse3d/addons/calibration.py
+++ b/harness/codeverse3d/addons/calibration.py
@@ -1,14 +1,16 @@
"""Judge calibration smoke: re-judge finished runs and ask "does the judge separate good from bad?"
``calibrate(run_dirs)`` reads each run's ``rounds/rNN.json`` (read-only), rebuilds
-the ``JudgeInput`` (spec, renders, measurement, gates, plan digest, acceptance),
+the ``JudgeInput`` the way ``3dcode judge`` does (``cli/_judge.build_judge_input``: spec,
+renders, measurement, gates, plan digest, acceptance, previous verdict, track context,
+stored clay views, GLB) and picks the judge class it would (``make_judge``),
re-judges every round with ``n_samples`` (montage order shuffled per sample) and
tabulates, per round: gate error count, the stored overall, the new mean/std,
per-criterion std, defects and caps. Across rounds it reports the Pearson and
Spearman correlation between gate error counts and the new scores (expected
negative), and between stored and new overalls. Optionally renders a clay /
-normals geometry set for the run's best round (``artifacts/object.glb``) so the
-geometry montage is exercised. Output goes to ``out_dir`` (never into the run).
+normals geometry set for a best round that stored none (``artifacts/object.glb``) so
+the geometry montage is exercised. Output goes to ``out_dir`` (never into the run).
CLI: ``python -m codeverse3d.addons.calibration runs/a runs/b --model gemini:gemini-3.7-flash --n 3 --out scratch/``
"""
@@ -26,15 +28,15 @@ from typing import Any
from pydantic import BaseModel, Field
+from codeverse3d.cli._judge import build_judge_input, make_judge, rubric_for
from codeverse3d.contracts.artifacts import Judgment, RenderSet
-from codeverse3d.contracts.common import TRACK_INFO
-from codeverse3d.contracts.plan import AcceptanceItem
-from codeverse3d.contracts.run import RoundRecord
+from codeverse3d.contracts.run import RoundRecord, RunRecord
from codeverse3d.contracts.spec import Spec
-from codeverse3d.judges.base import JudgeInput, judged_subset, plan_digest, resolve_paths
+from codeverse3d.judges.base import JudgeInput
from codeverse3d.judges.rubrics import is_degraded
from codeverse3d.judges.vlm_judge import VlmJudge
from codeverse3d.proc import read_json_or_none
+from codeverse3d.record.record import RecordError, load_record
from codeverse3d.workspace import Workspace
log = logging.getLogger(__name__)
@@ -157,13 +159,13 @@ class CalibrationTable(BaseModel):
def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[RoundCase]:
"""Rebuild judge inputs for every judgeable round of a run (rounds with renders)."""
run_dir = Path(run_dir)
- spec = Spec.model_validate_json((run_dir / "spec.json").read_text())
- plan = read_json_or_none(run_dir / "plan.json") or {}
- record = read_json_or_none(run_dir / "record.json") or {}
- best = record.get("best_round")
- acceptance = plan.get("acceptance") or []
- digest = plan_digest(plan)
ws = Workspace(run_dir)
+ try:
+ run = load_record(ws)
+ except RecordError: # an interrupted run: rounds/ + spec.json (+ whatever best_round it left)
+ run = RunRecord(spec=Spec.model_validate_json(ws.spec_path.read_text()), workspace=str(run_dir),
+ best_round=(read_json_or_none(ws.record_path) or {}).get("best_round"))
+ best = run.best_round
glb = run_dir / "artifacts" / "object.glb"
cases: list[RoundCase] = []
for path in sorted((run_dir / "rounds").glob("r*.json")):
@@ -172,15 +174,10 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Ro
continue
if rec.renders is None or not rec.renders.views:
continue
- rubric = rec.judgment.rubric if rec.judgment else TRACK_INFO[spec.track].rubric
- # the same views, at the same paths, that the in-run judge and `3dcode judge` see
- inp = JudgeInput(
- spec=spec, renders=judged_subset(resolve_paths(ws, rec.renders)), measurement=rec.measurement, gates=rec.gates,
- acceptance=[AcceptanceItem.model_validate(a) for a in acceptance], plan_summary=digest,
- round_index=rec.index,
- )
+ # the same input, views and paths the in-run judge and `3dcode judge` see
cases.append(RoundCase(
- run=_run_label(run_dir), round_index=rec.index, kind=rec.kind, rubric=rubric, inp=inp,
+ run=_run_label(run_dir), round_index=rec.index, kind=rec.kind, rubric=rubric_for(run, rec, None),
+ inp=build_judge_input(ws, run, rec),
gate_errors=sum(len(g.errors) for g in rec.gates),
gate_warnings=sum(1 for g in rec.gates for f in g.findings if f.severity.value == "warn"),
stored=rec.judgment, is_best=(best == rec.index) if best is not None else False,
@@ -237,7 +234,8 @@ def spearman(xs: list[float], ys: list[float]) -> float | None:
# --------------------------------------------------------------------------- main entry
def _judge_case(case: RoundCase, judge: VlmJudge, geometry: RenderSet | None, out: Path) -> CalibrationRow:
t0 = time.time()
- j = judge.judge(case.inp.model_copy(update={"geometry_views": geometry}) if geometry is not None else case.inp)
+ inp = case.inp.model_copy(update={"geometry_views": geometry}) if geometry is not None else case.inp
+ j = judge.judge(inp)
raw = json.loads(j.raw) if j.raw else {}
jdir = out / "judgments"
jdir.mkdir(parents=True, exist_ok=True)
@@ -252,7 +250,7 @@ def _judge_case(case: RoundCase, judge: VlmJudge, geometry: RenderSet | None, ou
per_criterion_mean=dict(j.scores), per_criterion_std=dict(raw.get("per_criterion_std", {})),
defects_present=[d for d, on in raw.get("defects", {}).items() if on],
caps=[c["rule"] for c in raw.get("caps", {}).get("caps_applied", [])],
- passed=j.passed, n_used=j.n_samples, geometry_views=geometry is not None,
+ passed=j.passed, n_used=j.n_samples, geometry_views=inp.geometry_views is not None,
cost_usd=j.usage.cost_usd, duration_s=round(time.time() - t0, 1), summary=j.summary,
error=j.summary if is_degraded(j) else "",
)
@@ -289,13 +287,18 @@ def calibrate(
if not cases:
raise ValueError(f"no judgeable rounds found under {list(map(str, run_dirs))}")
geometry: dict[int, RenderSet | None] = {}
- for i, c in enumerate(cases):
- geometry[i] = render_geometry_views(c, out, geometry_mode) if (geometry_mode and c.is_best) else None
- judges = {c.rubric: VlmJudge(c.rubric, model_id=model_id, n_samples=n_samples, thinking=thinking, # type: ignore[arg-type]
- cache_dir=out / "cache", label="calibrate", chat_model=chat_model,
- fixed_order=fixed_order) for c in cases}
+ for i, c in enumerate(cases): # a round's stored clay views win; render only where there are none
+ want = geometry_mode and c.is_best and c.inp.geometry_views is None
+ geometry[i] = render_geometry_views(c, out, geometry_mode) if want else None
+
+ def kind(c: RoundCase) -> tuple[str, bool]: # what make_judge's choice of class depends on
+ return c.rubric, bool(c.inp.spec.references)
+
+ judges = {kind(c): make_judge(c.inp.spec, c.rubric, model_id, n_samples, thinking=thinking,
+ cache_dir=out / "cache", label="calibrate", chat_model=chat_model,
+ fixed_order=fixed_order) for c in cases}
with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool:
- rows = list(pool.map(lambda ic: _judge_case(ic[1], judges[ic[1].rubric], geometry[ic[0]], out), enumerate(cases)))
+ rows = list(pool.map(lambda ic: _judge_case(ic[1], judges[kind(ic[1])], geometry[ic[0]], out), enumerate(cases)))
ok = [r for r in rows if not r.error]
table = CalibrationTable(
model_id=model_id, n_samples=n_samples, fixed_order=fixed_order, rows=rows,
diff --git a/harness/codeverse3d/cli/_common.py b/harness/codeverse3d/cli/_common.py
index a72ececb..a566fce4 100644
--- a/harness/codeverse3d/cli/_common.py
+++ b/harness/codeverse3d/cli/_common.py
@@ -1,6 +1,6 @@
"""Shared CLI plumbing: Rich formatting (run summaries, tables, observations, doctor
-rows), the run-mutation mutex, workspace opening / creation, lazy imports and the
-cost-quality dial resolution every command goes through."""
+rows), the run-mutation mutex, workspace opening / creation, the ``bench`` import and
+the cost-quality dial resolution every command goes through."""
from __future__ import annotations
@@ -142,36 +142,24 @@ class CliError(typer.Exit):
super().__init__(code=code)
-def lazy(module: str, attr: str | None = None) -> Any:
- """Import ``module`` (and ``attr``) at call time; a clear message when the
- sub-package is not implemented yet (packages are developed in parallel)."""
- try:
- mod = importlib.import_module(module)
- except ImportError as e:
- raise CliError(
- f"{module} is not available ({e}). This command needs that sub-package; "
- f"run `3dcode doctor` to see what is installed.", code=2
- ) from e
- if attr is None:
- return mod
- try:
- return getattr(mod, attr)
- except AttributeError as e:
- raise CliError(f"{module}.{attr} is missing ({e}); the sub-package may be incomplete.", code=2) from e
-
-
#: the evaluation code lives NEXT TO the harness (``<repo>/eval``), not inside it
EVAL_ROOT = REPO_ROOT.parent / "eval"
-def import_bench() -> Any:
- """``bench`` is ``<repo>/eval/bench`` (not part of the package); make it importable."""
+def import_bench(module: str) -> Any:
+ """``bench.<module>``: ``bench`` is ``<repo>/eval/bench`` (not part of the package), so it
+ is made importable here, and a checkout without it is a clear exit, not a traceback."""
+ name = f"bench.{module}"
try:
- return importlib.import_module("bench")
+ return importlib.import_module(name)
except ImportError:
if str(EVAL_ROOT) not in sys.path:
sys.path.insert(0, str(EVAL_ROOT))
- return lazy("bench")
+ try:
+ return importlib.import_module(name)
+ except ImportError as e:
+ raise CliError(f"{name} is not available ({e}). This command needs the evaluation code "
+ f"next to the harness ({EVAL_ROOT}).", code=2) from e
def runs_root(runs_dir: Path | None) -> Path:
diff --git a/harness/codeverse3d/cli/_judge.py b/harness/codeverse3d/cli/_judge.py
index 48bf5935..ecba400b 100644
--- a/harness/codeverse3d/cli/_judge.py
+++ b/harness/codeverse3d/cli/_judge.py
@@ -21,7 +21,14 @@ from codeverse3d.contracts.artifacts import BuildResult, RenderSet, RenderView
from codeverse3d.contracts.common import TRACK_INFO
from codeverse3d.contracts.plan import AcceptanceItem
from codeverse3d.contracts.run import RoundRecord, RunRecord
-from codeverse3d.judges.base import SLICE_TRACKS, judged_subset, plan_digest, resolve_paths
+from codeverse3d.contracts.spec import Spec
+from codeverse3d.judges.base import (
+ SLICE_TRACKS,
+ JudgeInput,
+ judged_subset,
+ plan_digest,
+ resolve_paths,
+)
from codeverse3d.proc import read_json_or_none
from codeverse3d.workspace import Workspace
@@ -70,7 +77,8 @@ def extra_context_for(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> str:
"""The track pipeline's OWN ``judge_context`` over the stored artifacts —
``judge_context(ws, plan, round_index, build, gates)`` needs no run context,
so the CLI never restates the per-track formatting."""
- get_track = C.lazy("codeverse3d.tracks", "get_track")
+ from codeverse3d.tracks import get_track
+
build = rnd.build if rnd.build is not None else BuildResult(ok=False, language=rec.spec.language.value)
try:
return get_track(rec.spec.track).make_pipeline().judge_context(ws, rec.plan, rnd.index, build, list(rnd.gates))
@@ -88,8 +96,7 @@ def clay_geometry_views(ws: Workspace, index: int) -> RenderSet | None:
return RenderSet(views=views, renderer="stored") if views else None
-def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:
- JudgeInput = C.lazy("codeverse3d.judges.base", "JudgeInput")
+def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> JudgeInput:
acceptance: list[AcceptanceItem] = list(getattr(rec.plan, "acceptance", []) or [])
if not acceptance and ws.plan_path.is_file():
try:
@@ -115,19 +122,19 @@ def stored_glb_path(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> str | No
return str(p) if p.is_file() else None
-def make_judge(rec: RunRecord, rubric_name: str, model_id: str, n: int) -> Any:
- """ReferenceJudge for measured rubrics / reference specs, VlmJudge otherwise."""
- load_rubric = C.lazy("codeverse3d.judges.rubrics", "load_rubric")
+def make_judge(spec: Spec, rubric_name: str, model_id: str, n: int, **options: Any) -> Any:
+ """ReferenceJudge for measured rubrics / reference specs, VlmJudge otherwise; ``options``
+ go to the judge (calibration's cache dir, thinking level, fixed order)."""
+ from codeverse3d.judges import vlm_judge
+ from codeverse3d.judges.rubrics import load_rubric
+
try:
rubric = load_rubric(rubric_name)
except Exception as e: # unknown rubric name / bad yaml
raise C.CliError(f"cannot load rubric {rubric_name!r}: {e}") from e
- measured = bool(rubric.measured_criteria())
- if measured or rec.spec.references:
- ReferenceJudge = C.lazy("codeverse3d.judges.vlm_judge", "ReferenceJudge")
- return ReferenceJudge(model_id=model_id, n_samples=n, rubric=rubric_name)
- VlmJudge = C.lazy("codeverse3d.judges.vlm_judge", "VlmJudge")
- return VlmJudge(rubric=rubric_name, model_id=model_id, n_samples=n)
+ if rubric.measured_criteria() or spec.references:
+ return vlm_judge.ReferenceJudge(model_id=model_id, n_samples=n, rubric=rubric_name, **options)
+ return vlm_judge.VlmJudge(rubric=rubric_name, model_id=model_id, n_samples=n, **options)
def count_prompt_images(inp: Any, rubric_name: str, judge: Any = None) -> int | None:
@@ -136,9 +143,9 @@ def count_prompt_images(inp: Any, rubric_name: str, judge: Any = None) -> int |
With ``judge`` given, its D48 slice payload is counted too, so a dirty round's
printed count matches what the verdict call actually sends."""
try:
- load_rubric = C.lazy("codeverse3d.judges.rubrics", "load_rubric")
- build_judge_messages = C.lazy("codeverse3d.judges.prompt_builder", "build_judge_messages")
from codeverse3d.contracts.chat import ImagePart
+ from codeverse3d.judges.prompt_builder import build_judge_messages
+ from codeverse3d.judges.rubrics import load_rubric
slices, elicit = judge.slice_payload(inp) if judge is not None else ([], False)
_, messages = build_judge_messages(inp, load_rubric(rubric_name),
diff --git a/harness/codeverse3d/cli/cost_cmd.py b/harness/codeverse3d/cli/cost_cmd.py
index fc464bd8..1aaee5d5 100644
--- a/harness/codeverse3d/cli/cost_cmd.py
+++ b/harness/codeverse3d/cli/cost_cmd.py
@@ -1,8 +1,8 @@
"""``3dcode cost`` — what the runs cost and where the money went.
3dcode cost <slug> # one run, from its own ledger
- 3dcode cost --runs-dir bench/out/x # a whole battery, aggregated
- 3dcode cost runs/ bench/out/* # any mix of run dirs / trees
+ 3dcode cost --runs-dir eval/bench/out/x # a whole battery, aggregated
+ 3dcode cost runs/ eval/bench/out/* # any mix of run dirs / trees
3dcode cost show runs/ --md report.md # the same, explicit + full markdown
3dcode cost runs/ --recheck # re-price with today's table (drift vs what was billed)
3dcode cost prices [--stale] [--unverified]
@@ -66,7 +66,7 @@ def cost(
@cost_app.command("show")
def show(
- paths: Annotated[list[Path], typer.Argument(help="run dirs / trees of runs (runs/, bench/out/<battery>)")],
+ paths: Annotated[list[Path], typer.Argument(help="run dirs / trees of runs (runs/, eval/bench/out/<battery>)")],
md: Annotated[Path | None, typer.Option("--md", help="also write the full markdown report here")] = None,
recheck: Annotated[bool, typer.Option("--recheck", help="re-price every call with today's table")] = False,
limit: Annotated[int, typer.Option("--limit", help="rows in the per-run table")] = 20,
diff --git a/harness/codeverse3d/cli/inspect_cmd.py b/harness/codeverse3d/cli/inspect_cmd.py
index f5b3607e..2ea9e31e 100644
--- a/harness/codeverse3d/cli/inspect_cmd.py
+++ b/harness/codeverse3d/cli/inspect_cmd.py
@@ -93,12 +93,8 @@ def status(
def _print_candidates(ws) -> None:
"""Best-of-N candidate table + pairwise verdict (rounds/candidates.json), when present."""
- p = ws.root / "rounds" / "candidates.json"
- if not p.is_file():
- return
- try:
- data = json.loads(p.read_text())
- except ValueError:
+ data = read_json_or_none(ws.root / "rounds" / "candidates.json")
+ if data is None:
return
cands = data.get("candidates") or []
rows: dict[str, str] = {"n": str(data.get("n", len(cands)))}
@@ -158,14 +154,16 @@ def render(
# rendering, so a CLI render did not match the one the judge saw
r = get_settings().render
if spec.track is Track.SCENE:
- render_scene = C.lazy("codeverse3d.spatial.render_scene", "render_scene")
+ from codeverse3d.spatial.render_scene import render_scene
+
rs = render_scene(ws, out_dir, cameras=None, width=width or r.scene_width,
height=height or r.scene_height)
else:
glb = ws.artifacts / "object.glb"
if not glb.is_file():
raise C.CliError(f"no artifact to render: {glb} (run a build first)")
- render_glb = C.lazy("codeverse3d.spatial.render", "render_glb")
+ from codeverse3d.spatial.render import render_glb
+
rs = render_glb(glb, out_dir, mode=mode, width=width or r.width, height=height or r.height)
console.print(
kv_table(
@@ -209,7 +207,7 @@ def judge(
raise C.CliError(f"round {idx} has no renders (rounds/r{idx:02d}.json / record.json)")
rubric_name = J.rubric_for(rec, rnd, rubric)
inp = J.build_judge_input(ws, rec, rnd)
- judge_obj = J.make_judge(rec, rubric_name, model or rec.spec.backends.judge, n)
+ judge_obj = J.make_judge(rec.spec, rubric_name, model or rec.spec.backends.judge, n)
n_images = J.count_prompt_images(inp, rubric_name, judge_obj)
from codeverse3d.cost.instrument import run_ledger
@@ -223,7 +221,8 @@ def judge(
except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothing
raise C.CliError(f"judge failed: {e}") from e
except Exception as e:
- ReferenceJudgeError = C.lazy("codeverse3d.judges.vlm_judge", "ReferenceJudgeError")
+ from codeverse3d.judges.vlm_judge import ReferenceJudgeError
+
if isinstance(e, ReferenceJudgeError):
raise C.CliError(f"judge failed: {e}") from e
raise
@@ -252,7 +251,8 @@ def _render_graphics(ws, spec: Spec, out_dir: Path) -> None:
"""Graphics runs have no GLB: regenerate the judged frames + sheet via the runtime."""
import shutil
- get_runtime = C.lazy("codeverse3d.languages", "get_runtime")
+ from codeverse3d.languages import get_runtime
+
br = get_runtime(spec.language).build(ws)
if not br.ok:
raise C.CliError(f"graphics build failed: {br.error_type}: {br.error_message}")
diff --git a/harness/codeverse3d/cli/main.py b/harness/codeverse3d/cli/main.py
index 32e8ffc5..8b5fa20d 100644
--- a/harness/codeverse3d/cli/main.py
+++ b/harness/codeverse3d/cli/main.py
@@ -1,8 +1,7 @@
"""``3dcode`` — the 3dcodeverse command line.
Thin by design: every command builds typed inputs and calls into the harness
-packages lazily (``cli/_common.lazy``), so the CLI imports and prints help
-even while some sub-packages are incomplete.
+packages through function-local imports, so ``3dcode --help`` loads none of them.
Owns the typer app and every registration, the run-starting commands ``make`` /
``resume`` (spec building, reference grounding, track dispatch, budget raising) and
@@ -55,7 +54,8 @@ def tools(
as_json: Annotated[bool, typer.Option("--json-out", help="print the Observation as JSON")] = False,
cards: Annotated[bool, typer.Option("--cards", help="(list) print prompt cards instead of a table")] = False,
) -> None:
- registry = C.lazy("codeverse3d.spatial.registry")
+ from codeverse3d.spatial import registry
+
if name == "list":
defs = registry.list_tools()
if cards:
@@ -114,8 +114,7 @@ def run_cmd(
"""Run every prompt of a battery through its track (N parallel workers); resumable."""
if not battery.is_file():
raise C.CliError(f"battery not found: {battery}")
- b = C.import_bench()
- run_bench = C.lazy("bench.run_bench")
+ run_bench = C.import_bench("run_bench")
# one source of truth for the worker count: BenchOptions.parallel (the measured knee)
par = {"parallel": parallel} if parallel is not None else {}
opts = run_bench.BenchOptions(generator=generator, planner=planner, judge=judge, rounds=rounds, max_minutes=max_minutes,
@@ -131,17 +130,15 @@ def run_cmd(
results = run_bench.run_battery(battery, out_dir, opts, on_result=_on)
ok(f"{len(results)} results → {out_dir / 'results.csv'}")
if report:
- rep = C.lazy("bench.report").build_report(out_dir)
+ rep = C.import_bench("report").build_report(out_dir)
console.print(rep.markdown)
ok(f"report → {out_dir / 'report.md'} / report.html")
- del b
@bench_app.command("report")
def report_cmd(out_dir: Annotated[Path, typer.Argument()]) -> None:
"""Aggregate results.jsonl → report.md + report.html (gallery of contact sheets)."""
- C.import_bench()
- rep = C.lazy("bench.report").build_report(out_dir)
+ rep = C.import_bench("report").build_report(out_dir)
console.print(rep.markdown)
ok(f"report → {out_dir / 'report.md'} / report.html")
@@ -150,7 +147,7 @@ def report_cmd(out_dir: Annotated[Path, typer.Argument()]) -> None:
gallery_app = typer.Typer(no_args_is_help=True)
RootsArg = Annotated[list[Path] | None, typer.Argument(
- help="run roots (default: ./runs plus every ./bench/out/*/runs that exists)")]
+ help="run roots (default: ./runs, every ./bench/out/*/runs and every <repo>/eval/bench/out/*/runs that exists)")]
def resolve_roots(roots: list[Path] | None) -> list[Path]:
@@ -163,13 +160,16 @@ def resolve_roots(roots: list[Path] | None) -> list[Path]:
raise C.CliError("not a directory: " + ", ".join(str(m) for m in missing))
return [Path(r) for r in roots]
found = default_roots(Path.cwd())
+ seen = {p.resolve() for p in found} # D77: the batteries live in <repo>/eval/bench/out/<b>/runs
+ found += [p for p in sorted((C.EVAL_ROOT / "bench" / "out").glob("*/runs")) if p.is_dir() and p.resolve() not in seen]
if not found:
from codeverse3d.config import get_settings
fallback = Path(get_settings().runs_dir)
if fallback.is_dir():
return [fallback]
- raise C.CliError(f"no run roots found under {Path.cwd()} (looked for runs/ and bench/out/*/runs); "
+ raise C.CliError(f"no run roots found (looked for runs/ and bench/out/*/runs under {Path.cwd()}, "
+ f"and {C.EVAL_ROOT / 'bench' / 'out'}/*/runs); "
f"pass one explicitly: `3dcode gallery serve path/to/runs`")
return found
@@ -518,8 +518,8 @@ def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:
def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None, force: bool = False) -> None:
from codeverse3d.cost.instrument import run_ledger
+ from codeverse3d.tracks import get_track
- get_track = C.lazy("codeverse3d.tracks", "get_track")
options: dict = {"n_candidates": candidates} if candidates else {}
options.update(C.round_policy_options(spec))
try:
diff --git a/harness/codeverse3d/cli/skills_cmd.py b/harness/codeverse3d/cli/skills_cmd.py
index e115100f..82e757b8 100644
--- a/harness/codeverse3d/cli/skills_cmd.py
+++ b/harness/codeverse3d/cli/skills_cmd.py
@@ -36,7 +36,7 @@ def list_skills(
``target`` is the deterministic quantity the bundle says it moves (docs/SKILLS_LEDGER.md);
``dir`` is the direction that counts as an improvement, and a ``?`` marks a bundle whose
claim no deterministic instrument can see. Read them out with
- ``python bench/skill_targets.py bench/out``.
+ ``python eval/bench/skill_targets.py eval/bench/out``.
"""
from rich.table import Table
diff --git a/harness/codeverse3d/cli/texture_cmd.py b/harness/codeverse3d/cli/texture_cmd.py
index bbff933f..cb545e6e 100644
--- a/harness/codeverse3d/cli/texture_cmd.py
+++ b/harness/codeverse3d/cli/texture_cmd.py
@@ -15,6 +15,7 @@ import typer
from codeverse3d.cli import _common as C
from codeverse3d.cli._common import console, kv_table, warn
from codeverse3d.contracts.common import Track
+from codeverse3d.contracts.plan import ScenePlan
texture_app = typer.Typer(name="texture", help="Text-to-image texturing: object pass + scene texture pack.",
no_args_is_help=True)
@@ -23,7 +24,9 @@ RunsDirOpt = C.RunsDirOpt
def _image_model(name: str | None):
- return C.lazy("codeverse3d.reference", "_image_model")(name or "")
+ from codeverse3d import reference
+
+ return reference._image_model(name or "")
@texture_app.command("pass")
@@ -41,10 +44,11 @@ def pass_(
spec = C.load_spec(ws)
if spec.track not in (Track.STATIC_OBJECT, Track.ARTICULATED_OBJECT):
raise C.CliError(f"texture pass is for object tracks; {spec.track.value} runs use `3dcode texture scene-pack`")
- load_plan = C.lazy("codeverse3d.spatial.tool_common", "load_plan")
- plan = load_plan(ws.plan_path)
- texture_pass = C.lazy("codeverse3d.texturing.run", "texture_pass")
from codeverse3d.cost.instrument import run_ledger
+ from codeverse3d.spatial.tool_common import load_plan
+ from codeverse3d.texturing.run import texture_pass
+
+ plan = load_plan(ws.plan_path)
# a post-hoc pass joins the run's ledger when it has one, else the per-process log;
# it rewrites the run's artifacts, so it holds the run mutex (one writer per run dir)
@@ -59,8 +63,9 @@ def pass_(
@texture_app.command("show")
def show(slug: str, runs_dir: RunsDirOpt = None) -> None:
"""Print the last texturing report of a run."""
+ from codeverse3d.texturing.run import load_report
+
ws = C.open_workspace(slug, runs_dir)
- load_report = C.lazy("codeverse3d.texturing.run", "load_report")
try:
rep = load_report(ws)
except FileNotFoundError as e:
@@ -69,7 +74,8 @@ def show(slug: str, runs_dir: RunsDirOpt = None) -> None:
def _print_report(rep, ws) -> None:
- plan_table = C.lazy("codeverse3d.texturing.plan", "plan_table")
+ from codeverse3d.texturing.plan import plan_table
+
console.print(plan_table(rep.plan))
rows = {tid: f"{Path(a.path).name} seam={a.seam_score:.3f} (raw {a.seam_score_raw:.3f}) cached={a.cached}"
+ (f" ERROR {a.error}" if a.error else "") for tid, a in rep.textures.textures.items()}
@@ -98,17 +104,16 @@ def scene_pack(
runs_dir: RunsDirOpt = None,
) -> None:
"""Generate the scene's tileable texture pack into public/textures/ (+ manifest.json)."""
+ from codeverse3d.cost.instrument import run_ledger
+ from codeverse3d.spatial.tool_common import load_plan
+ from codeverse3d.texturing.plan import scene_texture_pack, texture_pack_prompt
+
ws = C.open_workspace(slug, runs_dir)
spec = C.load_spec(ws)
- load_plan = C.lazy("codeverse3d.spatial.tool_common", "load_plan")
plan = load_plan(ws.plan_path)
- ScenePlan = C.lazy("codeverse3d.contracts.plan", "ScenePlan")
if not isinstance(plan, ScenePlan):
raise C.CliError("scene-pack needs a scene run (plan.json with zones)")
- scene_texture_pack = C.lazy("codeverse3d.texturing.plan", "scene_texture_pack")
- texture_pack_prompt = C.lazy("codeverse3d.texturing.plan", "texture_pack_prompt")
model_id = spec.backends.planner if model is None else model
- from codeverse3d.cost.instrument import run_ledger
# like `pass`: the pack's plan + image spend joins the run's ledger when it has one
with (C.mutating(ws, what=f"3dcode texture scene-pack {ws.root.name}", action="texture"),
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index ae1bd225..67ce775f 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -105,9 +105,9 @@ def test_tools_list_handles_an_optional_spatial_install():
assert "Traceback" not in r.output
-def test_lazy_import_message():
+def test_a_missing_bench_module_is_a_clear_exit():
with pytest.raises(C.CliError):
- C.lazy("codeverse3d.definitely_missing_module")
+ C.import_bench("definitely_missing_module")
def test_doctor_json(monkeypatch):
diff --git a/harness/tests/gallery/test_cli.py b/harness/tests/gallery/test_cli.py
index 92a7779a..4c3e6a71 100644
--- a/harness/tests/gallery/test_cli.py
+++ b/harness/tests/gallery/test_cli.py
@@ -48,14 +48,29 @@ def test_build_rejects_a_missing_root(tmp_path: Path):
def test_default_roots_from_cwd(gallery_tree: dict[str, Path], monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", gallery_tree["root"] / "no_eval")
monkeypatch.chdir(gallery_tree["root"])
roots = resolve_roots(None)
assert [r.name for r in roots] == ["runs", "runs"]
assert str(roots[1]).endswith(os.path.join("bench", "out", "static_v9", "runs"))
+def test_default_roots_find_the_eval_batteries_from_any_cwd(gallery_tree: dict[str, Path], tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch):
+ """D77 moved the batteries to <repo>/eval/bench/out/<b>/runs; the defaults only looked
+ under the cwd, so `3dcode gallery serve` from harness/ listed none of them."""
+ battery = tmp_path / "eval" / "bench" / "out" / "b1" / "runs"
+ battery.mkdir(parents=True)
+ monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", tmp_path / "eval")
+ monkeypatch.chdir(gallery_tree["root"])
+ assert resolve_roots(None)[-1] == battery and len(resolve_roots(None)) == 3
+ monkeypatch.chdir(tmp_path / "eval") # cwd IS the eval root: each battery listed once
+ assert resolve_roots(None) == [Path.cwd() / "bench" / "out" / "b1" / "runs"]
+
+
def test_no_roots_anywhere_is_a_clear_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", tmp_path / "no_eval")
monkeypatch.setenv("C3D_RUNS_DIR", str(tmp_path / "absent"))
from codeverse3d.config import get_settings
diff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.py
index d2723053..fca4abdf 100644
--- a/harness/tests/judges/test_calibration.py
+++ b/harness/tests/judges/test_calibration.py
@@ -102,3 +102,43 @@ def test_run_labels_stay_distinct_across_battery_layouts():
assert len({a, b, c, d, e}) == 5, (a, b, c, d, e)
# a plain harness run keeps its slug as the whole label
assert _run_label(Path("/home/u/proj/runs/chair_bl")) == "chair_bl"
+
+
+def test_calibration_judges_what_the_in_run_judge_saw(tmp_path, monkeypatch):
+ """It rebuilt a thinner JudgeInput than `3dcode judge` (no previous verdict, track
+ context, stored clay views or GLB for the D48 slices), re-rendered clay views a round
+ had stored, and always used VlmJudge where `3dcode judge` picks ReferenceJudge."""
+ import codeverse3d.addons.calibration as cal
+ import codeverse3d.judges.vlm_judge as vj
+ from codeverse3d.contracts.spec import ReferenceImage
+ from tests.flywheel_cli.conftest import _judgment, make_fake_run, tiny_png
+
+ ws, rec = make_fake_run(tmp_path / "runs")
+ rec.spec = rec.spec.model_copy(update={"references": [ReferenceImage(path=str(tiny_png(ws.root / "ref.png")))]})
+ ws.write_json(ws.record_path, rec)
+ for r in rec.rounds:
+ ws.write_json(ws.root / "rounds" / f"r{r.index:02d}.json", r)
+ tiny_png(ws.renders_dir(1) / "clay" / "view_front.png")
+ made, seen = [], []
+
+ class _Vlm:
+ def __init__(self, *a, **kw):
+ made.append(type(self).__name__)
+ self.model_id = kw["model_id"]
+
+ def judge(self, inp):
+ seen.append(inp)
+ return _judgment(0.6, False, [])
+
+ class _Ref(_Vlm):
+ pass
+
+ monkeypatch.setattr(vj, "VlmJudge", _Vlm)
+ monkeypatch.setattr(vj, "ReferenceJudge", _Ref)
+ monkeypatch.setattr(cal, "render_geometry_views", lambda *a, **k: pytest.fail("stored clay views re-rendered"))
+ table = calibrate([ws.root], model_id="fake:fake-1", n_samples=1, out_dir=tmp_path / "out", max_workers=1)
+ assert set(made) == {"_Ref"}
+ r1 = next(inp for inp in seen if inp.round_index == 1)
+ assert r1.previous is not None and r1.previous.overall == 0.55
+ assert r1.geometry_views is not None and r1.glb_path == str(ws.artifacts / "object.glb")
+ assert next(row for row in table.rows if row.round == 1).geometry_views