Official Codebase of 3DCodeVerse
Python
cli: plain local imports instead of lazy(); calibration re-judges with `3dcode judge`'s input and judge; the gallery finds eval/bench/out
gaoypeng455d2b4f034d
diff --git a/harness/codeverse3d/addons/calibration.py b/harness/codeverse3d/addons/calibration.pyindex 7fcfe1e0..b71002cb 100644--- a/harness/codeverse3d/addons/calibration.py+++ b/harness/codeverse3d/addons/calibration.py@@ -1,14 +1,16 @@"""Judge calibration smoke: re-judge finished runs and ask "does the judge separate good from bad?"``calibrate(run_dirs)`` reads each run's ``rounds/rNN.json`` (read-only), rebuilds-the ``JudgeInput`` (spec, renders, measurement, gates, plan digest, acceptance),+the ``JudgeInput`` the way ``3dcode judge`` does (``cli/_judge.build_judge_input``: spec,+renders, measurement, gates, plan digest, acceptance, previous verdict, track context,+stored clay views, GLB) and picks the judge class it would (``make_judge``),re-judges every round with ``n_samples`` (montage order shuffled per sample) andtabulates, per round: gate error count, the stored overall, the new mean/std,per-criterion std, defects and caps. Across rounds it reports the Pearson andSpearman correlation between gate error counts and the new scores (expectednegative), and between stored and new overalls. Optionally renders a clay /-normals geometry set for the run's best round (``artifacts/object.glb``) so the-geometry montage is exercised. Output goes to ``out_dir`` (never into the run).+normals geometry set for a best round that stored none (``artifacts/object.glb``) so+the geometry montage is exercised. Output goes to ``out_dir`` (never into the run).CLI: ``python -m codeverse3d.addons.calibration runs/a runs/b --model gemini:gemini-3.7-flash --n 3 --out scratch/``"""@@ -26,15 +28,15 @@ from typing import Anyfrom pydantic import BaseModel, Field+from codeverse3d.cli._judge import build_judge_input, make_judge, rubric_forfrom codeverse3d.contracts.artifacts import Judgment, RenderSet-from codeverse3d.contracts.common import TRACK_INFO-from codeverse3d.contracts.plan import AcceptanceItem-from codeverse3d.contracts.run import RoundRecord+from codeverse3d.contracts.run import RoundRecord, RunRecordfrom codeverse3d.contracts.spec import Spec-from codeverse3d.judges.base import JudgeInput, judged_subset, plan_digest, resolve_paths+from codeverse3d.judges.base import JudgeInputfrom codeverse3d.judges.rubrics import is_degradedfrom codeverse3d.judges.vlm_judge import VlmJudgefrom codeverse3d.proc import read_json_or_none+from codeverse3d.record.record import RecordError, load_recordfrom codeverse3d.workspace import Workspacelog = logging.getLogger(__name__)@@ -157,13 +159,13 @@ class CalibrationTable(BaseModel):def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[RoundCase]:"""Rebuild judge inputs for every judgeable round of a run (rounds with renders)."""run_dir = Path(run_dir)- spec = Spec.model_validate_json((run_dir / "spec.json").read_text())- plan = read_json_or_none(run_dir / "plan.json") or {}- record = read_json_or_none(run_dir / "record.json") or {}- best = record.get("best_round")- acceptance = plan.get("acceptance") or []- digest = plan_digest(plan)ws = Workspace(run_dir)+ try:+ run = load_record(ws)+ except RecordError: # an interrupted run: rounds/ + spec.json (+ whatever best_round it left)+ run = RunRecord(spec=Spec.model_validate_json(ws.spec_path.read_text()), workspace=str(run_dir),+ best_round=(read_json_or_none(ws.record_path) or {}).get("best_round"))+ best = run.best_roundglb = run_dir / "artifacts" / "object.glb"cases: list[RoundCase] = []for path in sorted((run_dir / "rounds").glob("r*.json")):@@ -172,15 +174,10 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Rocontinueif rec.renders is None or not rec.renders.views:continue- rubric = rec.judgment.rubric if rec.judgment else TRACK_INFO[spec.track].rubric- # the same views, at the same paths, that the in-run judge and `3dcode judge` see- inp = JudgeInput(- spec=spec, renders=judged_subset(resolve_paths(ws, rec.renders)), measurement=rec.measurement, gates=rec.gates,- acceptance=[AcceptanceItem.model_validate(a) for a in acceptance], plan_summary=digest,- round_index=rec.index,- )+ # the same input, views and paths the in-run judge and `3dcode judge` seecases.append(RoundCase(- run=_run_label(run_dir), round_index=rec.index, kind=rec.kind, rubric=rubric, inp=inp,+ run=_run_label(run_dir), round_index=rec.index, kind=rec.kind, rubric=rubric_for(run, rec, None),+ inp=build_judge_input(ws, run, rec),gate_errors=sum(len(g.errors) for g in rec.gates),gate_warnings=sum(1 for g in rec.gates for f in g.findings if f.severity.value == "warn"),stored=rec.judgment, is_best=(best == rec.index) if best is not None else False,@@ -237,7 +234,8 @@ def spearman(xs: list[float], ys: list[float]) -> float | None:# --------------------------------------------------------------------------- main entrydef _judge_case(case: RoundCase, judge: VlmJudge, geometry: RenderSet | None, out: Path) -> CalibrationRow:t0 = time.time()- j = judge.judge(case.inp.model_copy(update={"geometry_views": geometry}) if geometry is not None else case.inp)+ inp = case.inp.model_copy(update={"geometry_views": geometry}) if geometry is not None else case.inp+ j = judge.judge(inp)raw = json.loads(j.raw) if j.raw else {}jdir = out / "judgments"jdir.mkdir(parents=True, exist_ok=True)@@ -252,7 +250,7 @@ def _judge_case(case: RoundCase, judge: VlmJudge, geometry: RenderSet | None, ouper_criterion_mean=dict(j.scores), per_criterion_std=dict(raw.get("per_criterion_std", {})),defects_present=[d for d, on in raw.get("defects", {}).items() if on],caps=[c["rule"] for c in raw.get("caps", {}).get("caps_applied", [])],- passed=j.passed, n_used=j.n_samples, geometry_views=geometry is not None,+ passed=j.passed, n_used=j.n_samples, geometry_views=inp.geometry_views is not None,cost_usd=j.usage.cost_usd, duration_s=round(time.time() - t0, 1), summary=j.summary,error=j.summary if is_degraded(j) else "",)@@ -289,13 +287,18 @@ def calibrate(if not cases:raise ValueError(f"no judgeable rounds found under {list(map(str, run_dirs))}")geometry: dict[int, RenderSet | None] = {}- for i, c in enumerate(cases):- geometry[i] = render_geometry_views(c, out, geometry_mode) if (geometry_mode and c.is_best) else None- judges = {c.rubric: VlmJudge(c.rubric, model_id=model_id, n_samples=n_samples, thinking=thinking, # type: ignore[arg-type]- cache_dir=out / "cache", label="calibrate", chat_model=chat_model,- fixed_order=fixed_order) for c in cases}+ for i, c in enumerate(cases): # a round's stored clay views win; render only where there are none+ want = geometry_mode and c.is_best and c.inp.geometry_views is None+ geometry[i] = render_geometry_views(c, out, geometry_mode) if want else None++ def kind(c: RoundCase) -> tuple[str, bool]: # what make_judge's choice of class depends on+ return c.rubric, bool(c.inp.spec.references)++ judges = {kind(c): make_judge(c.inp.spec, c.rubric, model_id, n_samples, thinking=thinking,+ cache_dir=out / "cache", label="calibrate", chat_model=chat_model,+ fixed_order=fixed_order) for c in cases}with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool:- rows = list(pool.map(lambda ic: _judge_case(ic[1], judges[ic[1].rubric], geometry[ic[0]], out), enumerate(cases)))+ rows = list(pool.map(lambda ic: _judge_case(ic[1], judges[kind(ic[1])], geometry[ic[0]], out), enumerate(cases)))ok = [r for r in rows if not r.error]table = CalibrationTable(model_id=model_id, n_samples=n_samples, fixed_order=fixed_order, rows=rows,diff --git a/harness/codeverse3d/cli/_common.py b/harness/codeverse3d/cli/_common.pyindex a72ececb..a566fce4 100644--- a/harness/codeverse3d/cli/_common.py+++ b/harness/codeverse3d/cli/_common.py@@ -1,6 +1,6 @@"""Shared CLI plumbing: Rich formatting (run summaries, tables, observations, doctor-rows), the run-mutation mutex, workspace opening / creation, lazy imports and the-cost-quality dial resolution every command goes through."""+rows), the run-mutation mutex, workspace opening / creation, the ``bench`` import and+the cost-quality dial resolution every command goes through."""from __future__ import annotations@@ -142,36 +142,24 @@ class CliError(typer.Exit):super().__init__(code=code)-def lazy(module: str, attr: str | None = None) -> Any:- """Import ``module`` (and ``attr``) at call time; a clear message when the- sub-package is not implemented yet (packages are developed in parallel)."""- try:- mod = importlib.import_module(module)- except ImportError as e:- raise CliError(- f"{module} is not available ({e}). This command needs that sub-package; "- f"run `3dcode doctor` to see what is installed.", code=2- ) from e- if attr is None:- return mod- try:- return getattr(mod, attr)- except AttributeError as e:- raise CliError(f"{module}.{attr} is missing ({e}); the sub-package may be incomplete.", code=2) from e--#: the evaluation code lives NEXT TO the harness (``<repo>/eval``), not inside itEVAL_ROOT = REPO_ROOT.parent / "eval"-def import_bench() -> Any:- """``bench`` is ``<repo>/eval/bench`` (not part of the package); make it importable."""+def import_bench(module: str) -> Any:+ """``bench.<module>``: ``bench`` is ``<repo>/eval/bench`` (not part of the package), so it+ is made importable here, and a checkout without it is a clear exit, not a traceback."""+ name = f"bench.{module}"try:- return importlib.import_module("bench")+ return importlib.import_module(name)except ImportError:if str(EVAL_ROOT) not in sys.path:sys.path.insert(0, str(EVAL_ROOT))- return lazy("bench")+ try:+ return importlib.import_module(name)+ except ImportError as e:+ raise CliError(f"{name} is not available ({e}). This command needs the evaluation code "+ f"next to the harness ({EVAL_ROOT}).", code=2) from edef runs_root(runs_dir: Path | None) -> Path:diff --git a/harness/codeverse3d/cli/_judge.py b/harness/codeverse3d/cli/_judge.pyindex 48bf5935..ecba400b 100644--- a/harness/codeverse3d/cli/_judge.py+++ b/harness/codeverse3d/cli/_judge.py@@ -21,7 +21,14 @@ from codeverse3d.contracts.artifacts import BuildResult, RenderSet, RenderViewfrom codeverse3d.contracts.common import TRACK_INFOfrom codeverse3d.contracts.plan import AcceptanceItemfrom codeverse3d.contracts.run import RoundRecord, RunRecord-from codeverse3d.judges.base import SLICE_TRACKS, judged_subset, plan_digest, resolve_paths+from codeverse3d.contracts.spec import Spec+from codeverse3d.judges.base import (+ SLICE_TRACKS,+ JudgeInput,+ judged_subset,+ plan_digest,+ resolve_paths,+)from codeverse3d.proc import read_json_or_nonefrom codeverse3d.workspace import Workspace@@ -70,7 +77,8 @@ def extra_context_for(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> str:"""The track pipeline's OWN ``judge_context`` over the stored artifacts —``judge_context(ws, plan, round_index, build, gates)`` needs no run context,so the CLI never restates the per-track formatting."""- get_track = C.lazy("codeverse3d.tracks", "get_track")+ from codeverse3d.tracks import get_track+build = rnd.build if rnd.build is not None else BuildResult(ok=False, language=rec.spec.language.value)try:return get_track(rec.spec.track).make_pipeline().judge_context(ws, rec.plan, rnd.index, build, list(rnd.gates))@@ -88,8 +96,7 @@ def clay_geometry_views(ws: Workspace, index: int) -> RenderSet | None:return RenderSet(views=views, renderer="stored") if views else None-def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:- JudgeInput = C.lazy("codeverse3d.judges.base", "JudgeInput")+def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> JudgeInput:acceptance: list[AcceptanceItem] = list(getattr(rec.plan, "acceptance", []) or [])if not acceptance and ws.plan_path.is_file():try:@@ -115,19 +122,19 @@ def stored_glb_path(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> str | Noreturn str(p) if p.is_file() else None-def make_judge(rec: RunRecord, rubric_name: str, model_id: str, n: int) -> Any:- """ReferenceJudge for measured rubrics / reference specs, VlmJudge otherwise."""- load_rubric = C.lazy("codeverse3d.judges.rubrics", "load_rubric")+def make_judge(spec: Spec, rubric_name: str, model_id: str, n: int, **options: Any) -> Any:+ """ReferenceJudge for measured rubrics / reference specs, VlmJudge otherwise; ``options``+ go to the judge (calibration's cache dir, thinking level, fixed order)."""+ from codeverse3d.judges import vlm_judge+ from codeverse3d.judges.rubrics import load_rubric+try:rubric = load_rubric(rubric_name)except Exception as e: # unknown rubric name / bad yamlraise C.CliError(f"cannot load rubric {rubric_name!r}: {e}") from e- measured = bool(rubric.measured_criteria())- if measured or rec.spec.references:- ReferenceJudge = C.lazy("codeverse3d.judges.vlm_judge", "ReferenceJudge")- return ReferenceJudge(model_id=model_id, n_samples=n, rubric=rubric_name)- VlmJudge = C.lazy("codeverse3d.judges.vlm_judge", "VlmJudge")- return VlmJudge(rubric=rubric_name, model_id=model_id, n_samples=n)+ if rubric.measured_criteria() or spec.references:+ return vlm_judge.ReferenceJudge(model_id=model_id, n_samples=n, rubric=rubric_name, **options)+ return vlm_judge.VlmJudge(rubric=rubric_name, model_id=model_id, n_samples=n, **options)def count_prompt_images(inp: Any, rubric_name: str, judge: Any = None) -> int | None:@@ -136,9 +143,9 @@ def count_prompt_images(inp: Any, rubric_name: str, judge: Any = None) -> int |With ``judge`` given, its D48 slice payload is counted too, so a dirty round'sprinted count matches what the verdict call actually sends."""try:- load_rubric = C.lazy("codeverse3d.judges.rubrics", "load_rubric")- build_judge_messages = C.lazy("codeverse3d.judges.prompt_builder", "build_judge_messages")from codeverse3d.contracts.chat import ImagePart+ from codeverse3d.judges.prompt_builder import build_judge_messages+ from codeverse3d.judges.rubrics import load_rubricslices, elicit = judge.slice_payload(inp) if judge is not None else ([], False)_, messages = build_judge_messages(inp, load_rubric(rubric_name),diff --git a/harness/codeverse3d/cli/cost_cmd.py b/harness/codeverse3d/cli/cost_cmd.pyindex fc464bd8..1aaee5d5 100644--- a/harness/codeverse3d/cli/cost_cmd.py+++ b/harness/codeverse3d/cli/cost_cmd.py@@ -1,8 +1,8 @@"""``3dcode cost`` — what the runs cost and where the money went.3dcode cost <slug> # one run, from its own ledger- 3dcode cost --runs-dir bench/out/x # a whole battery, aggregated- 3dcode cost runs/ bench/out/* # any mix of run dirs / trees+ 3dcode cost --runs-dir eval/bench/out/x # a whole battery, aggregated+ 3dcode cost runs/ eval/bench/out/* # any mix of run dirs / trees3dcode cost show runs/ --md report.md # the same, explicit + full markdown3dcode cost runs/ --recheck # re-price with today's table (drift vs what was billed)3dcode cost prices [--stale] [--unverified]@@ -66,7 +66,7 @@ def cost(@cost_app.command("show")def show(- paths: Annotated[list[Path], typer.Argument(help="run dirs / trees of runs (runs/, bench/out/<battery>)")],+ paths: Annotated[list[Path], typer.Argument(help="run dirs / trees of runs (runs/, eval/bench/out/<battery>)")],md: Annotated[Path | None, typer.Option("--md", help="also write the full markdown report here")] = None,recheck: Annotated[bool, typer.Option("--recheck", help="re-price every call with today's table")] = False,limit: Annotated[int, typer.Option("--limit", help="rows in the per-run table")] = 20,diff --git a/harness/codeverse3d/cli/inspect_cmd.py b/harness/codeverse3d/cli/inspect_cmd.pyindex f5b3607e..2ea9e31e 100644--- a/harness/codeverse3d/cli/inspect_cmd.py+++ b/harness/codeverse3d/cli/inspect_cmd.py@@ -93,12 +93,8 @@ def status(def _print_candidates(ws) -> None:"""Best-of-N candidate table + pairwise verdict (rounds/candidates.json), when present."""- p = ws.root / "rounds" / "candidates.json"- if not p.is_file():- return- try:- data = json.loads(p.read_text())- except ValueError:+ data = read_json_or_none(ws.root / "rounds" / "candidates.json")+ if data is None:returncands = data.get("candidates") or []rows: dict[str, str] = {"n": str(data.get("n", len(cands)))}@@ -158,14 +154,16 @@ def render(# rendering, so a CLI render did not match the one the judge sawr = get_settings().renderif spec.track is Track.SCENE:- render_scene = C.lazy("codeverse3d.spatial.render_scene", "render_scene")+ from codeverse3d.spatial.render_scene import render_scene+rs = render_scene(ws, out_dir, cameras=None, width=width or r.scene_width,height=height or r.scene_height)else:glb = ws.artifacts / "object.glb"if not glb.is_file():raise C.CliError(f"no artifact to render: {glb} (run a build first)")- render_glb = C.lazy("codeverse3d.spatial.render", "render_glb")+ from codeverse3d.spatial.render import render_glb+rs = render_glb(glb, out_dir, mode=mode, width=width or r.width, height=height or r.height)console.print(kv_table(@@ -209,7 +207,7 @@ def judge(raise C.CliError(f"round {idx} has no renders (rounds/r{idx:02d}.json / record.json)")rubric_name = J.rubric_for(rec, rnd, rubric)inp = J.build_judge_input(ws, rec, rnd)- judge_obj = J.make_judge(rec, rubric_name, model or rec.spec.backends.judge, n)+ judge_obj = J.make_judge(rec.spec, rubric_name, model or rec.spec.backends.judge, n)n_images = J.count_prompt_images(inp, rubric_name, judge_obj)from codeverse3d.cost.instrument import run_ledger@@ -223,7 +221,8 @@ def judge(except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothingraise C.CliError(f"judge failed: {e}") from eexcept Exception as e:- ReferenceJudgeError = C.lazy("codeverse3d.judges.vlm_judge", "ReferenceJudgeError")+ from codeverse3d.judges.vlm_judge import ReferenceJudgeError+if isinstance(e, ReferenceJudgeError):raise C.CliError(f"judge failed: {e}") from eraise@@ -252,7 +251,8 @@ def _render_graphics(ws, spec: Spec, out_dir: Path) -> None:"""Graphics runs have no GLB: regenerate the judged frames + sheet via the runtime."""import shutil- get_runtime = C.lazy("codeverse3d.languages", "get_runtime")+ from codeverse3d.languages import get_runtime+br = get_runtime(spec.language).build(ws)if not br.ok:raise C.CliError(f"graphics build failed: {br.error_type}: {br.error_message}")diff --git a/harness/codeverse3d/cli/main.py b/harness/codeverse3d/cli/main.pyindex 32e8ffc5..8b5fa20d 100644--- a/harness/codeverse3d/cli/main.py+++ b/harness/codeverse3d/cli/main.py@@ -1,8 +1,7 @@"""``3dcode`` — the 3dcodeverse command line.Thin by design: every command builds typed inputs and calls into the harness-packages lazily (``cli/_common.lazy``), so the CLI imports and prints help-even while some sub-packages are incomplete.+packages through function-local imports, so ``3dcode --help`` loads none of them.Owns the typer app and every registration, the run-starting commands ``make`` /``resume`` (spec building, reference grounding, track dispatch, budget raising) and@@ -55,7 +54,8 @@ def tools(as_json: Annotated[bool, typer.Option("--json-out", help="print the Observation as JSON")] = False,cards: Annotated[bool, typer.Option("--cards", help="(list) print prompt cards instead of a table")] = False,) -> None:- registry = C.lazy("codeverse3d.spatial.registry")+ from codeverse3d.spatial import registry+if name == "list":defs = registry.list_tools()if cards:@@ -114,8 +114,7 @@ def run_cmd("""Run every prompt of a battery through its track (N parallel workers); resumable."""if not battery.is_file():raise C.CliError(f"battery not found: {battery}")- b = C.import_bench()- run_bench = C.lazy("bench.run_bench")+ run_bench = C.import_bench("run_bench")# one source of truth for the worker count: BenchOptions.parallel (the measured knee)par = {"parallel": parallel} if parallel is not None else {}opts = run_bench.BenchOptions(generator=generator, planner=planner, judge=judge, rounds=rounds, max_minutes=max_minutes,@@ -131,17 +130,15 @@ def run_cmd(results = run_bench.run_battery(battery, out_dir, opts, on_result=_on)ok(f"{len(results)} results → {out_dir / 'results.csv'}")if report:- rep = C.lazy("bench.report").build_report(out_dir)+ rep = C.import_bench("report").build_report(out_dir)console.print(rep.markdown)ok(f"report → {out_dir / 'report.md'} / report.html")- del b@bench_app.command("report")def report_cmd(out_dir: Annotated[Path, typer.Argument()]) -> None:"""Aggregate results.jsonl → report.md + report.html (gallery of contact sheets)."""- C.import_bench()- rep = C.lazy("bench.report").build_report(out_dir)+ rep = C.import_bench("report").build_report(out_dir)console.print(rep.markdown)ok(f"report → {out_dir / 'report.md'} / report.html")@@ -150,7 +147,7 @@ def report_cmd(out_dir: Annotated[Path, typer.Argument()]) -> None:gallery_app = typer.Typer(no_args_is_help=True)RootsArg = Annotated[list[Path] | None, typer.Argument(- help="run roots (default: ./runs plus every ./bench/out/*/runs that exists)")]+ help="run roots (default: ./runs, every ./bench/out/*/runs and every <repo>/eval/bench/out/*/runs that exists)")]def resolve_roots(roots: list[Path] | None) -> list[Path]:@@ -163,13 +160,16 @@ def resolve_roots(roots: list[Path] | None) -> list[Path]:raise C.CliError("not a directory: " + ", ".join(str(m) for m in missing))return [Path(r) for r in roots]found = default_roots(Path.cwd())+ seen = {p.resolve() for p in found} # D77: the batteries live in <repo>/eval/bench/out/<b>/runs+ found += [p for p in sorted((C.EVAL_ROOT / "bench" / "out").glob("*/runs")) if p.is_dir() and p.resolve() not in seen]if not found:from codeverse3d.config import get_settingsfallback = Path(get_settings().runs_dir)if fallback.is_dir():return [fallback]- raise C.CliError(f"no run roots found under {Path.cwd()} (looked for runs/ and bench/out/*/runs); "+ raise C.CliError(f"no run roots found (looked for runs/ and bench/out/*/runs under {Path.cwd()}, "+ f"and {C.EVAL_ROOT / 'bench' / 'out'}/*/runs); "f"pass one explicitly: `3dcode gallery serve path/to/runs`")return found@@ -518,8 +518,8 @@ def _ground_in_reference(spec: Spec, ws, *, n_views: int) -> Spec:def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None, force: bool = False) -> None:from codeverse3d.cost.instrument import run_ledger+ from codeverse3d.tracks import get_track- get_track = C.lazy("codeverse3d.tracks", "get_track")options: dict = {"n_candidates": candidates} if candidates else {}options.update(C.round_policy_options(spec))try:diff --git a/harness/codeverse3d/cli/skills_cmd.py b/harness/codeverse3d/cli/skills_cmd.pyindex e115100f..82e757b8 100644--- a/harness/codeverse3d/cli/skills_cmd.py+++ b/harness/codeverse3d/cli/skills_cmd.py@@ -36,7 +36,7 @@ def list_skills(``target`` is the deterministic quantity the bundle says it moves (docs/SKILLS_LEDGER.md);``dir`` is the direction that counts as an improvement, and a ``?`` marks a bundle whoseclaim no deterministic instrument can see. Read them out with- ``python bench/skill_targets.py bench/out``.+ ``python eval/bench/skill_targets.py eval/bench/out``."""from rich.table import Tablediff --git a/harness/codeverse3d/cli/texture_cmd.py b/harness/codeverse3d/cli/texture_cmd.pyindex bbff933f..cb545e6e 100644--- a/harness/codeverse3d/cli/texture_cmd.py+++ b/harness/codeverse3d/cli/texture_cmd.py@@ -15,6 +15,7 @@ import typerfrom codeverse3d.cli import _common as Cfrom codeverse3d.cli._common import console, kv_table, warnfrom codeverse3d.contracts.common import Track+from codeverse3d.contracts.plan import ScenePlantexture_app = typer.Typer(name="texture", help="Text-to-image texturing: object pass + scene texture pack.",no_args_is_help=True)@@ -23,7 +24,9 @@ RunsDirOpt = C.RunsDirOptdef _image_model(name: str | None):- return C.lazy("codeverse3d.reference", "_image_model")(name or "")+ from codeverse3d import reference++ return reference._image_model(name or "")@texture_app.command("pass")@@ -41,10 +44,11 @@ def pass_(spec = C.load_spec(ws)if spec.track not in (Track.STATIC_OBJECT, Track.ARTICULATED_OBJECT):raise C.CliError(f"texture pass is for object tracks; {spec.track.value} runs use `3dcode texture scene-pack`")- load_plan = C.lazy("codeverse3d.spatial.tool_common", "load_plan")- plan = load_plan(ws.plan_path)- texture_pass = C.lazy("codeverse3d.texturing.run", "texture_pass")from codeverse3d.cost.instrument import run_ledger+ from codeverse3d.spatial.tool_common import load_plan+ from codeverse3d.texturing.run import texture_pass++ plan = load_plan(ws.plan_path)# a post-hoc pass joins the run's ledger when it has one, else the per-process log;# it rewrites the run's artifacts, so it holds the run mutex (one writer per run dir)@@ -59,8 +63,9 @@ def pass_(@texture_app.command("show")def show(slug: str, runs_dir: RunsDirOpt = None) -> None:"""Print the last texturing report of a run."""+ from codeverse3d.texturing.run import load_report+ws = C.open_workspace(slug, runs_dir)- load_report = C.lazy("codeverse3d.texturing.run", "load_report")try:rep = load_report(ws)except FileNotFoundError as e:@@ -69,7 +74,8 @@ def show(slug: str, runs_dir: RunsDirOpt = None) -> None:def _print_report(rep, ws) -> None:- plan_table = C.lazy("codeverse3d.texturing.plan", "plan_table")+ from codeverse3d.texturing.plan import plan_table+console.print(plan_table(rep.plan))rows = {tid: f"{Path(a.path).name} seam={a.seam_score:.3f} (raw {a.seam_score_raw:.3f}) cached={a.cached}"+ (f" ERROR {a.error}" if a.error else "") for tid, a in rep.textures.textures.items()}@@ -98,17 +104,16 @@ def scene_pack(runs_dir: RunsDirOpt = None,) -> None:"""Generate the scene's tileable texture pack into public/textures/ (+ manifest.json)."""+ from codeverse3d.cost.instrument import run_ledger+ from codeverse3d.spatial.tool_common import load_plan+ from codeverse3d.texturing.plan import scene_texture_pack, texture_pack_prompt+ws = C.open_workspace(slug, runs_dir)spec = C.load_spec(ws)- load_plan = C.lazy("codeverse3d.spatial.tool_common", "load_plan")plan = load_plan(ws.plan_path)- ScenePlan = C.lazy("codeverse3d.contracts.plan", "ScenePlan")if not isinstance(plan, ScenePlan):raise C.CliError("scene-pack needs a scene run (plan.json with zones)")- scene_texture_pack = C.lazy("codeverse3d.texturing.plan", "scene_texture_pack")- texture_pack_prompt = C.lazy("codeverse3d.texturing.plan", "texture_pack_prompt")model_id = spec.backends.planner if model is None else model- from codeverse3d.cost.instrument import run_ledger# like `pass`: the pack's plan + image spend joins the run's ledger when it has onewith (C.mutating(ws, what=f"3dcode texture scene-pack {ws.root.name}", action="texture"),diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.pyindex ae1bd225..67ce775f 100644--- a/harness/tests/flywheel_cli/test_cli.py+++ b/harness/tests/flywheel_cli/test_cli.py@@ -105,9 +105,9 @@ def test_tools_list_handles_an_optional_spatial_install():assert "Traceback" not in r.output-def test_lazy_import_message():+def test_a_missing_bench_module_is_a_clear_exit():with pytest.raises(C.CliError):- C.lazy("codeverse3d.definitely_missing_module")+ C.import_bench("definitely_missing_module")def test_doctor_json(monkeypatch):diff --git a/harness/tests/gallery/test_cli.py b/harness/tests/gallery/test_cli.pyindex 92a7779a..4c3e6a71 100644--- a/harness/tests/gallery/test_cli.py+++ b/harness/tests/gallery/test_cli.py@@ -48,14 +48,29 @@ def test_build_rejects_a_missing_root(tmp_path: Path):def test_default_roots_from_cwd(gallery_tree: dict[str, Path], monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", gallery_tree["root"] / "no_eval")monkeypatch.chdir(gallery_tree["root"])roots = resolve_roots(None)assert [r.name for r in roots] == ["runs", "runs"]assert str(roots[1]).endswith(os.path.join("bench", "out", "static_v9", "runs"))+def test_default_roots_find_the_eval_batteries_from_any_cwd(gallery_tree: dict[str, Path], tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch):+ """D77 moved the batteries to <repo>/eval/bench/out/<b>/runs; the defaults only looked+ under the cwd, so `3dcode gallery serve` from harness/ listed none of them."""+ battery = tmp_path / "eval" / "bench" / "out" / "b1" / "runs"+ battery.mkdir(parents=True)+ monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", tmp_path / "eval")+ monkeypatch.chdir(gallery_tree["root"])+ assert resolve_roots(None)[-1] == battery and len(resolve_roots(None)) == 3+ monkeypatch.chdir(tmp_path / "eval") # cwd IS the eval root: each battery listed once+ assert resolve_roots(None) == [Path.cwd() / "bench" / "out" / "b1" / "runs"]++def test_no_roots_anywhere_is_a_clear_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):monkeypatch.chdir(tmp_path)+ monkeypatch.setattr("codeverse3d.cli._common.EVAL_ROOT", tmp_path / "no_eval")monkeypatch.setenv("C3D_RUNS_DIR", str(tmp_path / "absent"))from codeverse3d.config import get_settingsdiff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.pyindex d2723053..fca4abdf 100644--- a/harness/tests/judges/test_calibration.py+++ b/harness/tests/judges/test_calibration.py@@ -102,3 +102,43 @@ def test_run_labels_stay_distinct_across_battery_layouts():assert len({a, b, c, d, e}) == 5, (a, b, c, d, e)# a plain harness run keeps its slug as the whole labelassert _run_label(Path("/home/u/proj/runs/chair_bl")) == "chair_bl"+++def test_calibration_judges_what_the_in_run_judge_saw(tmp_path, monkeypatch):+ """It rebuilt a thinner JudgeInput than `3dcode judge` (no previous verdict, track+ context, stored clay views or GLB for the D48 slices), re-rendered clay views a round+ had stored, and always used VlmJudge where `3dcode judge` picks ReferenceJudge."""+ import codeverse3d.addons.calibration as cal+ import codeverse3d.judges.vlm_judge as vj+ from codeverse3d.contracts.spec import ReferenceImage+ from tests.flywheel_cli.conftest import _judgment, make_fake_run, tiny_png++ ws, rec = make_fake_run(tmp_path / "runs")+ rec.spec = rec.spec.model_copy(update={"references": [ReferenceImage(path=str(tiny_png(ws.root / "ref.png")))]})+ ws.write_json(ws.record_path, rec)+ for r in rec.rounds:+ ws.write_json(ws.root / "rounds" / f"r{r.index:02d}.json", r)+ tiny_png(ws.renders_dir(1) / "clay" / "view_front.png")+ made, seen = [], []++ class _Vlm:+ def __init__(self, *a, **kw):+ made.append(type(self).__name__)+ self.model_id = kw["model_id"]++ def judge(self, inp):+ seen.append(inp)+ return _judgment(0.6, False, [])++ class _Ref(_Vlm):+ pass++ monkeypatch.setattr(vj, "VlmJudge", _Vlm)+ monkeypatch.setattr(vj, "ReferenceJudge", _Ref)+ monkeypatch.setattr(cal, "render_geometry_views", lambda *a, **k: pytest.fail("stored clay views re-rendered"))+ table = calibrate([ws.root], model_id="fake:fake-1", n_samples=1, out_dir=tmp_path / "out", max_workers=1)+ assert set(made) == {"_Ref"}+ r1 = next(inp for inp in seen if inp.round_index == 1)+ assert r1.previous is not None and r1.previous.overall == 0.55+ assert r1.geometry_views is not None and r1.glb_path == str(ws.artifacts / "object.glb")+ assert next(row for row in table.rows if row.round == 1).geometry_views