Official Codebase of 3DCodeVerse
Python
cost controls, corrected by measurement: revert the agent turn cap (A/B: +$0.02, -0.21 score) and the prompt reordering (net +tokens below the cache floor); keep the replay-verified regression/marginal stops (2 runs cut, $1.14, 0 best rounds lost); repair the judge-skip branches; fan_out propagates contextvars so parallel ledger rows attribute correctly; finalise texture pass is idempotent; honest profiles; 1245 tests, ruff clean
gaoypeng515f36f2d1e8
diff --git a/harness/CLAUDE.md b/harness/CLAUDE.mdindex a04721a5..2199fab4 100644--- a/harness/CLAUDE.md+++ b/harness/CLAUDE.md@@ -48,6 +48,8 @@ calibration) before changing anything.range). Judge default `gemini-3.1-pro-preview` (`Settings.default_judge`; tracks usen_samples=1); flash + `--n 3` is the cheap fallback, pro for calibration/eval.- Live runs under `runs/` (`e2e_*`): read-only reference material; never modify.+- Every model call is priced into the run's `telemetry/cost.jsonl` (`codeverse.cost.instrument`;+ `CV3D_COST_LEDGER=off` to disable). `docs/COST.md` Part II has the measured cost controls.## Commands```@@ -55,6 +57,7 @@ pip install -e /home/yipeng/3dcodeverse/harness # once (entry points: 3dcodevecd /home/yipeng/3dcodeverse/harness3dcodeverse doctor [--live] [--no-gpu] [--json]3dcodeverse make "a mid-century wooden dining chair" --track static_object --language blender+3dcodeverse make "..." --profile economy|balanced|quality # one dial: models, judge n, rounds, candidates, texture, ceilings (--profile == CV3D_PROFILE)3dcodeverse make "..." --track static_object --language threejs --generator gemini-cli:gemini-3.7-flash3dcodeverse make "..." --track articulated_object --language urdf_blender3dcodeverse make "..." --track scene --language scene_threejs --rounds 2 --max-usd 3@@ -63,6 +66,8 @@ cd /home/yipeng/3dcodeverse/harness3dcv resume <slug> · 3dcv status <slug> · 3dcv render <slug> [--mode wire] · 3dcv judge <slug> [--model ... --n 3]3dcv texture pass <slug> [--no-judge] · 3dcv texture scene-pack <slug> · 3dcv texture show <slug>3dcv tools list · 3dcv tools measure --workspace runs/<slug> · 3dcv mcp --workspace runs/<slug>+3dcv cost <slug> · 3dcv cost --runs-dir bench/out/<battery> · 3dcv cost cache <slug>+3dcv cost prices [--stale] · 3dcv cost profiles · 3dcv cost estimate gemini:gemini-3.1-pro-preview --in 120003dcv flywheel export runs/ dataset/ [--pack --drop-duplicates --captions-dir d/] · 3dcv flywheel pairs runs/ pairs.jsonl3dcv flywheel caption <slug> [--out dir] · 3dcv flywheel gallery runs/ gallery.html3dcv bench run bench/prompts/static_objects_v1.yaml --generator ... --judge gemini:gemini-3.1-pro-previewdiff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.pyindex c481a6d8..be51bb0c 100644--- a/harness/bench/compare_backends.py+++ b/harness/bench/compare_backends.py@@ -76,6 +76,7 @@ from codeverse.config import get_settings # noqa: E402from codeverse.contracts.artifacts import RenderSet # noqa: E402from codeverse.contracts.run import RunRecord # noqa: E402from codeverse.contracts.spec import Spec # noqa: E402+from codeverse.cost import run_ledger # noqa: E402from codeverse.tracks.generation import MultiFileParseError # noqa: E402from codeverse.workspace import Workspace # noqa: E402@@ -208,7 +209,11 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, deps: CompareDeps,if not resume:run_ws.create()run_ws.write_json(run_ws.spec_path, spec)- rec = deps.run_track(spec, run_ws, resume)+ # the harness arm is a real run: give it its own ledger, nested inside the cell's+ # (run_ledger restores the outer one on the way out, so the fixed evaluation that+ # follows keeps landing in the cell ledger)+ with run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}"):+ rec = deps.run_track(spec, run_ws, resume)res.gen_cost_usd = rec.total_usage.cost_usdres.tool_calls = rec.total_usage.tool_callsres.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), rec.final_score@@ -229,16 +234,20 @@ def run_cell(battery: Battery, item: BenchPrompt, arm: Arm, out: Path, opts: Comeval_ws = _fresh_ws(cell / "eval")eval_ws.write_json(eval_ws.spec_path, spec)try:- if arm.kind == "harness":- _run_harness(spec, cell, eval_ws, deps, res)- else:- _generate_oneshot(arm, spec, cell, eval_ws, opts, deps, res)- if (eval_ws.root / MODEL_FILE).is_file():- outcome = deps.evaluator.evaluate(eval_ws, spec)- eval_ws.write_json(eval_ws.root / "eval.json", outcome)- _fill_from_outcome(res, outcome)- else:- res.status, res.score, res.passed, res.build_ok = "no_code", 0.0, False, False+ # one ledger per cell: generation (one-shot arms) AND the fixed evaluation's+ # judge calls are priced into cells/<prompt>/<arm>/telemetry/cost.jsonl instead+ # of the per-process fallback log. Context-local, so --parallel keeps cells apart.+ with run_ledger(cell, run=f"{item.id}:{arm.slug}"):+ if arm.kind == "harness":+ _run_harness(spec, cell, eval_ws, deps, res)+ else:+ _generate_oneshot(arm, spec, cell, eval_ws, opts, deps, res)+ if (eval_ws.root / MODEL_FILE).is_file():+ outcome = deps.evaluator.evaluate(eval_ws, spec)+ eval_ws.write_json(eval_ws.root / "eval.json", outcome)+ _fill_from_outcome(res, outcome)+ else:+ res.status, res.score, res.passed, res.build_ok = "no_code", 0.0, False, Falseexcept Exception as e: # noqa: BLE001 — one cell must never kill the matrixres.status = "error"res.error = (res.error + "; " if res.error else "") + f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1200:]}"@@ -343,7 +352,9 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Armif on_result:on_result(r)if opts.pairwise:- run_pairwise(battery, done, arms, out, opts, deps)+ # the arena is battery-level spend: its own ledger at <out>/telemetry/cost.jsonl+ with run_ledger(out, run=f"{battery.name}:pairwise"):+ run_pairwise(battery, done, arms, out, opts, deps)build_compare_report(out)return list(done.values())diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.pyindex bb37c950..b5a35703 100644--- a/harness/bench/run_bench.py+++ b/harness/bench/run_bench.py@@ -9,6 +9,12 @@ Layout of ``out_dir``::Every prompt becomes a ``Spec`` with the battery's track/language, a FIXED judgemodel (methodology: paired runs share the judge), and the generator under test.++Each prompt runs inside its own ``codeverse.cost.run_ledger``, exactly like a+``3dcv make``: the batteries are where most runs come from, so without it the+priced per-call rows of a whole battery went to the per-process fallback log and+``3dcv cost --runs-dir <out>/runs`` had to reconstruct them from trajectories.+The binding is context-local, so ``--parallel N`` keeps N ledgers apart."""from __future__ import annotations@@ -30,6 +36,7 @@ from codeverse.config import get_settingsfrom codeverse.contracts.common import Backends, Budget, Language, Trackfrom codeverse.contracts.run import RunRecordfrom codeverse.contracts.spec import Constraints, Spec+from codeverse.cost import run_ledgerfrom codeverse.workspace import WorkspaceRESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_final", "passed", "rounds", "cost_usd",@@ -193,7 +200,8 @@ def run_battery(ws.write_json(ws.spec_path, spec)t0 = time.time()try:- rec = run_fn(spec, ws, resume)+ with run_ledger(ws.root, run=item.id):+ rec = run_fn(spec, ws, resume)except Exception as e: # one failing prompt must not kill the batteryreturn BenchItemResult(id=item.id, tier=item.tier, category=item.category, status="error",errors=f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}",diff --git a/harness/codeverse/cli/_common.py b/harness/codeverse/cli/_common.pyindex 8ad59d64..521ec8ca 100644--- a/harness/codeverse/cli/_common.py+++ b/harness/codeverse/cli/_common.py@@ -7,6 +7,7 @@ import hashlibimport importlibimport shutilimport sys+from dataclasses import dataclassfrom pathlib import Pathfrom typing import Any@@ -113,3 +114,118 @@ def echo_json(obj: Any) -> None:import jsonconsole.print_json(json.dumps(obj, default=str, ensure_ascii=False))+++# --------------------------------------------------------------------------- cost profile+def active_profile(settings: Any | None = None) -> Any:+ """The :class:`~codeverse.cost.profiles.Profile` the current settings name."""+ from codeverse.cost.profiles import get_profile++ return get_profile((settings or get_settings()).profile)+++@dataclass(frozen=True)+class ResolvedDial:+ """Every profile-controlled value, after the profile and the user's own+ settings/flags have been folded together. This is what a run actually gets."""++ profile: str+ generator: str+ planner: str+ judge: str+ captioner: str+ judge_samples: int+ judge_max_px: int+ judge_montages: int+ judge_detail_crops: int+ agent_max_turns: int+ rounds: int+ candidates: int+ texture: bool+ max_usd: float+ max_minutes: float+++def resolve_dial(+ settings: Any | None = None,+ profile_flag: str | None = None,+ *,+ rounds: int | None = None,+ candidates: int | None = None,+ max_usd: float | None = None,+ max_minutes: float | None = None,+ texture: bool = False,+) -> ResolvedDial:+ """The single resolver for the cost dial — used by ``3dcv make`` and by the tests.++ ``--profile X`` (``profile_flag``) *forces* the dial over anything the user+ stated in ``config.yaml`` / ``CV3D_*``; ``CV3D_PROFILE=X`` set the same dial+ as a *default* when :func:`codeverse.config.get_settings` built the settings.+ With nothing else stated the two paths therefore resolve **identically** —+ that equality is what ``tests/cost/test_profiles.py`` asserts, and it+ is the bug this function exists to prevent: the run shape (candidates, the+ texture pass) used to be read off the flag and so applied to only one path.++ An explicit CLI flag (``--rounds`` / ``--candidates`` / ``--max-usd`` /+ ``--max-minutes`` / ``--texture``) always wins over both.+ """+ settings = settings or get_settings()+ prof = settings.apply_profile(profile_flag, force=True) if profile_flag else active_profile(settings)+ return ResolvedDial(+ profile=prof.name,+ generator=settings.default_generator,+ planner=settings.default_planner,+ judge=settings.default_judge,+ captioner=settings.default_captioner,+ judge_samples=int(settings.judge.samples),+ judge_max_px=int(settings.judge.max_px),+ judge_montages=int(settings.judge.montages),+ judge_detail_crops=int(settings.judge.detail_crops),+ agent_max_turns=int(settings.limits.agent_max_turns),+ rounds=prof.rounds if rounds is None else int(rounds),+ candidates=int(settings.default_candidates if candidates is None else candidates),+ texture=bool(texture or prof.texture),+ max_usd=prof.max_usd if max_usd is None else float(max_usd),+ max_minutes=prof.max_minutes if max_minutes is None else float(max_minutes),+ )+++def round_policy_options(spec: Spec, settings: Any | None = None) -> dict[str, Any]:+ """``get_track(...)`` options the active profile implies.++ Only the judge sample count needs a ``RoundPolicy`` (rounds travel on+ ``spec.budget``, best-of-N on ``spec.options``), so a run at the default+ ``n=1`` gets **no** policy and keeps the track's own — including the+ rubric-derived stop target that ``BaseTrack.after_plan`` binds when no policy+ was injected. When a profile does ask for more samples we bind that target+ here instead, from the same rubric the track will use."""+ settings = settings or get_settings()+ samples = int(getattr(settings.judge, "samples", 1) or 1)+ if samples <= 1:+ return {}+ from dataclasses import replace++ from codeverse.orchestrator.rounds import RoundPolicy++ policy = RoundPolicy(max_rounds=spec.budget.max_rounds, judge_samples=samples)+ threshold = rubric_threshold(spec)+ if threshold is not None:+ policy = replace(policy, target=float(threshold))+ return {"policy": policy}+++def rubric_threshold(spec: Spec) -> float | None:+ """Pass threshold of the rubric this spec will be judged with (``None`` when+ the rubric cannot be loaded — the caller then keeps the policy default)."""+ try:+ from codeverse.contracts.common import Track+ from codeverse.judges.rubrics import load_rubric+ from codeverse.tracks import get_track+ from codeverse.tracks.lifecycle import REFERENCE_RUBRIC++ # same selection rule as BaseTrack.build_context+ name = (REFERENCE_RUBRIC if spec.references and spec.track is Track.STATIC_OBJECT+ else get_track(spec.track).rubric)+ return float(load_rubric(name).pass_threshold)+ except Exception: # noqa: BLE001 - a missing rubric must not stop a run+ return Nonediff --git a/harness/codeverse/cli/cost_cmd.py b/harness/codeverse/cli/cost_cmd.pyindex aa2d5341..d68548e1 100644--- a/harness/codeverse/cli/cost_cmd.py+++ b/harness/codeverse/cli/cost_cmd.py@@ -1,22 +1,65 @@"""``3dcv cost`` — what the runs cost and where the money went.- 3dcv cost runs/ # audit a tree of runs- 3dcv cost runs/e2e_chair_blender # one run, stage by stage- 3dcv cost bench/out/static_v1_flash --md report.md+ 3dcv cost <slug> # one run, from its own ledger+ 3dcv cost --runs-dir bench/out/x # a whole battery, aggregated+ 3dcv cost runs/ bench/out/* # any mix of run dirs / trees+ 3dcv cost show runs/ --md report.md # the same, explicit + full markdown3dcv cost runs/ --recheck # re-price with today's table (drift vs what was billed)- 3dcv cost prices # the price table + provenance+ 3dcv cost prices [--stale] [--unverified]+ 3dcv cost profiles # the economy / balanced / quality dial+ 3dcv cost estimate gemini:… --in 12000 --out 800++A run that wrote a live ledger (``telemetry/cost.jsonl``) is read from it — one+priced row per real model call. Older runs are reconstructed from their+trajectories, verdicts and events, so all 61 reference runs still audit."""from __future__ import annotations+from datetime import date, datetimefrom pathlib import Pathfrom typing import Annotated+import clickimport typer+from typer.core import TyperGroupfrom codeverse.cli._fmt import console, ok, warn-cost_app = typer.Typer(no_args_is_help=True)+#: a price row older than this needs re-checking against the provider's page+STALE_AFTER_DAYS = 90++class _CostGroup(TyperGroup):+ """``3dcv cost <slug|path> ...`` — a first argument that is not a subcommand+ is forwarded to ``show`` instead of failing with "No such command"."""++ def resolve_command(self, ctx: click.Context, args: list[str]): # type: ignore[override]+ try:+ return super().resolve_command(ctx, args)+ except click.UsageError:+ return "show", self.get_command(ctx, "show"), args+++cost_app = typer.Typer(no_args_is_help=True, invoke_without_command=True, cls=_CostGroup)+++@cost_app.callback()+def cost(+ ctx: typer.Context,+ runs_dir: Annotated[Path | None, typer.Option("--runs-dir", help="audit every run under this root")] = None,+ recheck: Annotated[bool, typer.Option("--recheck", help="re-price every call with today's table")] = False,+ md: Annotated[Path | None, typer.Option("--md", help="also write the full markdown report here")] = None,+) -> None:+ """Cost breakdown: per stage, per role, per model, waste, $ per passing artifact.++ ``3dcv cost <slug>`` reports one run from its own ledger; ``--runs-dir <dir>``+ aggregates every run under a root (a bench battery)."""+ if ctx.invoked_subcommand is not None:+ return+ if runs_dir is None:+ console.print(ctx.get_help())+ raise typer.Exit()+ _report([Path(runs_dir)], md=md, recheck=recheck, limit=20, per_run=True)@cost_app.command("show")@@ -26,8 +69,23 @@ def show(recheck: Annotated[bool, typer.Option("--recheck", help="re-price every call with today's table")] = False,limit: Annotated[int, typer.Option("--limit", help="rows in the per-run table")] = 20,per_run: Annotated[bool, typer.Option("--per-run/--no-per-run")] = True,+ runs_dir: Annotated[Path | None, typer.Option("--runs-dir", help="runs root a bare slug is resolved against")] = None,) -> None:- """Cost breakdown: per stage, per role, per model, waste, $ per passing artifact."""+ """Cost breakdown for runs named by slug or path (``3dcv cost <slug>`` lands here)."""+ _report([_resolve(str(p), runs_dir) for p in paths], md=md, recheck=recheck, limit=limit, per_run=per_run)+++def _resolve(arg: str, runs_dir: Path | None) -> Path:+ """A bare argument is a path when it exists, else a slug under the runs root."""+ from codeverse.cli import _common as C++ p = Path(arg)+ if p.exists():+ return p+ return C.runs_root(runs_dir) / arg+++def _report(paths: list[Path], *, md: Path | None, recheck: bool, limit: int, per_run: bool) -> None:from codeverse.cost import audit_runsfrom codeverse.cost.report import console as text_reportfrom codeverse.cost.report import markdown, runs_table@@ -36,9 +94,13 @@ def show(if not audit.runs:raise typer.BadParameter(f"no runs with a record.json under {', '.join(str(p) for p in paths)}")console.print(text_report(audit), soft_wrap=True)- if per_run:+ if len(audit.runs) == 1:+ _single_run_lines(audit.runs[0])+ elif per_run:console.print("")console.print(runs_table(audit, limit=limit), soft_wrap=True)+ live = sum(1 for r in audit.runs if r.source == "live")+ console.print(f"[dim]ledger source: {live} live / {len(audit.runs) - live} reconstructed[/dim]")if recheck:drift = audit.total_usd - audit.recorded_usdwarn(f"re-priced total ${audit.total_usd:.4f} vs recorded ${audit.recorded_usd:.4f} ({drift:+.4f})")@@ -51,33 +113,150 @@ def show(ok(f"wrote {md}")+def _single_run_lines(run: object) -> None:+ """Reconciliation for one run: the ledger against what the record was billed."""+ from codeverse.cli._fmt import kv_table++ ledger = float(getattr(run, "ledger_usd", 0.0))+ recorded = float(getattr(run, "recorded_usd", 0.0))+ drift = ledger - recorded+ pct = (100.0 * drift / recorded) if recorded else 0.0+ rows = {+ "source": getattr(run, "source", "?"),+ "ledger": f"${ledger:.4f} over {len(getattr(run, 'rows', []))} calls",+ "record.total_usage": f"${recorded:.4f}",+ "difference": f"${drift:+.4f} ({pct:+.2f}%)",+ "status": f"{getattr(run, 'status', '')} ({getattr(run, 'stop_reason', '')})",+ "budget": f"${float(getattr(run, 'budget_usd', 0.0)):.2f}",+ }+ console.print("")+ console.print(kv_table("reconciliation", rows))+ for note in getattr(run, "notes", []):+ warn(note)+++# --------------------------------------------------------------------------- prices@cost_app.command("prices")def prices(provider: Annotated[str | None, typer.Option("--provider", help="gemini | anthropic | openai")] = None,unverified: Annotated[bool, typer.Option("--unverified", help="only rows that are not verified")] = False,+ stale: Annotated[bool, typer.Option("--stale", help=f"only rows older than {STALE_AFTER_DAYS} days or approximate")] = False,+ days: Annotated[int, typer.Option("--days", help="staleness threshold in days")] = STALE_AFTER_DAYS,) -> None:- """The USD price table with its provenance (source + checked date + status)."""+ """The USD price table with its provenance (source + checked date + status).++ A row is flagged when it was last checked more than ``--days`` ago, when the+ price is marked approximate, or when its status is not ``verified`` — those+ are the dollars an audit cannot stand behind."""from codeverse.models.pricing import PRICES, price_provenance- rows = []+ rows, flagged = [], 0for (prov, model) in sorted(PRICES):if provider and prov != provider:continuerow = price_provenance(prov, model)if unverified and row.status == "verified":continue+ age = _age_days(row.checked)+ problems = _price_flags(row, age, days)+ if stale and not problems:+ continue+ flagged += bool(problems)p = row.priceassert p is not Nonerows.append((f"{prov}:{model}", f"{p.input:g}", f"{p.cached:g}", f"{p.output:g}",f"{p.image_usd:g}" if p.image_usd else "-", row.status, row.checked,- (row.provenance.note if row.provenance else "")[:60]))- header = ("model", "in/M", "cached/M", "out/M", "$/img", "status", "checked", "note")- widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))]+ "-" if age is None else str(age), ",".join(problems) or "-",+ (row.provenance.note if row.provenance else "")[:52]))+ header = ("model", "in/M", "cached/M", "out/M", "$/img", "status", "checked", "age_d", "flags", "note")+ widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))] if rows else \+ [len(h) for h in header]console.print(" ".join(h.ljust(w) for h, w in zip(header, widths, strict=True)), soft_wrap=True)for r in rows:console.print(" ".join(str(c).ljust(w) for c, w in zip(r, widths, strict=True)), soft_wrap=True)+ if flagged:+ warn(f"{flagged} of {len(rows)} row(s) flagged: re-check them against the provider's pricing page")+ elif rows:+ ok(f"{len(rows)} price row(s), none older than {days} days or approximate")+++def _price_flags(row: object, age: int | None, days: int) -> list[str]:+ flags = []+ if age is not None and age > days:+ flags.append(f"stale>{days}d")+ price = getattr(row, "price", None)+ if price is not None and getattr(price, "approximate", False):+ flags.append("approximate")+ status = getattr(row, "status", "")+ if status not in ("verified", ""):+ flags.append(status)+ return flags+++def _age_days(checked: str) -> int | None:+ try:+ return (date.today() - datetime.strptime(checked, "%Y-%m-%d").date()).days+ except (TypeError, ValueError):+ return None+++# --------------------------------------------------------------------------- cache+@cost_app.command("cache")+def cache(+ paths: Annotated[list[Path], typer.Argument(help="run slugs or run dirs")],+ runs_dir: Annotated[Path | None, typer.Option("--runs-dir")] = None,+) -> None:+ """Did prompt caching actually happen? Per session: the cold first call, the+ cached share, the dollars the cache saved and the dollars the cold head cost.++ Only a live ledger has per-call ``cached_tokens``; reconstructed runs show+ what their transcripts recorded."""+ from codeverse.cost.caching import session_cache+ from codeverse.cost.ledger import load_ledger+ from codeverse.cost.reconstruct import reconstruct_run++ for arg in paths:+ root = _resolve(str(arg), runs_dir)+ rows = load_ledger(root) or reconstruct_run(root).rows+ sessions = session_cache(rows)+ if not sessions:+ warn(f"{root}: no priced calls found")+ continue+ console.print(f"[bold]{root.name}[/bold] ({len(rows)} calls, {len(sessions)} sessions)")+ header = ("session", "calls", "first in", "first cached", "cached", "saved $", "cold $")+ table = [header] + [+ (s.key, str(s.n_calls), f"{s.first_input:,}", f"{s.first_cached:,}",+ f"{s.cached_fraction:.0%}", f"{s.saved_usd:.4f}", f"{s.cold_usd:.4f}") for s in sessions]+ widths = [max(len(r[i]) for r in table) for i in range(len(header))]+ for r in table:+ console.print(" " + " ".join(c.rjust(w) if i else c.ljust(w)+ for i, (c, w) in enumerate(zip(r, widths, strict=True))),+ soft_wrap=True)+ tot_in = sum(s.input_tokens for s in sessions)+ tot_cached = sum(s.cached_tokens for s in sessions)+ console.print(f" → {tot_cached:,} of {tot_in:,} input tokens cached ({tot_cached / tot_in if tot_in else 0:.0%}), "+ f"saved ${sum(s.saved_usd for s in sessions):.4f}, "+ f"cold heads cost ${sum(s.cold_usd for s in sessions):.4f}\n")+++# --------------------------------------------------------------------------- profiles+@cost_app.command("profiles")+def profiles() -> None:+ """The economy / balanced / quality dial and what each is measured to cost."""+ from codeverse.config import get_settings+ from codeverse.cost.profiles import profile_table++ header = ("profile", "generator", "judge", "shape", "payload", "$/run", "measured quality")+ rows = [tuple(str(c) for c in r) for r in profile_table()]+ widths = [max(len(r[i]) for r in [header, *rows]) for i in range(6)]+ for r in [header, *rows]:+ console.print(" ".join(c.ljust(w) for c, w in zip(r[:6], widths, strict=True)) + " " + r[6],+ soft_wrap=True)+ console.print(f"\n[dim]active profile: {get_settings().profile} · "+ f"`3dcv make ... --profile quality` to switch one run[/dim]")+# --------------------------------------------------------------------------- estimate@cost_app.command("estimate")def estimate(model: Annotated[str, typer.Argument(help="model id, e.g. gemini:gemini-3.1-pro-preview")],diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.pyindex aad46fa6..bdc6e81e 100644--- a/harness/codeverse/cli/main.py+++ b/harness/codeverse/cli/main.py@@ -62,9 +62,12 @@ def make(judge: Annotated[str | None, typer.Option()] = None,captioner: Annotated[str | None, typer.Option()] = None,image: Annotated[list[Path] | None, typer.Option("--image", help="reference image(s)")] = None,- rounds: Annotated[int, typer.Option("--rounds", min=0)] = 4,- max_usd: Annotated[float, typer.Option("--max-usd")] = 5.0,- max_minutes: Annotated[float, typer.Option("--max-minutes")] = 60.0,+ profile: Annotated[str | None, typer.Option("--profile", help="cost/quality dial: economy | balanced | quality "+ "(sets models, judge samples, rounds, candidates, turn cap, "+ "montage size and the texture pass together)")] = None,+ rounds: Annotated[int | None, typer.Option("--rounds", min=0, help="refine rounds after the baseline (default: the profile's)")] = None,+ max_usd: Annotated[float | None, typer.Option("--max-usd")] = None,+ max_minutes: Annotated[float | None, typer.Option("--max-minutes")] = None,candidates: Annotated[int | None, typer.Option("--candidates", min=1, help="best-of-N baseline: N parallel candidates, keep the best (default: settings.default_candidates or 1)")] = None,slug: Annotated[str | None, typer.Option("--slug")] = None,runs_dir: RunsDirOpt = None,@@ -86,7 +89,14 @@ def make(if language is None:language = TRACK_LANGUAGES[track][0]run_slug = C.make_slug(prompt, track.value, language.value, slug)- backends = get_settings().backends(generator=generator, planner=planner, judge=judge, captioner=captioner)+ settings = get_settings()+ # ONE resolver for the whole dial, so `--profile X` and `CV3D_PROFILE=X` land the same+ # values (they used to disagree on candidates + texture); an explicit flag beats both.+ dial = C.resolve_dial(settings, profile, rounds=rounds, candidates=candidates,+ max_usd=max_usd, max_minutes=max_minutes, texture=texture)+ rounds, max_usd, max_minutes = dial.rounds, dial.max_usd, dial.max_minutes+ candidates, texture = dial.candidates, dial.texture+ backends = settings.backends(generator=generator, planner=planner, judge=judge, captioner=captioner)# validate the whole Spec BEFORE touching the filesystem: an invalid# track/language combination must not leave an orphan run directory behind.try:@@ -96,7 +106,10 @@ def make(constraints=Constraints(dimensions_m=C.parse_kv_floats(dim, "--dim") or None, must_have=must,must_not=must_not, style=style),budget=Budget(max_rounds=rounds, max_usd=max_usd, max_minutes=max_minutes),- backends=backends, options=RunOptions(candidates=candidates, texture=texture),+ # options.profile records the dial this run resolved to, whichever way it was+ # named (flag, CV3D_PROFILE, config.yaml), so `3dcv resume` re-applies it+ backends=backends, options=RunOptions(candidates=candidates, texture=texture,+ profile=dial.profile),seed=seed, tags=tag,)except ValueError as e:@@ -106,7 +119,11 @@ def make(ws.commit("spec")console.print(kv_table("run", {"slug": run_slug, "workspace": ws.root, "track": track.value,"language": language.value, "generator": backends.generator,- "candidates": candidates or "default"}))+ "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",+ "judge": f"{backends.judge} n={dial.judge_samples} "+ f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",+ "rounds": rounds, "max_usd": max_usd,+ "candidates": candidates, "texture": texture}))if no_run:ok(f"spec written: {ws.spec_path} (not run; `3dcv resume {run_slug}` to start)")return@@ -114,10 +131,15 @@ def make(def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None) -> None:+ from codeverse.cost.instrument import run_ledger+get_track = C.lazy("codeverse.tracks", "get_track")- options = {"n_candidates": candidates} if candidates else {}+ options: dict = {"n_candidates": candidates} if candidates else {}+ options.update(C.round_policy_options(spec))try:- record = get_track(spec.track, **options).run(spec, ws, resume=resume)+ # every model call and agent session of this run lands in telemetry/cost.jsonl+ with run_ledger(ws.root, run=ws.root.name):+ record = get_track(spec.track, **options).run(spec, ws, resume=resume)except KeyboardInterrupt:raise C.CliError(f"interrupted; resume with `3dcv resume {ws.root.name}`", code=130) from Noneexcept Exception as e: # the track failed outside its own error handling@@ -143,6 +165,8 @@ def resume(first — the only way to continue a BUDGET-stopped run."""ws = C.open_workspace(slug, runs_dir)spec = C.load_spec(ws)+ if spec.options.profile: # the dial the run was created with (judge samples, montage px, turn cap)+ get_settings().apply_profile(spec.options.profile, force=True)raised = {k: v for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items() if v is not None}if raised:from codeverse.events import EventLog@@ -279,8 +303,13 @@ def judge(inp = J.build_judge_input(ws, rec, rnd)judge_obj = J.make_judge(rec, rubric_name, model or rec.spec.backends.judge, n)n_images = J.count_prompt_images(inp, rubric_name)+ from codeverse.cost.instrument import run_ledger+try:- verdict = judge_obj.judge(inp)+ # a re-judge joins the run's ledger when it has one; otherwise the per-process log+ # (a ledger holding only this verdict would be read as the whole run's cost)+ with run_ledger(ws.root, run=ws.root.name, create=False):+ verdict = judge_obj.judge(inp)except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothingraise C.CliError(f"judge failed: {e}") from eexcept Exception as e:diff --git a/harness/codeverse/cli/texture_cmd.py b/harness/codeverse/cli/texture_cmd.pyindex 69267eb1..5b12044d 100644--- a/harness/codeverse/cli/texture_cmd.py+++ b/harness/codeverse/cli/texture_cmd.py@@ -45,8 +45,13 @@ def pass_(load_plan = C.lazy("codeverse.spatial.tool_common", "load_plan")plan = load_plan(ws.plan_path)texture_pass = C.lazy("codeverse.texturing.run", "texture_pass")- rep = texture_pass(ws, spec, plan, model_id=model or spec.backends.planner, image_model=_image_model(image_model),- judge=judge, judge_model_id=judge_model, size=size)+ from codeverse.cost.instrument import run_ledger++ # a post-hoc pass joins the run's ledger when it has one, else the per-process log+ with run_ledger(ws.root, run=ws.root.name, create=False):+ rep = texture_pass(ws, spec, plan, model_id=model or spec.backends.planner,+ image_model=_image_model(image_model), judge=judge,+ judge_model_id=judge_model, size=size)_print_report(rep)diff --git a/harness/codeverse/config.py b/harness/codeverse/config.pyindex 3f23f969..71de10a8 100644--- a/harness/codeverse/config.py+++ b/harness/codeverse/config.py@@ -15,6 +15,7 @@ import reimport shutilfrom functools import lru_cachefrom pathlib import Path+from typing import Anyimport yamlfrom pydantic import BaseModel, Field@@ -53,6 +54,21 @@ class Limits(BaseModel):bpy_rlimit_gb: int = 12max_parallel_agents: int = 6max_parallel_builds: int = 3+ agent_max_turns: int = Field(+ default=0,+ description="hard cap on model turns per agent session (0 = the backend's own default, "+ "which is what ships: a 28-turn cap cost $0.02 more and 0.205 of a score point in its "+ "own A/B — docs/COST.md §17 — so no profile sets one; name it here if you want one).",+ )+++class Judge(BaseModel):+ """Judge payload size — what one verdict is allowed to send (docs/COST.md §3)."""++ max_px: int = Field(default=1024, description="longest edge of a montage / crop sent to the judge")+ montages: int = Field(default=3, description="max 2x2 montages per verdict")+ detail_crops: int = Field(default=2, description="max zoomed detail crops per verdict")+ samples: int = Field(default=1, description="default VLM samples per verdict")class Settings(BaseSettings):@@ -63,6 +79,7 @@ class Settings(BaseSettings):binaries: Binaries = Field(default_factory=Binaries)render: Render = Field(default_factory=Render)limits: Limits = Field(default_factory=Limits)+ judge: Judge = Field(default_factory=Judge)gemini_api_keys: list[str] = Field(default_factory=list)anthropic_api_key: str = ""@@ -77,6 +94,12 @@ class Settings(BaseSettings):default_captioner: str = Field(default_factory=lambda: Backends().captioner)default_candidates: int = Field(default=1, description="best-of-N baseline candidates (tracks read it)")+ # Cost dial: one name that sets model-per-role, judge samples, rounds, candidates,+ # turn cap, montage size and the texture pass together (codeverse/cost/profiles.py).+ profile: str = Field(default="balanced", description="economy | balanced | quality")+ cost_ledger: bool = Field(default=True, description="append one priced row per model call "+ "to the run's telemetry/cost.jsonl (or a per-process log)")+# ------------------------------------------------------------------ helpersdef backends(self, **overrides: str | None) -> Backends:"""The ``Backends`` for a new Spec from the settings defaults. Keyword@@ -96,6 +119,37 @@ class Settings(BaseSettings):base[role] = valuereturn Backends(**base)+ def apply_profile(self, name: str | None = None, *, force: bool = False) -> Any:+ """Set every profile-controlled default on this Settings object and return+ the :class:`~codeverse.cost.profiles.Profile`.++ A field the user stated themselves (config file / ``CV3D_*`` env) is left+ alone unless ``force`` — so a profile is a *default* dial, while+ ``3dcv make --profile X`` (which forces) is an instruction. The CLI's own+ flags are applied after this and always win."""+ from codeverse.cost.profiles import get_profile++ p = get_profile(name or self.profile)+ stated = set() if force else set(self.model_fields_set)++ def put(field: str, value: Any) -> None:+ if field not in stated and value not in (None, ""):+ setattr(self, field, value)++ self.profile = p.name+ put("default_generator", p.generator)+ put("default_planner", p.planner)+ put("default_judge", p.judge)+ put("default_captioner", p.captioner)+ put("default_candidates", p.candidates)+ if "judge" not in stated:+ self.judge = self.judge.model_copy(update={+ "max_px": p.judge_max_px, "montages": p.judge_montages,+ "detail_crops": p.judge_detail_crops, "samples": p.judge_samples})+ if "limits" not in stated:+ self.limits = self.limits.model_copy(update={"agent_max_turns": p.max_turns})+ return p+def resolve_blender(self) -> str:if self.binaries.blender and Path(self.binaries.blender).exists():return self.binaries.blender@@ -152,4 +206,8 @@ def get_settings() -> Settings:# de-dupe keys preserving orderseen: set[str] = set()s.gemini_api_keys = [k for k in s.gemini_api_keys if not (k in seen or seen.add(k))]+ # the dial named in config.yaml / CV3D_PROFILE, applied as a *default*: a value the+ # user stated themselves survives it. `3dcv make --profile X` forces the same dial+ # (codeverse.cli._common.resolve_dial is the one resolver both paths go through).+ s.apply_profile(s.profile)return sdiff --git a/harness/codeverse/contracts/spec.py b/harness/codeverse/contracts/spec.pyindex f1bcddae..d5281a3d 100644--- a/harness/codeverse/contracts/spec.py+++ b/harness/codeverse/contracts/spec.py@@ -37,6 +37,9 @@ class RunOptions(BaseModel):candidates: int | None = Field(default=None, ge=1, description="best-of-N baseline candidates")texture: bool = Field(default=False, description="run the derived texture pass after the loop")+ profile: str = Field(default="", description="cost/quality dial this run was created with "+ "(codeverse/cost/profiles.py); '' = the settings default, and a resume "+ "re-applies it so judge samples / montage size / turn cap survive")class Spec(BaseModel):diff --git a/harness/codeverse/cost/__init__.py b/harness/codeverse/cost/__init__.pyindex 536ce667..a6ef3d72 100644--- a/harness/codeverse/cost/__init__.py+++ b/harness/codeverse/cost/__init__.py@@ -1,7 +1,8 @@"""Cost accounting: a typed ledger of every model call, a price-provenance layer-on top of ``models/pricing.py``, an audit over recorded runs, and the two cheap-helpers that keep the bill down (cache-friendly prompt ordering, a pre-send cost-guard).+on top of ``models/pricing.py``, an audit over recorded runs, a pre-send cost+guard, and the prompt-prefix measurement helpers (``Block`` / ``order_blocks`` /+``prefix_report`` — measurement only: reordering the generation prompts for the+cache was measured and reverted, see ``docs/COST.md`` §13).Public API::@@ -26,6 +27,7 @@ from codeverse.cost.caching import (prefix_signature,render_blocks,)+from codeverse.cost.context import CallContext, bind_run, call_contextfrom codeverse.cost.guard import (CostEstimate,CostGuard,@@ -33,15 +35,25 @@ from codeverse.cost.guard import (estimate_call,text_tokens,)+from codeverse.cost.instrument import (+ MeteredAgent,+ MeteredChatModel,+ metered_agent,+ metered_chat_model,+ per_call_metering,+ run_ledger,+)from codeverse.cost.ledger import (CostLedger,ledger_path,load_ledger,+ open_run_ledger,price_call,record_call,set_default_ledger,summarise,)+from codeverse.cost.profiles import PROFILES, Profile, get_profilefrom codeverse.cost.reconstruct import RunLedger, find_runs, reconstruct, reconstruct_runfrom codeverse.cost.report import console, markdownfrom codeverse.cost.routing import (@@ -54,11 +66,56 @@ from codeverse.cost.routing import (from codeverse.cost.types import CallCost, CostBucket, Role, Stage, Summary__all__ = [- "ROUTES", "Audit", "Block", "CallCost", "CostBucket", "CostEstimate", "CostGuard", "CostLedger",- "PrefixReport", "Role", "Route", "RunLedger", "Stage", "Summary", "WasteItem", "audit_dir",- "audit_runs", "cache_efficiency", "cheapest_affordable", "console", "default_route",- "estimate_call", "find_runs", "ledger_path", "load_ledger", "markdown", "order_blocks",- "prefix_report", "prefix_signature", "price_call", "pro_break_even", "reconstruct",- "reconstruct_run", "record_call", "render_blocks", "samples_for_precision",- "set_default_ledger", "summarise", "text_tokens",+ "Audit",+ "Block",+ "CallContext",+ "CallCost",+ "CostBucket",+ "CostEstimate",+ "CostGuard",+ "CostLedger",+ "MeteredAgent",+ "MeteredChatModel",+ "PROFILES",+ "PrefixReport",+ "Profile",+ "ROUTES",+ "Role",+ "Route",+ "RunLedger",+ "Stage",+ "Summary",+ "WasteItem",+ "audit_dir",+ "audit_runs",+ "bind_run",+ "cache_efficiency",+ "call_context",+ "cheapest_affordable",+ "console",+ "default_route",+ "estimate_call",+ "find_runs",+ "get_profile",+ "ledger_path",+ "load_ledger",+ "markdown",+ "metered_agent",+ "metered_chat_model",+ "open_run_ledger",+ "order_blocks",+ "per_call_metering",+ "prefix_report",+ "prefix_signature",+ "price_call",+ "pro_break_even",+ "reconstruct",+ "reconstruct_run",+ "record_call",+ "render_blocks",+ "run_ledger",+ "samples_for_precision",+ "set_default_ledger",+ "summarise",+ "text_tokens",]diff --git a/harness/codeverse/cost/caching.py b/harness/codeverse/cost/caching.pyindex 67ec46d0..9c5b8117 100644--- a/harness/codeverse/cost/caching.py+++ b/harness/codeverse/cost/caching.py@@ -17,6 +17,32 @@ Usage:::func:`prefix_report` measures a real sequence of prompts: how many tokens theyshare, and how many tokens each call pays full price for because of divergence.++**Measured, and NOT applied to the generation prompts (2026-08-23).** Wave 2+reordered ``generate_static`` / ``generate_articulated`` / ``refine_object``+stable-prefix-first, by hoisting a shared head into every one of them. Measured+on the real templates (``prefix_report`` over one baseline + three refines):++=============================== ============================== ===============+ per-prompt tokens mean/call+=============================== ============================== ===============+task-first (shipped, restored) 5,072 / 1,325 / 1,325 / 1,325 2,261+stable-head-first (reverted) 5,110 / 5,211 / 5,211 / 5,211 5,186+=============================== ============================== ===============++The head was **duplicated** into every prompt rather than shared with anything+already being sent, so the run-level change is +2,925 tokens on the average call+(+3,886 on every refine) — and the prompt is re-sent on every turn of the+session, so it is that figure times the turn count. What it bought was a 4,970+token shared prefix, below the ~12k floor §4 of ``docs/COST.md`` measured for+Gemini's implicit cache, so there is no cache read to set against it. The one+prefix that is genuinely shared without duplication — the api-agent's system+prompt (``AGENTS.md``, 4,879 tokens) plus its tool declarations (1,691) — already+sits first in every request and totals **6,570 tokens**, also below the floor.++So: keep these helpers for measuring and for any future prompt family that is+big enough, but do not reorder a prompt for the cache without measuring the+prefix against the floor first."""from __future__ import annotations@@ -123,3 +149,77 @@ def cache_efficiency(usages: Sequence[object]) -> tuple[int, int, float]:cached = sum(int(getattr(u, "cached_tokens", 0)) for u in usages)total = sum(int(getattr(u, "input_tokens", 0)) for u in usages)return cached, total, (cached / total if total else 0.0)+++# --------------------------------------------------------------------------- ledger view+@dataclass+class SessionCache:+ """Cache behaviour of ONE agent session (or one-shot call) as the ledger saw it."""++ key: str+ n_calls: int = 0+ first_input: int = 0+ first_cached: int = 0+ input_tokens: int = 0+ cached_tokens: int = 0+ cost_usd: float = 0.0+ price_input: float = 0.0+ price_cached: float = 0.0++ @property+ def cached_fraction(self) -> float:+ return self.cached_tokens / self.input_tokens if self.input_tokens else 0.0++ @property+ def first_cached_fraction(self) -> float:+ return self.first_cached / self.first_input if self.first_input else 0.0++ @property+ def saved_usd(self) -> float:+ """USD the cache reads saved against paying full input price for them."""+ return self.cached_tokens * max(0.0, self.price_input - self.price_cached) / 1e6++ @property+ def cold_usd(self) -> float:+ """USD still paid at full price for the uncached head of the session."""+ return self.first_input * self.price_input / 1e6 if not self.first_cached else 0.0+++def session_key(label: str) -> str:+ """Collapse a per-call label to its session: ``api-agent:refine_legs:t7`` →+ ``refine_legs``, ``judge:static_object_v1:r02:s1`` → ``judge:static_object_v1``.++ A session is what shares a prompt prefix: every turn of one agent session,+ every sample of one verdict."""+ parts = [p for p in (label or "").split(":") if p]+ if parts and parts[0] == "api-agent":+ parts = parts[1:]+ while parts and len(parts[-1]) > 1 and parts[-1][0] in "tsr" and parts[-1][1:].isdigit():+ parts = parts[:-1]+ return ":".join(parts)+++def session_cache(rows: Sequence[object], *, key: str = "label") -> list[SessionCache]:+ """Per-session cache behaviour from ledger rows (``codeverse.cost.CallCost``).++ Sessions are ``(round, session_key(<key>))`` groups in ledger order, so the first row of a+ group is that session's cold turn — the one the audit shows pays full price+ (docs/COST.md §4). Use it to check whether a prompt-ordering change actually+ moved ``cached_tokens``, not just the prompt."""+ out: dict[tuple[object, str], SessionCache] = {}+ order: list[tuple[object, str]] = []+ for row in rows:+ k = (getattr(row, "round", None), session_key(str(getattr(row, key, "") or "")))+ s = out.get(k)+ if s is None:+ s = out[k] = SessionCache(key=f"r{k[0] if k[0] is not None else '-'}:{k[1]}",+ first_input=int(getattr(row, "input_tokens", 0)),+ first_cached=int(getattr(row, "cached_tokens", 0)),+ price_input=float(getattr(row, "price_input", 0.0)),+ price_cached=float(getattr(row, "price_cached", 0.0)))+ order.append(k)+ s.n_calls += 1+ s.input_tokens += int(getattr(row, "input_tokens", 0))+ s.cached_tokens += min(int(getattr(row, "cached_tokens", 0)), int(getattr(row, "input_tokens", 0)))+ s.cost_usd += float(getattr(row, "cost_usd", 0.0))+ return [out[k] for k in order]diff --git a/harness/codeverse/cost/context.py b/harness/codeverse/cost/context.pynew file mode 100644index 00000000..b56f3cdd--- /dev/null+++ b/harness/codeverse/cost/context.py@@ -0,0 +1,230 @@+"""Where a call sits in a run — the attribution the ledger needs.++A :class:`CallContext` answers "which run, which round, which stage, which+role" for the call that is about to happen. Precedence:++1. what the caller passed explicitly (a judge knows it is judging) — always wins;+2. a label that names a **job of its own** (``judge:…``, ``planner``,+ ``texture_gate``, ``caption…``, ``pairwise:…`` — :data:`SELF_DESCRIBING`).+ Such a call is not the work of whatever agent session happens to surround it,+ so it outranks the ambient context. This is the fix for a measured+ mis-attribution: an in-process tool that bills a model *inside* an agent+ session (a texture pass, a captioner, a judge invoked as a tool) was filed+ under the session's ``stage=refine / role=generator``;+3. the ambient context (:func:`call_context`) — set by+ :mod:`codeverse.cost.instrument` around an agent session. It beats a+ *generation* label because the session knows more than one of its turns does:+ a best-of-N candidate runs with ``job.kind="candidate"`` while its turns are+ labelled ``api-agent:baseline:t3``;+4. what is left of the label (a generation stage, the round tag) — which also+ survives a thread hop that a ``ContextVar`` does not.++A label that names nothing (:data:`~codeverse.cost.types.Stage.OTHER`) states+nothing and never displaces anything.++The **run binding** (:func:`bind_run`, the run slug) is a ``ContextVar`` with a+process-wide fallback: a nested run (a bench worker that opens its own ledger in+its own thread) gets its own binding, while a plain worker thread that inherits+nothing — ``codeverse.fanout`` does not copy context — still sees the process's+one run.+"""++from __future__ import annotations++import threading+from collections.abc import Iterator+from contextlib import contextmanager+from contextvars import ContextVar+from dataclasses import dataclass, replace++from codeverse.cost.types import Role, Stage, role_for_stage, stage_for_label+++@dataclass(frozen=True)+class CallContext:+ """Attribution for one call (all fields optional; ``None`` = "not stated")."""++ run: str = ""+ round: int | None = None+ stage: Stage | None = None+ role: Role | None = None+ label: str = ""++ def merge(self, other: CallContext) -> CallContext:+ """``other`` wins field by field where it says something."""+ return CallContext(+ run=other.run or self.run,+ round=self.round if other.round is None else other.round,+ stage=other.stage or self.stage,+ role=other.role or self.role,+ label=other.label or self.label,+ )++ def resolved(self) -> CallContext:+ """Fill role from stage (and stage from label) so the row is never ``other``+ when something better can be derived."""+ stage = self.stage or (stage_for_label(self.label) if self.label else None)+ role = self.role or (role_for_stage(stage) if stage else None)+ return replace(self, stage=stage, role=role)+++_EMPTY = CallContext()+_ctx: ContextVar[CallContext] = ContextVar("cv3d_cost_ctx", default=_EMPTY)++#: the run this execution context belongs to ("" = nothing bound here)+_run_var: ContextVar[str] = ContextVar("cv3d_cost_run", default="")++_binding_lock = threading.Lock()+_run_binding: CallContext = _EMPTY+++def bind_run(run: str) -> None:+ """Name the run every subsequent row belongs to.++ Sets both the context-local binding (so a bench worker that runs its own+ prompt in its own thread cannot have its rows stolen by a sibling) and the+ process-wide fallback (so a worker thread that inherited no context — the+ fan-out pools — still finds the run)."""+ global _run_binding+ _run_var.set(run or "")+ with _binding_lock:+ _run_binding = CallContext(run=run)+++def run_binding() -> CallContext:+ """The context-local run if this thread bound one, else the process's."""+ local = _run_var.get()+ if local:+ return CallContext(run=local)+ with _binding_lock:+ return _run_binding+++def current() -> CallContext:+ """Run binding merged with the ambient (thread-local) context."""+ return run_binding().merge(_ctx.get())+++@contextmanager+def call_context(+ *,+ run: str = "",+ round: int | None = None, # noqa: A002 - matches the record/round vocabulary+ stage: Stage | str | None = None,+ role: Role | str | None = None,+ label: str = "",+) -> Iterator[CallContext]:+ """Set the ambient attribution for the duration of the block (this thread).++ Used by :mod:`codeverse.cost.instrument` around an agent session and by the+ judges around a verdict; nesting merges, so an inner block only has to name+ what it changes."""+ ctx = _ctx.get().merge(CallContext(run=run, round=round, stage=_stage(stage), role=_role(role), label=label))+ token = _ctx.set(ctx)+ try:+ yield ctx+ finally:+ _ctx.reset(token)+++def _stage(value: Stage | str | None) -> Stage | None:+ if value is None or isinstance(value, Stage):+ return value+ try:+ return Stage(value)+ except ValueError:+ return stage_for_label(str(value))+++def _role(value: Role | str | None) -> Role | None:+ if value is None or isinstance(value, Role):+ return value+ try:+ return Role(value)+ except ValueError:+ return None+++# --------------------------------------------------------------------------- labels+#: request-label prefix → (stage, role). Longest prefix wins; anything else is+#: routed through ``stage_for_label`` (which understands generation labels).+_LABEL_HINTS: tuple[tuple[str, Stage, Role], ...] = (+ ("planner", Stage.PLAN, Role.PLANNER),+ ("plan", Stage.PLAN, Role.PLANNER),+ ("judge", Stage.JUDGE, Role.JUDGE),+ ("pairwise", Stage.PAIRWISE, Role.JUDGE),+ ("texture_gate", Stage.TEXTURE, Role.JUDGE),+ ("texture", Stage.TEXTURE, Role.OTHER),+ ("scene_texture", Stage.TEXTURE, Role.OTHER),+ ("caption", Stage.CAPTION, Role.CAPTIONER),+ ("calibrate", Stage.JUDGE, Role.JUDGE),+)+++def context_from_label(label: str) -> CallContext:+ """Best-effort attribution from a ``ChatRequest.label``.++ Understands the api-agent's ``api-agent:<job label>:t<turn>`` form (the job+ label carries the stage) and the judge's ``…:r<NN>:s<k>`` round tag."""+ low = (label or "").strip()+ if not low:+ return _EMPTY+ round_index = _round_from_label(low)+ body = low+ if low.startswith("api-agent:"):+ parts = low.split(":")+ body = parts[1] if len(parts) > 1 else ""+ return CallContext(round=round_index, stage=_stated(stage_for_label(body)),+ role=Role.GENERATOR, label=low)+ head = body.split(":", 1)[0].lower()+ best: tuple[int, Stage, Role] | None = None+ for prefix, stage, role in _LABEL_HINTS:+ if head.startswith(prefix) and (best is None or len(prefix) > best[0]):+ best = (len(prefix), stage, role)+ if best is not None:+ return CallContext(round=round_index, stage=best[1], role=best[2], label=low)+ return CallContext(round=round_index, stage=_stated(stage_for_label(head)), label=low)+++def _stated(stage: Stage | None) -> Stage | None:+ """``Stage.OTHER`` is "the label told us nothing" — not a statement, so it must+ not outrank the ambient session context."""+ return None if stage is Stage.OTHER else stage+++def _round_from_label(label: str) -> int | None:+ """``judge:static_object_v1:r02:s1`` → 2 (``None`` when the label has no round tag)."""+ for token in label.replace("/", ":").split(":"):+ t = token.strip()+ if len(t) >= 2 and t[0] == "r" and t[1:].isdigit():+ return int(t[1:])+ return None+++#: Stages a label can name that describe a job of their own. A call whose label+#: says "judge" / "plan" / "texture" / "caption" / "pairwise" is doing that job+#: even when an agent session surrounds it, so it outranks the ambient context;+#: a generation stage in a label does not (see the module docstring).+SELF_DESCRIBING: frozenset[Stage] = frozenset(+ {Stage.PLAN, Stage.JUDGE, Stage.PAIRWISE, Stage.TEXTURE, Stage.CAPTION})+++def attribute(explicit: CallContext | None = None, *, label: str = "") -> CallContext:+ """The final attribution for one call.++ Precedence (see the module docstring): explicit > a label that names a job of+ its own > the ambient session context > the rest of the label. The row's+ ``label`` is always the most specific one available."""+ ctx = current()+ if label:+ from_label = context_from_label(label)+ ctx = ctx.merge(from_label) if from_label.stage in SELF_DESCRIBING else from_label.merge(ctx)+ if explicit is not None:+ ctx = ctx.merge(explicit)+ if label and not (explicit is not None and explicit.label):+ ctx = replace(ctx, label=label)+ return ctx.resolved()+++__all__ = ["CallContext", "attribute", "bind_run", "call_context", "context_from_label", "current",+ "run_binding"]diff --git a/harness/codeverse/cost/instrument.py b/harness/codeverse/cost/instrument.pynew file mode 100644index 00000000..35cf35e4--- /dev/null+++ b/harness/codeverse/cost/instrument.py@@ -0,0 +1,348 @@+"""Meter every model call and every agent session into the ledger.++Two thin proxies do the whole job:++* :class:`MeteredChatModel` wraps a :class:`~codeverse.models.base.ChatModel`.+ Every ``generate`` appends one :class:`~codeverse.cost.types.CallCost` row+ (tokens, the unit prices actually used + their provenance, $, latency,+ cache hit, outcome). Because ``models.registry.get_chat_model`` returns the+ proxy, this covers the planner, the judges, the captioner, single-shot+ generation **and every turn of the in-process api-agent**.+* :class:`MeteredAgent` wraps a :class:`~codeverse.agents.base.CodingAgent`.+ It sets the ambient attribution (round / stage / label) for the session so+ the model rows above land in the right bucket, applies the profile's turn cap+ and, for backends whose calls we cannot see (the subscription CLIs — see+ :data:`IN_PROCESS_AGENT_KINDS`), records one session row from+ ``AgentResult.usage``.++Accounting is never allowed to fail a run: every hook is wrapped, and an+exception in the ledger is logged and swallowed. The proxies forward every+other attribute, so a caller that reaches for ``model.pool`` or ``agent.kind``+sees the real object's.+"""++from __future__ import annotations++import logging+import threading+import time+from collections.abc import Iterator+from contextlib import contextmanager+from contextvars import ContextVar+from pathlib import Path+from typing import Any++from codeverse.contracts.agent import AgentJob, AgentResult+from codeverse.contracts.chat import ChatRequest, ChatResponse+from codeverse.contracts.common import Usage+from codeverse.cost.context import bind_run, call_context, run_binding+from codeverse.cost.ledger import (+ CostLedger,+ default_ledger_path,+ existing_ledger_path,+ open_run_ledger,+ record_call,+ set_default_ledger,+)+from codeverse.cost.types import Role, Stage, stage_for_label++log = logging.getLogger(__name__)++#: True while a run is metered call-by-call (:func:`run_ledger`). Aggregate+#: writers — ``orchestrator.budget.BudgetGuard``, which appends one row per+#: *charge* — must not also write, or every dollar lands in the ledger twice.+#:+#: Context-local first, with a process-wide **count** of the open run ledgers as+#: the fallback for a worker thread that inherited no context. A plain global+#: boolean cannot survive ``bench.run_bench --parallel N``: two threads entering+#: and leaving their own ledgers interleave, and the second one restores "on"+#: after the first turned it off.+_per_call_var: ContextVar[bool | None] = ContextVar("cv3d_cost_per_call", default=None)+_active_lock = threading.Lock()+_active_ledgers = 0+++def per_call_metering() -> bool:+ """True when every model call of the current run is already written to the+ ledger one by one, so a coarser writer should skip its own row."""+ local = _per_call_var.get()+ if local is not None:+ return local+ with _active_lock:+ return _active_ledgers > 0+++#: Agent kinds whose model calls go through ``models.get_chat_model`` and are+#: therefore ALREADY one ledger row each (``MeteredChatModel`` wraps that+#: factory). Every other kind is an opaque subscription CLI: we never see its+#: turns, it hands back one ``AgentResult.usage`` for the whole session, and that+#: session row is the only record of the money.+#:+#: This used to be decided by a thread-local "did anybody write a row while the+#: session ran?" counter, which is wrong in both directions and was reproduced by+#: the verifier (``adversarial.py``): a CLI session during which any in-process+#: tool billed a model (a texture pass, a summariser) looked metered and its whole+#: session — $1.23 in the reproduction — was silently dropped; and an in-process+#: session whose turns ran in another thread looked unmetered and was counted+#: twice. The backend either meters itself or it does not; that is a property of+#: the backend, not of what happened to run alongside it.+IN_PROCESS_AGENT_KINDS = frozenset({"api-agent"})+++def meters_own_calls(agent: Any) -> bool:+ """True when this backend's individual model calls are already on the ledger.++ A backend may state it with a ``meters_own_calls`` attribute; otherwise the+ kind decides (:data:`IN_PROCESS_AGENT_KINDS`)."""+ declared = getattr(agent, "meters_own_calls", None)+ if declared is not None:+ return bool(declared)+ return str(getattr(agent, "kind", "")) in IN_PROCESS_AGENT_KINDS+++class MeteredChatModel:+ """ChatModel proxy that appends one ledger row per call."""++ def __init__(self, inner: Any):+ self._inner = inner++ # -- protocol ---------------------------------------------------------+ @property+ def provider(self) -> str:+ return getattr(self._inner, "provider", "")++ @property+ def model(self) -> str:+ return getattr(self._inner, "model", "")++ @property+ def id(self) -> str:+ return str(self._inner.id)++ def supports_vision(self) -> bool:+ return bool(self._inner.supports_vision())++ def __getattr__(self, name: str) -> Any: # everything else is the real model's+ return getattr(self._inner, name)++ def __repr__(self) -> str: # pragma: no cover - debugging aid+ return f"MeteredChatModel({self._inner!r})"++ # -- the meter --------------------------------------------------------+ def generate(self, request: ChatRequest) -> ChatResponse:+ t0 = time.perf_counter()+ try:+ resp = self._inner.generate(request)+ except BaseException as exc: # noqa: BLE001 - record the attempt, then re-raise+ self._record(Usage(backend=self.provider, model=self.model), request,+ outcome=_outcome(exc), ms=int((time.perf_counter() - t0) * 1000))+ raise+ self._record(resp.usage, request, outcome="ok",+ ms=int((time.perf_counter() - t0) * 1000))+ return resp++ def _record(self, usage: Usage, request: ChatRequest, *, outcome: str, ms: int) -> None:+ try:+ record_call(usage, label=request.label, backend=usage.backend or self.provider,+ model=usage.model or self.model, outcome=outcome,+ latency_ms=usage.latency_ms or ms)+ except Exception as e: # pragma: no cover - accounting must never break a call+ log.debug("cost: could not record %s: %s", request.label, e)+++def _outcome(exc: BaseException) -> str:+ name = type(exc).__name__.lower()+ if "timeout" in name:+ return "timeout"+ return "error"+++class MeteredAgent:+ """CodingAgent proxy: attribution for the session + a turn-cap backstop.++ ``tracks.generation.agent_max_turns`` is where the cap is *decided* (it reads+ ``Settings.limits.agent_max_turns`` and handles the wrap-up continuation).+ There is no default cap — a 28-turn one lost its A/B, ``docs/COST.md`` §17 —+ so this is only a backstop for a caller that names one: a bench script, a+ test, ``$CV3D_AGENT_MAX_TURNS``. It only ever *lowers* ``job.max_turns``, so+ it composes with whatever the caller set."""++ def __init__(self, inner: Any, *, max_turns: int | None = None):+ self._inner = inner+ self.max_turns = max_turns++ @property+ def kind(self) -> str:+ return str(getattr(self._inner, "kind", ""))++ @property+ def model(self) -> str:+ return str(getattr(self._inner, "model", ""))++ @property+ def id(self) -> str:+ return str(self._inner.id)++ def available(self) -> tuple[bool, str]:+ return self._inner.available()++ def __getattr__(self, name: str) -> Any:+ return getattr(self._inner, name)++ def __repr__(self) -> str: # pragma: no cover - debugging aid+ return f"MeteredAgent({self._inner!r})"++ def run(self, job: AgentJob) -> AgentResult:+ job = self._capped(job)+ stage = stage_for_label(job.kind or job.label)+ t0 = time.perf_counter()+ with call_context(round=job.round, stage=stage, role=Role.GENERATOR, label=job.label):+ result = self._inner.run(job)+ if not meters_own_calls(self._inner): # an opaque CLI: its session row is the only record+ self._record_session(job, result, stage, int((time.perf_counter() - t0) * 1000))+ return result++ def _capped(self, job: AgentJob) -> AgentJob:+ cap = self.max_turns if self.max_turns is not None else _settings_turn_cap()+ if cap and 0 < cap < job.max_turns:+ return job.model_copy(update={"max_turns": cap})+ return job++ def _record_session(self, job: AgentJob, result: AgentResult, stage: Stage, ms: int) -> None:+ usage = result.usage+ if not (usage.input_tokens or usage.output_tokens or usage.cost_usd):+ return+ try:+ record_call(usage, round=job.round, stage=stage, role=Role.GENERATOR, label=job.label,+ backend=usage.backend or self.kind, model=usage.model or self.model,+ outcome="ok" if result.ok else (result.exit_reason or "error"),+ latency_ms=usage.latency_ms or ms,+ n_calls=max(1, int(result.tool_calls or 1)), source="session")+ except Exception as e: # pragma: no cover+ log.debug("cost: could not record agent session %s: %s", job.label, e)+++def _settings_turn_cap() -> int:+ try:+ from codeverse.config import get_settings++ return int(get_settings().limits.agent_max_turns)+ except Exception: # pragma: no cover - settings must never break a run+ return 0+++# --------------------------------------------------------------------------- factories+def metered_chat_model(model: Any) -> Any:+ """Wrap ``model`` unless it is already metered or metering is off."""+ if model is None or isinstance(model, MeteredChatModel) or not metering_enabled():+ return model+ return MeteredChatModel(model)+++def metered_agent(agent: Any) -> Any:+ if agent is None or isinstance(agent, MeteredAgent) or not metering_enabled():+ return agent+ return MeteredAgent(agent)+++def metering_enabled() -> bool:+ """``Settings.cost_ledger`` (env ``CV3D_COST_LEDGER=off`` also disables it)."""+ try:+ from codeverse.config import get_settings++ return bool(get_settings().cost_ledger)+ except Exception: # pragma: no cover+ return True+++# --------------------------------------------------------------------------- agent seam+_install_lock = threading.Lock()+_installed = False+++def install_agent_metering() -> bool:+ """Make ``get_coding_agent`` hand out metered agents (idempotent).++ The agent package builds its backends through one factory, so wrapping that+ factory is the whole integration; when ``agents/registry.py`` starts calling+ :func:`metered_agent` itself this seam becomes a no-op."""+ global _installed+ with _install_lock:+ if _installed:+ return False+ try:+ import codeverse.agents as agents_pkg+ from codeverse.agents import registry as agents_registry+ except Exception as e: # pragma: no cover - agents are optional for cost-only use+ log.debug("cost: agent metering unavailable: %s", e)+ return False+ inner = agents_registry.get_coding_agent+ if getattr(inner, "_cv3d_metered", False):+ _installed = True+ return False++ def get_coding_agent(agent_id: str) -> Any:+ return metered_agent(inner(agent_id))++ get_coding_agent._cv3d_metered = True # type: ignore[attr-defined]+ get_coding_agent.__doc__ = inner.__doc__+ agents_registry.get_coding_agent = get_coding_agent # type: ignore[assignment]+ agents_pkg.get_coding_agent = get_coding_agent # type: ignore[attr-defined]+ _installed = True+ return True+++# --------------------------------------------------------------------------- run activation+@contextmanager+def run_ledger(workspace: str | Path, *, run: str = "", create: bool = True) -> Iterator[CostLedger | None]:+ """Meter one run into ``<workspace>/telemetry/cost.jsonl``.++ Binds the run name, points :func:`~codeverse.cost.ledger.record_call` at the+ run's ledger and installs the agent seam; restores the previous default on+ the way out so a second run in the same process is not mixed in.++ ``create=False`` is for work done *after* a run finished (``3dcv judge``,+ a post-hoc texture pass): it appends only when the run already keeps a+ ledger, because a ledger holding nothing but the re-judge would be read as+ the whole run's cost and hide everything the run really spent. Without one+ the rows go to the per-process log instead.++ **Nests and parallelises.** The previous ledger and run binding are restored+ on the way out rather than cleared, so a bench cell that opens a ledger for+ the cell and then a second one for the harness run inside it keeps both; and+ both are context-local first (see :func:`codeverse.cost.context.bind_run`), so+ ``bench.run_bench`` can run N prompts in N threads without their rows mixing."""+ if not metering_enabled():+ yield None+ return+ global _active_ledgers+ ws = Path(workspace)+ prev_run = run_binding().run+ prev_path = default_ledger_path()+ prev_metering = _per_call_var.get()+ if not create and existing_ledger_path(ws) is None:+ bind_run(run or ws.name)+ try:+ yield None+ finally:+ bind_run(prev_run)+ return+ ledger = open_run_ledger(ws)+ set_default_ledger(ledger.path)+ bind_run(run or ws.name)+ install_agent_metering()+ _per_call_var.set(True)+ with _active_lock:+ _active_ledgers += 1+ try:+ yield ledger+ finally:+ _per_call_var.set(prev_metering)+ with _active_lock:+ _active_ledgers -= 1+ set_default_ledger(prev_path)+ bind_run(prev_run)+++__all__ = ["MeteredAgent", "MeteredChatModel", "install_agent_metering", "metered_agent",+ "metered_chat_model", "metering_enabled", "per_call_metering", "run_ledger"]diff --git a/harness/codeverse/cost/ledger.py b/harness/codeverse/cost/ledger.pyindex e5dbbf3a..a683302b 100644--- a/harness/codeverse/cost/ledger.py+++ b/harness/codeverse/cost/ledger.py@@ -20,10 +20,12 @@ import osimport threadingimport timefrom collections.abc import Iterable, Sequence+from contextvars import ContextVarfrom pathlib import Pathfrom typing import Anyfrom codeverse.contracts.common import Usage+from codeverse.cost.context import CallContext, attributefrom codeverse.cost.types import (CallCost,CostBucket,@@ -31,6 +33,7 @@ from codeverse.cost.types import (Stage,Summary,normalise_ids,+ role_for_stage,stage_for_label,)from codeverse.models.pricing import (@@ -42,11 +45,20 @@ from codeverse.models.pricing import (log = logging.getLogger(__name__)-#: default ledger file inside a run workspace+#: the ledger of a run lives in its telemetry bucket (docs/RUN_LAYOUT.md §telemetry)+TELEMETRY_DIR = "telemetry"+TELEMETRY_LEDGER = f"{TELEMETRY_DIR}/cost.jsonl"++#: legacy/alias name at the run root. ``flywheel.telemetry.live_ledger_path`` looks+#: here for "does this run have a live ledger", and runs recorded before the+#: telemetry bucket existed have the real file here — so it stays readable, and+#: :func:`open_run_ledger` leaves a symlink pointing at the telemetry copy.LEDGER_NAME = "cost_ledger.jsonl"#: env var that points ``record_call`` at a ledger when no path is passed+#: (``off`` / ``0`` / ``none`` disables ledger writing for the process)LEDGER_ENV = "CV3D_COST_LEDGER"+_LEDGER_OFF = frozenset({"off", "0", "no", "none", "false"})DIMENSIONS = ("run", "stage", "role", "backend", "model", "provider", "round", "outcome")@@ -84,32 +96,130 @@ class CostLedger:_default_lock = threading.Lock()_default: CostLedger | None = None+_fallback: CostLedger | None = None+_fallback_read = False++#: sentinel: "this execution context did not state a ledger"+_UNSET: Any = object()+#: context-local override of the default ledger. Mirrors+#: ``codeverse.cost.context.bind_run``: a bench worker running its own prompt in+#: its own thread must not have its rows land in a sibling run's file, while a+#: plain worker thread that inherited no context still finds the process's ledger.+_default_var: ContextVar[Any] = ContextVar("cv3d_cost_default_ledger", default=_UNSET)def set_default_ledger(path: str | Path | None) -> CostLedger | None:- """Point the module-level :func:`record_call` at ``path`` (``None`` disables it)."""- global _default+ """Point the module-level :func:`record_call` at ``path``.++ ``None`` clears the override (back to ``$CV3D_COST_LEDGER`` / the per-process+ log). Sets both the context-local override and the process-wide one, so a+ nested or parallel run is attributed correctly and an uninstrumented worker+ thread still writes somewhere sensible."""+ global _default, _fallback_read+ led = CostLedger(path) if path is not None else None+ _default_var.set(led if led is not None else _UNSET)+ with _default_lock:+ _default = led+ if led is None:+ _fallback_read = False # re-read $CV3D_COST_LEDGER next time+ return led+++def default_ledger_path() -> Path | None:+ """The path :func:`set_default_ledger` last set here (``None`` = no override).+ Used to save/restore around a nested :func:`~codeverse.cost.instrument.run_ledger`."""+ led = _default_var.get()+ if led is not _UNSET:+ return led.path if led is not None else Nonewith _default_lock:- _default = CostLedger(path) if path is not None else None- return _default+ return _default.path if _default is not None else Nonedef default_ledger() -> CostLedger | None:"""The ledger :func:`record_call` writes to when no ``ledger=`` is given:- whatever :func:`set_default_ledger` set, else ``$CV3D_COST_LEDGER``."""- global _default+ whatever :func:`set_default_ledger` set, else ``$CV3D_COST_LEDGER``, else the+ per-process fallback log (:func:`process_ledger_path`) so a call made outside+ any run — ``3dcv judge``, a bench script, a notebook — is still accounted for.+ ``CV3D_COST_LEDGER=off`` turns writing off entirely."""+ global _fallback, _fallback_read+ led = _default_var.get()+ if led is not _UNSET:+ return ledwith _default_lock:if _default is not None:return _default+ if _fallback_read:+ return _fallbackenv = os.environ.get(LEDGER_ENV, "").strip()- if env:- _default = CostLedger(env)- return _default+ _fallback = None if env.lower() in _LEDGER_OFF else CostLedger(env or process_ledger_path())+ _fallback_read = True+ return _fallback+++def process_ledger_path() -> Path:+ """The fallback log for calls made with no run context: one file per process+ under ``<cache_dir>/cost/``. Never inside a run directory, so it can never be+ mistaken for a run's own ledger."""+ try:+ from codeverse.config import get_settings++ base = Path(get_settings().cache_dir)+ except Exception: # pragma: no cover - settings must never break accounting+ base = Path.home() / ".cache" / "codeverse"+ stamp = time.strftime("%Y%m%d", time.localtime())+ return base / "cost" / f"p{os.getpid()}-{stamp}.jsonl"def ledger_path(workspace: str | Path) -> Path:- """The conventional ledger path of a run workspace (``<ws>/cost_ledger.jsonl``)."""- return Path(workspace) / LEDGER_NAME+ """The live ledger of a run workspace: ``<ws>/telemetry/cost.jsonl``."""+ return Path(workspace) / TELEMETRY_LEDGER+++def existing_ledger_path(workspace: str | Path) -> Path | None:+ """The ledger file a run actually has — the telemetry one, else the legacy+ root file — or ``None``."""+ root = Path(workspace)+ for name in (TELEMETRY_LEDGER, LEDGER_NAME):+ p = root / name+ if p.is_file():+ return p+ return None+++def open_run_ledger(workspace: str | Path) -> CostLedger:+ """The run's live ledger, ready to append to.++ Writes to ``telemetry/cost.jsonl`` and leaves ``<run>/cost_ledger.jsonl`` as a+ relative symlink to it, so there is exactly one physical copy and the run+ layout's ``live_ledger_path`` / ``telemetry/usage.jsonl`` alias keep working."""+ root = Path(workspace)+ path = root / TELEMETRY_LEDGER+ try:+ path.parent.mkdir(parents=True, exist_ok=True)+ alias = root / LEDGER_NAME+ if not alias.exists() and not alias.is_symlink():+ os.symlink(TELEMETRY_LEDGER, alias)+ except OSError as e: # pragma: no cover - read-only dir / no symlinks+ log.debug("cost: could not prepare the run ledger at %s: %s", path, e)+ return CostLedger(path)+++def _as_stage(value: Stage | str | None) -> Stage | None:+ if value is None or isinstance(value, Stage):+ return value+ try:+ return Stage(value)+ except ValueError:+ return stage_for_label(str(value))+++def _as_role(value: Role | str | None) -> Role | None:+ if value is None or isinstance(value, Role):+ return value+ try:+ return Role(value)+ except ValueError:+ return Role.OTHERdef price_call(@@ -162,7 +272,7 @@ def record_call(usage: Usage | None = None,*,run: str = "",- stage: Stage | str = Stage.OTHER,+ stage: Stage | str | None = None,role: Role | str | None = None,round: int | None = None, # noqa: A002 - matches the record/round vocabularylabel: str = "",@@ -186,21 +296,17 @@ def record_call(record_call(res.usage, run=ws.slug, round=idx, stage="baseline",role="generator", ledger=ws.root / "cost_ledger.jsonl")+ Anything the caller leaves out (``run`` / ``round`` / ``stage`` / ``role``) is+ resolved from the ambient run context and the call label — see+ :mod:`codeverse.cost.context`.+Never raises: a failure to account is logged, not propagated."""u = usage or Usage()- try:- st = Stage(stage) if not isinstance(stage, Stage) else stage- except ValueError:- st = stage_for_label(str(stage))- if role is None:- from codeverse.cost.types import role_for_stage-- rl = role_for_stage(st)- else:- try:- rl = Role(role) if not isinstance(role, Role) else role- except ValueError:- rl = Role.OTHER+ ctx = attribute(CallContext(run=run, round=round, stage=_as_stage(stage), role=_as_role(role),+ label=label), label=label)+ st = ctx.stage or Stage.OTHER+ rl = ctx.role or role_for_stage(st)+ run, round, label = ctx.run, ctx.round, ctx.label # noqa: A001 - see the signaturetry:cost, price_fields = price_call(u, backend=backend, model=model,cache_write_tokens=cache_write_tokens, reprice=reprice)@@ -237,11 +343,15 @@ def record_call(def load_ledger(path: str | Path) -> list[CallCost]:- """Read a ledger file (or a directory containing one). Bad lines are skipped- with a debug log — a truncated last line never loses the rest of the file."""+ """Read a ledger file (or a run directory containing one: ``telemetry/cost.jsonl``+ first, then the legacy root ``cost_ledger.jsonl``). Bad lines are skipped with a+ debug log — a truncated last line never loses the rest of the file."""p = Path(path)if p.is_dir():- p = p / LEDGER_NAME+ found = existing_ledger_path(p)+ if found is None:+ return []+ p = foundif not p.is_file():return []rows: list[CallCost] = []diff --git a/harness/codeverse/cost/profiles.py b/harness/codeverse/cost/profiles.pynew file mode 100644index 00000000..2f75d9a5--- /dev/null+++ b/harness/codeverse/cost/profiles.py@@ -0,0 +1,170 @@+"""Cost/quality profiles — one name that sets the whole dial coherently.++``3dcv make --profile economy|balanced|quality`` picks a :class:`Profile`; it+resolves the model per role, the judge sample count, the number of refine+rounds, the best-of-N width, the judge payload, the texture pass and the budget+ceilings in one consistent move. Mixing knobs by hand is how a run ends up+paying for a pro judge on a single-shot artifact, or running four refine rounds+on a graphics shader that passed at round 0.++``--profile X`` and ``CV3D_PROFILE=X`` (or ``profile:`` in ``config.yaml``)+resolve to the **same** dial: ``codeverse.cli._common.resolve_dial`` is the one+place that reads it, and ``tests/cost/test_profiles.py`` asserts every+field of the resolved dial from both entry points. The only difference is+precedence — the flag forces the dial over a value you stated yourself, the+env var yields to it.++Every number is taken from the measurements in ``docs/COST.md``:++* **economy** — single-shot generation (graphics: 5/6 passed at $0.10/run; a+ one-file generator plus compiler feedback needs no tool loop), the flash judge+ at ``n=2`` (σ 0.083/√2 ≈ 0.059 — enough to rank two rounds, not enough for a+ pass/fail that persists) and **2 refine rounds** (r01 buys a score point for+ $4.93, r02 for $6.80, r03 for $33.49). It does **not** shrink the judge+ payload and does **not** carry its own turn cap — both were measured and+ neither paid (see the field comments below).+* **balanced** — exactly today's defaults (api-agent generator, pro judge at+ n=1, 4 rounds, no best-of-N, no texture): the arm every number in the audit+ was measured on.+* **quality** — api-agent + the pro judge at ``n=3`` (σ 0.030/√3 ≈ 0.017),+ best-of-2 baselines and the texture pass behind its before/after gate.++The expected $ per profile comes from the same 61 runs; see+:func:`Profile.expectation` and ``docs/COST.md`` §8–9.+"""++from __future__ import annotations++from dataclasses import dataclass, replace++#: profile names in increasing order of spend+PROFILE_NAMES = ("economy", "balanced", "quality")++DEFAULT_PROFILE = "balanced"+++@dataclass(frozen=True)+class Profile:+ """One coherent setting of the cost/quality dial."""++ name: str+ #: model per role ("" = leave the settings default alone)+ generator: str = ""+ planner: str = ""+ judge: str = ""+ captioner: str = ""+ #: run shape+ rounds: int = 4 # refine rounds after the baseline+ candidates: int = 1 # best-of-N baselines+ judge_samples: int = 1 # VLM judge samples per verdict+ max_turns: int = 0 # agent turn cap (0 = the backend's own default)+ texture: bool = False # run the derived texture pass+ #: judge payload+ judge_max_px: int = 1024 # measured: 768 bills the same and is noisier — see PROFILES+ judge_montages: int = 3+ judge_detail_crops: int = 2 # one crop ≈ 1,198 input tokens ($0.0024 on the pro judge);+ # NOT reduced by any profile — the score effect is inside the+ # judge's own noise in both directions (docs/COST.md §14)+ #: budget ceilings a run of this shape should not need to exceed+ max_usd: float = 5.0+ max_minutes: float = 60.0+ #: measured expectation (docs/COST.md) — reported by ``3dcv cost profiles``+ expected_usd: float = 0.0+ expected_score: str = ""+ note: str = ""++ def expectation(self) -> str:+ return f"~${self.expected_usd:.2f}/run, {self.expected_score}"++ def with_overrides(self, **kw: object) -> Profile:+ """Copy with the fields the user stated explicitly (``None`` = not stated)."""+ clean = {k: v for k, v in kw.items() if v is not None and hasattr(self, k)}+ return replace(self, **clean) if clean else self # type: ignore[arg-type]+++#: the three dials. Generator ids are harness backend ids; judge/planner ids are+#: ``<provider>:<model>``.+PROFILES: dict[str, Profile] = {+ "economy": Profile(+ name="economy",+ generator="single-shot:gemini:gemini-3.7-flash",+ planner="gemini:gemini-3.7-flash",+ judge="gemini:gemini-3.7-flash",+ captioner="gemini:gemini-3.7-flash",+ rounds=2, candidates=1, judge_samples=2, texture=False,+ # max_turns=0 (2026-08-23): economy's generator is ``single-shot:``, which opens no+ # agent session, so the 20-turn cap this profile used to carry could never fire. And+ # the cap it was modelled on did not survive its own A/B — 3 runs per arm, a 28-turn+ # cap cost $0.02 MORE and 0.205 of a score point (docs/COST.md §17) — so there is no+ # default cap anywhere now. A caller who wants one still has+ # ``Settings.limits.agent_max_turns`` / ``$CV3D_AGENT_MAX_TURNS``.+ max_turns=0,+ # 1024 px, not 768: measured on 8 recorded rounds, Gemini bills a montage the same at+ # both sizes (156,834 vs 156,874 input tokens over 16 verdicts) while 768 raised the+ # sampling σ 0.033 → 0.042.+ # detail_crops=2, i.e. the same payload as every other profile. Dropping to 1 crop saves+ # 1,198 input tokens ($0.0024/verdict on pro) and was adopted on a single draw that read+ # 0.595 → 0.600; an independent re-draw of the same round read 0.600 → 0.552, a 0.048+ # swing = 1.6x the pro judge's measured σ (0.030). Two draws that disagree by more than+ # the instrument's noise do not license a payload cut worth a fifth of a cent, so the+ # experiment is recorded as INCONCLUSIVE in docs/COST.md §14 and the budget is unchanged.+ judge_max_px=1024, judge_montages=3, judge_detail_crops=2,+ max_usd=1.50, max_minutes=30.0,+ expected_usd=0.30,+ expected_score="graphics 0.81 median (5/6 passed); objects clear the gates less often "+ "— a one-shot flash generator scored 0.14 on compare_v1's hard cells",+ note="single-shot generator + flash judge n=2 + 2 refine rounds; no best-of-N, no texture, "+ "full judge payload",+ ),+ "balanced": Profile(+ name="balanced",+ generator="api-agent:gemini:gemini-3.7-flash",+ planner="gemini:gemini-3.7-flash",+ judge="gemini:gemini-3.1-pro-preview",+ captioner="gemini:gemini-3.7-flash",+ rounds=4, candidates=1, judge_samples=1, max_turns=0, texture=False,+ judge_max_px=1024, judge_montages=3, judge_detail_crops=2,+ max_usd=5.0, max_minutes=60.0,+ expected_usd=1.47,+ expected_score="0.835 mean on compare_v1, 36/61 runs passed ($2.50 per passing artifact)",+ note="today's defaults — the arm every number in docs/COST.md was measured on",+ ),+ "quality": Profile(+ name="quality",+ generator="api-agent:gemini:gemini-3.7-flash",+ planner="gemini:gemini-3.7-flash",+ judge="gemini:gemini-3.1-pro-preview",+ captioner="gemini:gemini-3.7-flash",+ rounds=4, candidates=2, judge_samples=3, max_turns=0, texture=True,+ judge_max_px=1024, judge_montages=3, judge_detail_crops=2,+ max_usd=8.0, max_minutes=90.0,+ expected_usd=3.20,+ expected_score="best-of-2 lifted the stool baseline 0.563 → 0.612 and the texture pass "+ "0.686 → 0.701; judge σ 0.017 at n=3",+ note="api-agent + pro judge n=3 + 4 rounds + best-of-2 + texture pass",+ ),+}+++def get_profile(name: str | None) -> Profile:+ """The named profile (case-insensitive). Unknown names raise ``ValueError``."""+ key = (name or DEFAULT_PROFILE).strip().lower()+ if key not in PROFILES:+ raise ValueError(f"unknown profile {name!r}; known: {', '.join(PROFILE_NAMES)}")+ return PROFILES[key]+++def profile_table() -> list[tuple[str, str, str, str, str, str, str]]:+ """Rows for ``3dcv cost profiles``: name, generator, judge, shape, montage, $, note."""+ rows = []+ for name in PROFILE_NAMES:+ p = PROFILES[name]+ shape = f"{p.rounds}r × {p.candidates}cand, judge n={p.judge_samples}" + (+ f", ≤{p.max_turns} turns" if p.max_turns else "")+ rows.append((p.name, p.generator, p.judge, shape,+ f"{p.judge_max_px}px/{p.judge_detail_crops}crop" + (" +texture" if p.texture else ""),+ f"${p.expected_usd:.2f}", p.expected_score))+ return rows+++__all__ = ["DEFAULT_PROFILE", "PROFILES", "PROFILE_NAMES", "Profile", "get_profile", "profile_table"]diff --git a/harness/codeverse/cost/reconstruct.py b/harness/codeverse/cost/reconstruct.pyindex 517886c6..cd4a1bcd 100644--- a/harness/codeverse/cost/reconstruct.py+++ b/harness/codeverse/cost/reconstruct.py@@ -30,6 +30,7 @@ from __future__ import annotationsimport jsonimport logging+import refrom collections.abc import Iteratorfrom dataclasses import dataclass, fieldfrom pathlib import Path@@ -113,6 +114,7 @@ class RunLedger:parallel: bool = Falseresumed: bool = Falseselected_candidate: str = "" # "c1" when the run ran best-of-N and c1 won+ source: str = "reconstructed" # live (the run wrote its own ledger) | reconstructedrows: list[CallCost] = field(default_factory=list)notes: list[str] = field(default_factory=list)@@ -165,6 +167,24 @@ def _gemini_cli_usages(stdout: dict[str, Any]) -> list[Usage]:return out+#: a retried / wrapped-up agent session records itself as ``<label>.a2`` / ``<label>.wrapup``+_ATTEMPT_SUFFIX = re.compile(r"\.(a\d+|wrapup)$")+++def _base_label(label: str) -> str:+ """The *job* label a session label was derived from.++ A retry is recorded as ``<label>.a2``, a wrap-up as ``<label>.wrapup`` and a+ best-of-N candidate as ``c<k>:<label>``, while the ``generate.done`` event that+ summarises the whole job carries the plain job label. De-duplication has to+ use this key: keying it on the session's own label left the retry's dollars in+ a different bucket, the job's pool ran dry, and the event was counted a SECOND+ time. That artifact — not lost money — was most of the "$4.80 invisible to+ record.total_usage" in docs/COST.md §6; §6 now quotes the corrected figure."""+ lab = label.split(":", 1)[-1] if ":" in label else label+ return _ATTEMPT_SUFFIX.sub("", lab)++def _session_rows(d: Path, run: str, *, recheck: bool,candidate: str = "") -> tuple[list[CallCost], str, int | None, float]:"""Rows for one ``trajectories/<label>_rNN`` directory. ``candidate`` is the@@ -203,6 +223,35 @@ def _session_rows(d: Path, run: str, *, recheck: bool,return rows, label, rnd, duration+def _live_rows(root: Path, *, run: str, recheck: bool) -> list[CallCost]:+ """The run's own ledger rows, when it wrote one while it ran.++ Rows carry their run name from the writer; stamp the directory's name on any+ that do not (a battery copies runs around). ``recheck`` re-prices them from+ today's table exactly like a reconstructed row."""+ try:+ from codeverse.cost.ledger import load_ledger++ rows = load_ledger(root)+ except Exception as e: # noqa: BLE001 - a corrupt ledger falls back to reconstruction+ log.debug("cost: live ledger unreadable in %s: %s", root, e)+ return []+ out: list[CallCost] = []+ for row in rows:+ if not row.run:+ row.run = run+ if recheck:+ usage = Usage(backend=row.backend, model=row.model_id or row.model,+ input_tokens=row.input_tokens, cached_tokens=row.cached_tokens,+ output_tokens=row.output_tokens, thoughts_tokens=row.thoughts_tokens,+ tool_calls=row.tool_calls, cost_usd=row.recorded_usd)+ cost, fields = price_call(usage, backend=row.backend, model=row.model_id or row.model,+ cache_write_tokens=row.cache_write_tokens, reprice=True)+ row = row.model_copy(update={"cost_usd": cost, **fields})+ out.append(row)+ return out++def _trajectory_dirs(root: Path) -> list[tuple[Path, str]]:"""Every session directory of a run as ``(dir, candidate)``, including sceneasset sub-workspaces (``_assets/<name>/trajectories/*``) and best-of-N@@ -220,6 +269,12 @@ def _trajectory_dirs(root: Path) -> list[tuple[Path, str]]:def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:"""Build the ledger of one run directory.+ A run that wrote its own ledger while it ran (``telemetry/cost.jsonl``, see+ :mod:`codeverse.cost.instrument`) is read from that ledger — one priced row+ per real call, nothing inferred. Runs recorded before the ledger existed are+ rebuilt from ``trajectories/`` + ``record.json`` + ``events.jsonl`` exactly as+ before, so all 61 reference runs keep auditing.+``recheck=True`` re-prices every row with the CURRENT price table (and rebuildsgemini-cli rows from the raw CLI stats) — that is the audit view; the defaultkeeps the dollars the run was actually billed."""@@ -246,15 +301,21 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:led.n_rounds = len(rounds)led.passed = bool(record.get("status") == "passed")- covered: dict[str, float] = {} # label -> $ already accounted for by its agent session(s);- # a ``generate.done`` event within that budget is a duplicate+ live_rows = _live_rows(root, run=run, recheck=recheck)+ led.source = "live" if live_rows else "reconstructed"+ led.rows += live_rows++ covered: dict[str, float] = {} # JOB label (see _base_label) -> $ already accounted for by+ # its session(s); a ``generate.done`` event within that+ # budget is a duplicate, retries and wrap-ups includedmodel_s = 0.0- for d, cand in _trajectory_dirs(root):+ for d, cand in (() if live_rows else _trajectory_dirs(root)):rows, label, rnd, duration = _session_rows(d, run, recheck=recheck, candidate=cand)if rows:led.rows += rows- covered[label] = covered.get(label, 0.0) + sum(r.recorded_usd for r in rows)+ key = _base_label(label)+ covered[key] = covered.get(key, 0.0) + sum(r.recorded_usd for r in rows)model_s += durationjudged_rounds: set[int] = set()@@ -267,6 +328,8 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:ju = _usage(judgment.get("usage"))if ju.input_tokens or ju.cost_usd:judged_rounds.add(idx)+ model_s += ju.latency_ms / 1000.0+ if (ju.input_tokens or ju.cost_usd) and not live_rows:n = int(judgment.get("n_samples") or 1)degraded = bool((judgment.get("raw") or {}).get("degraded")) if isinstance(judgment.get("raw"), dict) else Falseled.rows.append(_row(ju, run=run, stage=Stage.JUDGE, label=str(judgment.get("rubric") or "judge"),@@ -274,7 +337,6 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:outcome="degraded" if degraded else "ok", reprice=recheck,backend=str(judgment.get("judge_backend") or ju.backend) or ju.backend,model=str(judgment.get("judge_backend") or ju.model) or ju.model))- model_s += ju.latency_ms / 1000.0# ------------------------------------------------------------------ eventsfirst_t = last_t = 0.0@@ -295,14 +357,16 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:if name == "judge.done":model_s += float(ev.get("duration_s") or 0.0) if int(ev.get("round", -1)) not in judged_rounds else 0.0cost = ev.get("cost_usd")- if not isinstance(cost, (int, float)) or not cost or name in _AGGREGATE_EVENTS:+ if live_rows or not isinstance(cost, (int, float)) or not cost or name in _AGGREGATE_EVENTS:continuernd = ev.get("round")rnd = int(rnd) if isinstance(rnd, (int, float)) else Nonelabel = str(ev.get("label") or name.split(".")[0])- if name == "generate.done" and covered.get(label, 0.0) >= float(cost) - 0.0005:- covered[label] = covered.get(label, 0.0) - float(cost)- continue # a session already contributed this spend+ if name == "generate.done":+ key = _base_label(label)+ if covered.get(key, 0.0) >= float(cost) - 0.0005:+ covered[key] = covered[key] - float(cost)+ continue # a session already contributed this spendif name == "judge.done" and rnd in judged_rounds:continuestage = Stage.TEXTURE if name.startswith("texture") else stage_for_label(label)@@ -363,6 +427,12 @@ def reconstruct_cell(cell_dir: str | Path, *, recheck: bool = False) -> RunLedgestatus=str(cell.get("status") or ""), passed=bool(cell.get("passed")),final_score=cell.get("score"), n_rounds=1,wall_s=float(cell.get("wall_s") or 0.0), model_s=float(cell.get("gen_seconds") or 0.0))+ live = _live_rows(root, run=run, recheck=recheck)+ if live: # the cell wrote its own ledger while it ran: one priced row per real call+ led.source = "live"+ led.rows += live+ led.recorded_usd = float(cell.get("gen_cost_usd") or 0.0) + float(cell.get("judge_cost_usd") or 0.0)+ return ledfor attempt in sorted(root.glob("gen/attempt*")):result = _read_json(attempt / "result.json")usage = _usage(result.get("usage"))@@ -415,5 +485,10 @@ def reconstruct(path: str | Path, *, recheck: bool = False) -> RunLedger:if (p / "run" / "record.json").is_file():led = reconstruct_run(p / "run", recheck=recheck)led.run = f"{p.parent.name}/{p.name}"+ # a compare cell also spends OUTSIDE the harness run it wraps: the fixed+ # evaluator's judge, and a repair arm's extra generations. `run_cell` opens a+ # ledger for the cell itself (nested around the run's own), and that spend is+ # exactly the §6 gap the audit sees on the two cmp_easy_stool harness cells.+ led.rows += _live_rows(p, run=led.run, recheck=recheck)return ledreturn reconstruct_cell(p, recheck=recheck)diff --git a/harness/codeverse/fanout.py b/harness/codeverse/fanout.pyindex f111d18d..9d5a8fdb 100644--- a/harness/codeverse/fanout.py+++ b/harness/codeverse/fanout.py@@ -9,6 +9,7 @@ retried). Exceptions that are ``BaseException`` but not ``Exception``from __future__ import annotations+import contextvarsimport loggingimport timefrom collections.abc import Callable, Iterable, Sequence@@ -41,6 +42,17 @@ class FanOutReport:}+def _in_caller_context(snapshot: contextvars.Context, fn: Callable[[int], None], i: int) -> None:+ """Run ``fn(i)`` with the values the caller's context held at fan-out time.++ ``Context.run`` cannot be re-entered from several threads, so each worker+ replays the snapshot's variables into its own (already fresh) context.+ """+ for var, value in snapshot.items():+ var.set(value)+ fn(i)++def fan_out(items: Sequence[T] | Iterable[T],fn: Callable[[T], R],@@ -57,6 +69,11 @@ def fan_out(seconds)`` — keep it cheap and thread-safe (e.g. an EventLog.emit)."""items = list(items)+ # Workers inherit the caller's context so ambient state set with contextvars+ # (the cost ledger's run/stage/role attribution) follows a parallel judge+ # sample, best-of-N candidate or bench cell instead of falling back to the+ # process default.+ ctx_snapshot = contextvars.copy_context()results: list[R | Exception] = [None] * len(items) # type: ignore[list-item]if not items:return results@@ -85,7 +102,8 @@ def fan_out(_run(i)else:with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=label) as pool:- list(pool.map(_run, range(len(items))))+ list(pool.map(lambda i: contextvars.copy_context().run(_in_caller_context, ctx_snapshot, _run, i),+ range(len(items))))n_failed = sum(1 for r in results if isinstance(r, Exception))report = FanOutReport(label, len(items), len(items) - n_failed, n_failed, durations, time.time() - t_all)diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.pyindex 358c4d23..b2ec16a9 100644--- a/harness/codeverse/judges/vlm_judge.py+++ b/harness/codeverse/judges/vlm_judge.py@@ -65,9 +65,9 @@ class VlmJudge:*,thinking: Literal["off", "low", "medium", "high"] = "low",max_attempts: int = 3,- max_montages: int = 3,- detail_crops: int = 2,- max_px: int = 1024,+ max_montages: int | None = None,+ detail_crops: int | None = None,+ max_px: int | None = None,chat_model: ChatModel | None = None,cache_dir: Path | None = None,label: str = "judge",@@ -78,9 +78,11 @@ class VlmJudge:self.temperature = temperatureself.thinking = thinkingself.max_attempts = max(1, int(max_attempts))- self.max_montages = max(1, int(max_montages))- self.detail_crops = max(0, int(detail_crops))- self.max_px = max_px+ # payload size: the profile's dial (Settings.judge) unless the caller states one+ jd = get_settings().judge+ self.max_montages = max(1, int(jd.montages if max_montages is None else max_montages))+ self.detail_crops = max(0, int(jd.detail_crops if detail_crops is None else detail_crops))+ self.max_px = int(jd.max_px if max_px is None else max_px)self._model = chat_modelself.cache_dir = cache_dirself.label = label@@ -122,7 +124,10 @@ class VlmJudge:)reqs.append(ChatRequest(messages=messages, system=system, response_schema=schema, temperature=self.temperature,- thinking=self.thinking, label=f"{self.label}:{self.rubric.name}:s{k}",+ thinking=self.thinking,+ # the label is the ledger's attribution when the sample runs in a worker+ # thread (fan_out does not carry context vars): keep stage + round in it+ label=f"{self.label}:{self.rubric.name}:r{inp.round_index:02d}:s{k}",))if len(reqs) == 1: # serial path: no pool, no thread hopoutcomes = [self._sample(reqs[0], acceptance_ids, measured)]diff --git a/harness/codeverse/models/gemini_image.py b/harness/codeverse/models/gemini_image.pyindex 1af91063..1ae7e619 100644--- a/harness/codeverse/models/gemini_image.py+++ b/harness/codeverse/models/gemini_image.py@@ -120,6 +120,19 @@ def _is_model_missing(err: ModelError) -> bool:return err.status == 404 or (err.status == 400 and ("not found" in s or "not supported" in s))+def _record(usage: Usage, n_images: int) -> None:+ """One ledger row per image batch (the image model is not a ChatModel, so+ ``models.registry`` cannot meter it). Never raises."""+ try:+ from codeverse.cost import record_call+ from codeverse.cost.types import Role, Stage++ record_call(usage, stage=Stage.TEXTURE, role=Role.IMAGE, label="image",+ backend="gemini-image", model=usage.model, n_calls=max(1, n_images))+ except Exception: # noqa: BLE001 - accounting must never break a texture pass+ pass++class GeminiImageModel:"""See module docstring. ``model`` is the primary; ``fallback`` the second try."""@@ -193,6 +206,7 @@ class GeminiImageModel:usage = usage + uif len(images) > n:images = images[:n]+ _record(usage, len(images))return images, usage# ------------------------------------------------------------------ internalsdiff --git a/harness/codeverse/models/registry.py b/harness/codeverse/models/registry.pyindex 69dc710e..56bf1644 100644--- a/harness/codeverse/models/registry.py+++ b/harness/codeverse/models/registry.py@@ -1,4 +1,9 @@-"""Model id parsing + factory. Ids: ``<provider>:<model>``."""+"""Model id parsing + factory. Ids: ``<provider>:<model>``.++Every model handed out here is wrapped by ``codeverse.cost.instrument`` so each+call lands in the cost ledger (the run's ``telemetry/cost.jsonl`` when a run is+active, otherwise a per-process log). ``CV3D_COST_LEDGER=off`` /+``Settings.cost_ledger=false`` returns the bare model."""from __future__ import annotations@@ -20,6 +25,13 @@ def parse_model_id(model_id: str) -> tuple[str, str]:@lru_cache(maxsize=32)def get_chat_model(model_id: str) -> ChatModel:+ """The (metered) ChatModel for ``<provider>:<model>``."""+ from codeverse.cost.instrument import metered_chat_model++ return metered_chat_model(_build_chat_model(model_id))+++def _build_chat_model(model_id: str) -> ChatModel:provider, model = parse_model_id(model_id)if provider == "gemini":from codeverse.models.gemini import GeminiModeldiff --git a/harness/codeverse/orchestrator/budget.py b/harness/codeverse/orchestrator/budget.pyindex e9ed6170..90733ef5 100644--- a/harness/codeverse/orchestrator/budget.py+++ b/harness/codeverse/orchestrator/budget.py@@ -4,6 +4,17 @@ Every model / agent call yields a ``Usage``; the track charges it here. Wheneither **hard** ceiling is crossed ``BudgetExceeded`` is raised so the roundloop can stop cleanly (status ``budget``) instead of silently overspending.+**One door for every dollar.** ``charge`` and ``add`` are thin wrappers over+:meth:`BudgetGuard.spend`, which is the only place money enters the run: it+accumulates the ``Usage``, buckets it by stage and by round, appends one priced+row to the run's cost ledger (``<run>/cost_ledger.jsonl``,+``codeverse.cost.record_call``) and then — for ``charge`` — enforces the+ceilings. The audit (docs/COST.md §6) found $4.80 of real spend that never+reached ``record.total_usage``: rounds the budget cut after the work was done,+retried ``.a2`` agent sessions and post-hoc texture passes. Anything that goes+through ``spend`` is visible to the guard, to the record and to the ledger, so+that class of blindness cannot come back.+On top of that a **soft** sub-budget (``soft_fraction`` of the hard ceilings)lets a track see the wall coming: ``soft_ok()`` / ``soft_exceeded()`` say"the baseline has used its share — degrade now (fewer assets, single-shot@@ -15,11 +26,19 @@ build + render + judge) instead of dying with no score at all.from __future__ import annotations+import loggingimport threadingimport time+from pathlib import Path+from typing import Anyfrom codeverse.contracts.common import Budget, Usage+log = logging.getLogger(__name__)++#: stage label used when a caller does not say where the money went+OTHER_STAGE = "other"+class BudgetExceeded(RuntimeError):"""Raised when a run crosses its USD or wall-clock ceiling."""@@ -38,7 +57,8 @@ class BudgetGuard:``BudgetExceeded`` when ``max_usd`` or ``max_minutes`` is exceeded."""- def __init__(self, budget: Budget, start_time: float | None = None, *, soft_fraction: float = 1.0):+ def __init__(self, budget: Budget, start_time: float | None = None, *, soft_fraction: float = 1.0,+ run: str = "", ledger: str | Path | Any | None = None):self.budget = budgetself.start_time = time.time() if start_time is None else start_timeself.spent = Usage()@@ -49,24 +69,87 @@ class BudgetGuard:#: one-off extension of the HARD ceilings (finalise/salvage headroom)self.grace_usd = 0.0self.grace_minutes = 0.0+ #: run slug + ledger sink for the per-call rows (None = do not write a ledger)+ self.run = run+ self.ledger = ledger+ #: stage -> USD and round -> {stage -> USD}, so a round can report what it burned+ self.by_stage: dict[str, float] = {}+ self.by_round: dict[int, dict[str, float]] = {}# ----------------------------------------------------------------- accounting- def charge(self, usage: Usage | None) -> None:- """Add ``usage`` to the running total, then enforce the ceilings."""- self.add(usage)- self.check()+ def spend(+ self,+ usage: Usage | None,+ *,+ stage: str = OTHER_STAGE,+ role: str | None = None,+ label: str = "",+ round_index: int | None = None,+ outcome: str = "ok",+ enforce: bool = True,+ ) -> None:+ """THE door every dollar goes through.++ Accumulates ``usage``, buckets it by ``stage`` (and by round), appends a+ priced row to the run's cost ledger, and — when ``enforce`` — raises+ ``BudgetExceeded`` if a hard ceiling is now crossed. Accounting never+ fails a run: a broken ledger is logged, not propagated."""+ if usage is None:+ return+ with self._lock:+ self.spent = self.spent + usage+ self.calls += 1+ key = stage or OTHER_STAGE+ self.by_stage[key] = self.by_stage.get(key, 0.0) + float(usage.cost_usd)+ if round_index is not None:+ per = self.by_round.setdefault(int(round_index), {})+ per[key] = per.get(key, 0.0) + float(usage.cost_usd)+ self._ledger_row(usage, stage=key, role=role, label=label, round_index=round_index, outcome=outcome)+ if enforce:+ self.check()- def add(self, usage: Usage | None) -> None:- """Accumulate ``usage`` WITHOUT enforcing the ceilings.+ def charge(self, usage: Usage | None, **kw: Any) -> None:+ """Account for ``usage``, then enforce the ceilings (see :meth:`spend`)."""+ kw.setdefault("enforce", True)+ self.spend(usage, **kw)++ def add(self, usage: Usage | None, **kw: Any) -> None:+ """Account for ``usage`` WITHOUT enforcing the ceilings.For work that is already done and persisted (a completed judge verdict,a pairwise tie-break, the texture pass): the money is spent either way,and raising here would throw away a finished, paid-for result. The- round loop stops at its next ``ok()`` check instead."""- if usage is not None:- with self._lock:- self.spent = self.spent + usage- self.calls += 1+ round loop stops at its next ``ok()`` check instead. The dollar is+ still bucketed and still written to the ledger — "not enforced" never+ means "not seen"."""+ kw["enforce"] = False+ self.spend(usage, **kw)++ def _ledger_row(self, usage: Usage, *, stage: str, role: str | None, label: str,+ round_index: int | None, outcome: str) -> None:+ if self.ledger is None:+ return+ try:+ from codeverse.cost import per_call_metering, record_call++ # When the run is metered call by call (codeverse.cost.instrument wraps every+ # ChatModel / CodingAgent), those rows already ARE this dollar with per-call+ # tokens, cache hits and latency — one aggregate row on top would double count.+ if per_call_metering():+ return+ record_call(usage, run=self.run, stage=stage, role=role, round=round_index, label=label,+ backend=usage.backend, model=usage.model, outcome=outcome, ledger=self.ledger)+ except Exception as e: # noqa: BLE001 — accounting must never fail a run+ log.debug("cost ledger row failed (%s): %s", self.ledger, e)++ def mark(self) -> Usage:+ """Snapshot of the running total — diff it with :func:`usage_delta` to see+ what a round burned even when the round itself raised half-way."""+ return self.spent.model_copy(deep=True)++ def round_costs(self, round_index: int) -> dict[str, float]:+ """``{stage: USD}`` charged against one round index (empty when none)."""+ return dict(self.by_round.get(int(round_index), {}))def elapsed_minutes(self) -> float:return (time.time() - self.start_time) / 60.0@@ -162,3 +245,23 @@ class BudgetGuard:"input_tokens": self.spent.input_tokens,"output_tokens": self.spent.output_tokens,}++ def stage_summary(self) -> dict[str, float]:+ """``{stage: USD}`` over the whole run (what the ``cost.round`` events add up to)."""+ return {k: round(v, 6) for k, v in sorted(self.by_stage.items(), key=lambda kv: -kv[1])}+++def usage_delta(after: Usage, before: Usage) -> Usage:+ """``after - before`` field by field (never negative). Used to report what a+ round burned when it raised before it could fold its own usage together."""+ return Usage(+ backend=after.backend or before.backend,+ model=after.model or before.model,+ input_tokens=max(0, after.input_tokens - before.input_tokens),+ output_tokens=max(0, after.output_tokens - before.output_tokens),+ cached_tokens=max(0, after.cached_tokens - before.cached_tokens),+ thoughts_tokens=max(0, after.thoughts_tokens - before.thoughts_tokens),+ tool_calls=max(0, after.tool_calls - before.tool_calls),+ cost_usd=max(0.0, after.cost_usd - before.cost_usd),+ latency_ms=max(0, after.latency_ms - before.latency_ms),+ )diff --git a/harness/codeverse/orchestrator/rounds.py b/harness/codeverse/orchestrator/rounds.pyindex ee457ac3..57bc9f85 100644--- a/harness/codeverse/orchestrator/rounds.py+++ b/harness/codeverse/orchestrator/rounds.py@@ -1,11 +1,28 @@"""Round loop policy: when to stop, which round is best, what to refine next.All thresholds live in ``RoundPolicy`` (one calibrated object, no scattered-ifs). ``StopPolicy.decide`` is a pure function of the round history;+ifs). ``StopPolicy.evaluate`` is a pure function of the round history;``BestSelector.pick`` is a pure function of the rounds; ``build_refine_instructions`` compiles gate errors + failed acceptance + the judge'simprovement plan into typed ``RefineTask``s, and ``plan_parallel_groups``decides which tasks may run concurrently from file ownership.++**Two money stops** were added from the cost audit (docs/COST.md §5), both+measured against the judge's own noise rather than a magic constant — the+per-model σ table lives in ONE place, ``codeverse.cost.routing.JUDGE_NOISE``+(pro 0.030, flash 0.083), and is read here through :func:`judge_sigma`:++* ``regression`` — a refine round that scored more than 1 σ *below* the best is+ not evidence that another round of the same shape will do better. The first+ such round switches strategy (one whole-artifact rewrite instead of the same+ per-part task set); a second one stops the run. 19 rounds in the audit scored+ below the best and were thrown away ($14.23); this rule governs what the loop+ is allowed to do *after* one of them (measured replay: 4 rounds re-shaped, and+ a hard stop instead would have killed the recovery round that made+ ``furn_hard_rolltop_desk`` pass).+* ``diminishing_returns`` — from r03 on, a round only runs when the previous+ round gained more than 1.5 σ and the run is still below target. r03 cost+ $33.49 per score point in the audit, against $4.93 for r01."""from __future__ import annotations@@ -26,7 +43,42 @@ from codeverse.judges.metrics import best_indexlog = logging.getLogger(__name__)-StopReason = Literal["pass", "plateau", "budget", "continue", "max_rounds", "judge_unavailable"]+StopReason = Literal["pass", "plateau", "budget", "continue", "max_rounds", "judge_unavailable",+ "regression", "diminishing_returns"]++#: what the next round should look like when the loop continues+Strategy = Literal["same", "switch"]++#: ``RoundRecord.kind`` of a round that already changed strategy after a regression+REWRITE_KIND = "rewrite"++#: σ for a judge that is not in the measured table: the default judge's+#: (``gemini-3.1-pro-preview``, ``Settings.default_judge``), because that is what+#: an unnamed judge almost always is. Calibrate a new judge+#: (``python -m codeverse.judges.calibration``) and add it to JUDGE_NOISE rather+#: than tuning the multipliers around it.+DEFAULT_JUDGE_SIGMA = 0.030+++def judge_sigma(judge_model: str = "", *, default: float = DEFAULT_JUDGE_SIGMA) -> float:+ """Measured score noise (std) of one judge model.++ The numbers come from ``judges/calibration.py`` (n=3 on the e2e rounds) and+ are tabulated ONCE in :data:`codeverse.cost.routing.JUDGE_NOISE`; this is+ the only reader in the orchestrator. Accepts a bare model name or a full+ backend id (``gemini:gemini-3.1-pro-preview``)."""+ name = (judge_model or "").strip().split(":")[-1]+ if not name:+ return default+ try:+ from codeverse.cost.routing import JUDGE_NOISE+ except Exception as e: # noqa: BLE001 — the cost package is optional at import time+ log.debug("judge noise table unavailable: %s", e)+ return default+ hit = JUDGE_NOISE.get(name)+ if hit is None: # version suffixes: gemini-3.7-flash-002 → gemini-3.7-flash+ hit = next((v for k, v in JUDGE_NOISE.items() if name.startswith(k)), None)+ return float(hit[0]) if hit else default@dataclass(frozen=True)@@ -45,6 +97,19 @@ class RoundPolicy:pairwise_margin: float = 0.03 # |Δscore| below this = judge noise → pairwise tie-breakpairwise_min_confidence: float = 0.6 # new round replaces best only when pairwise is this surejudge_samples: int = 1 # VLM judge samples per round (flash: std ≈ 0.001 between samples → 1 is enough)+ # ---- money stops (docs/COST.md §5); thresholds are multiples of the judge's measured σ+ judge_model: str = "" # judge backend id → σ via judge_sigma(); "" = the default judge's σ+ regression_sigma: float = 1.0 # a round below (best − this × σ) regressed: change shape or stop+ regression_allow_switch: bool = True # False = a regression stops the run outright+ marginal_sigma: float = 1.5 # from marginal_from_round on, the last gain must beat this × σ+ marginal_from_round: int = 3 # first refine round index the marginal-value test applies to+ # ---- agent session shape. A 28-turn cap was TESTED AND REJECTED: a controlled+ # A/B on one prompt (n=3 per arm, same generator/judge/budget) cost $1.381 mean at+ # 0.479 mean score capped, against $1.360 at 0.684 uncapped — no saving and −0.205+ # score (docs/COST.md §17). 0 = no policy cap: the session runs under the backend's+ # own AgentJob.max_turns. Callers who want one set it explicitly (cost profiles do).+ agent_max_turns: int = 0 # model turns one generation session may take (0 = uncapped)+ agent_wrapup_turns: int = 6 # turns granted to land a final build + summary when a cap IS setdef with_candidates(self, n: int | None) -> RoundPolicy:"""Copy with ``n_candidates`` set (``None`` → unchanged)."""@@ -52,6 +117,43 @@ class RoundPolicy:return selfreturn replace(self, n_candidates=max(1, int(n)))+ def with_judge(self, judge_model: str | None) -> RoundPolicy:+ """Copy that knows which judge scores the rounds (its σ sizes both money stops)."""+ if not judge_model or judge_model == self.judge_model:+ return self+ return replace(self, judge_model=judge_model)++ @property+ def sigma(self) -> float:+ """Measured judge noise for this run's judge (see :func:`judge_sigma`)."""+ return judge_sigma(self.judge_model)++ @property+ def regression_delta(self) -> float:+ """How far below the best a round must score to count as a regression."""+ return self.regression_sigma * self.sigma++ @property+ def marginal_delta(self) -> float:+ """Gain the previous round must have produced for r03+ to be worth starting."""+ return self.marginal_sigma * self.sigma+++@dataclass(frozen=True)+class StopDecision:+ """``StopPolicy``'s answer: stop (and why), or continue (and in what shape)."""++ reason: StopReason+ strategy: Strategy = "same"+ detail: str = ""++ @property+ def stop(self) -> bool:+ return self.reason != "continue"++ def event(self) -> dict[str, object]:+ return {"reason": self.reason, "strategy": self.strategy, "detail": self.detail}+class StopPolicy:"""Decide whether the loop goes on. ``history`` = rounds so far (baseline first)."""@@ -60,19 +162,92 @@ class StopPolicy:self.policy = policydef decide(self, history: Sequence[RoundRecord], *, budget_ok: bool = True) -> StopReason:+ """The stop reason only (``evaluate`` also says what shape the next round takes)."""+ return self.evaluate(history, budget_ok=budget_ok).reason++ def evaluate(self, history: Sequence[RoundRecord], *, budget_ok: bool = True) -> StopDecision:if not budget_ok:- return "budget"+ return StopDecision("budget")if not history:- return "continue"+ return StopDecision("continue")last = history[-1]if last.judgment is not None and last.judgment.passed and last.judgment.overall >= self.policy.target:- return "pass"+ return StopDecision("pass")refine_rounds = len(history) - 1if refine_rounds >= self.policy.max_rounds:- return "max_rounds"+ return StopDecision("max_rounds")+ # Order is the economics. A regression whose strategy switch is already spent+ # is the most specific answer there is, so it comes first. Otherwise the+ # marginal-value test decides late rounds (at r03 no *shape* is worth $0.5),+ # and only before that does a first regression buy one change of shape.+ # Plateau is the fallback: the same money statement, less precisely.+ exhausted = self._regression(history, switch=False)+ if exhausted is not None:+ return exhausted+ marginal = self._diminishing(history)+ if marginal is not None:+ return marginal+ regressed = self._regression(history)+ if regressed is not None:+ return regressedif self._plateaued(history):- return "plateau"- return "continue"+ return StopDecision("plateau")+ return StopDecision("continue")++ # ---------------------------------------------------------------- money stops+ def _regression(self, history: Sequence[RoundRecord], *, switch: bool = True) -> StopDecision | None:+ """The last round scored below the best by more than the judge's own noise.++ Another round of the SAME shape is not justified by that evidence: the+ first regression buys one change of strategy (a whole-artifact rewrite),+ a second one — or one after a rewrite already failed to recover — stops+ the run. ``switch=False`` asks only "is the switch already spent?", which+ is the question that outranks every other stop."""+ last = history[-1]+ score, best_before = last.score, best_score(history[:-1])+ if score is None or best_before is None:+ return None+ drop = best_before - score+ if drop <= self.policy.regression_delta:+ return None+ detail = (f"r{last.index:02d} scored {score:.3f} vs best {best_before:.3f} "+ f"({-drop:+.3f}, judge σ {self.policy.sigma:.3f})")+ if not self.policy.regression_allow_switch:+ return StopDecision("regression", detail=detail)+ if last.kind == REWRITE_KIND or self._regressions(history) > 1:+ return StopDecision("regression", detail=detail + " — a changed strategy did not recover it")+ return StopDecision("continue", strategy="switch", detail=detail) if switch else None++ def _diminishing(self, history: Sequence[RoundRecord]) -> StopDecision | None:+ """From ``marginal_from_round`` on, only run when the last round actually+ bought something (> ``marginal_sigma`` × σ) and the target is still open."""+ nxt = len(history)+ if nxt < self.policy.marginal_from_round:+ return None+ gain = last_gain(history)+ best = best_score(history)+ need = self.policy.marginal_delta+ if gain is None:+ return None+ if gain <= need:+ return StopDecision("diminishing_returns",+ detail=f"r{nxt:02d} not started: last gain {gain:+.3f} ≤ {need:.3f} "+ f"(1.5 × judge σ {self.policy.sigma:.3f})")+ if best is not None and best >= self.policy.target:+ return StopDecision("diminishing_returns",+ detail=f"r{nxt:02d} not started: best {best:.3f} already at target "+ f"{self.policy.target:.2f}")+ return None++ @staticmethod+ def _regressions(history: Sequence[RoundRecord]) -> int:+ """How many scored rounds ended below the best of everything before them."""+ n = 0+ for i in range(1, len(history)):+ s, before = history[i].score, best_score(history[:i])+ if s is not None and before is not None and s < before:+ n += 1+ return ndef _plateaued(self, history: Sequence[RoundRecord]) -> bool:"""True when the last ``plateau_window`` scored rounds did not raise the best@@ -87,25 +262,54 @@ class StopPolicy:return (after - before) < self.policy.min_delta+def best_score(history: Sequence[RoundRecord]) -> float | None:+ """Best judged score in ``history`` (``None`` when nothing was scored)."""+ scores = [r.score for r in history if r.score is not None]+ return max(scores) if scores else None+++def last_gain(history: Sequence[RoundRecord]) -> float | None:+ """How much the LAST round moved the best score (``None`` when unscored).++ Negative when it regressed; 0.0 when it changed nothing measurable."""+ if len(history) < 2 or history[-1].score is None:+ return None+ before = best_score(history[:-1])+ if before is None:+ return None+ return float(history[-1].score) - before++class BestSelector:- """Best round = highest score, then fewer gate errors, then earlier.+ """Best round = highest score, then fewer gate errors, then later.- ``codeverse.judges.metrics.best_index`` owns the ranking rule; rounds- without any score fall back to the last successfully built one."""+ ``codeverse.judges.metrics.best_index`` owns the ranking rule. A round that+ did not build is never picked (it has no score, and delivering code that does+ not run is never an improvement); when NO round has a score the fallback is+ the built round with the fewest gate errors — later on a tie — so a round that+ broke the gates cannot displace a clean earlier artifact just by being last."""def pick(self, rounds: Sequence[RoundRecord]) -> int | None:- scored = [(i, r) for i, r in enumerate(rounds) if r.score is not None]+ scored = [(i, r) for i, r in enumerate(rounds) if r.score is not None and not _build_failed(r)]if scored:k = best_index([(float(r.score), gate_error_count(r)) for _, r in scored]) # type: ignore[arg-type]return scored[int(k)][0]built = [i for i, r in enumerate(rounds) if r.build is not None and r.build.ok]- return built[-1] if built else None+ if not built:+ return None+ return min(built, key=lambda i: (gate_error_count(rounds[i]), -i))def gate_error_count(r: RoundRecord) -> int:return sum(len(g.errors) for g in r.gates)+def _build_failed(r: RoundRecord) -> bool:+ """The round is KNOWN not to build (a record with no build at all is not a failure:+ resumed/rejudged records may carry a score without one)."""+ return r.build is not None and not r.build.ok++# ----------------------------------------------------------------------------- refine tasksclass RefineTask(BaseModel):"""One targeted change request compiled for the next round."""diff --git a/harness/codeverse/spatial/tools_texture.py b/harness/codeverse/spatial/tools_texture.pyindex e3f76c6b..70f1e66d 100644--- a/harness/codeverse/spatial/tools_texture.py+++ b/harness/codeverse/spatial/tools_texture.py@@ -46,6 +46,15 @@ def _spec_plan(ctx: ToolContext) -> tuple[Spec, StaticPlan]:def texture_pass_tool(ctx: ToolContext, args: TexturePassArgs) -> Observation:glb = glb_path(ctx)spec, plan = _spec_plan(ctx)+ texture_requested = lazy("codeverse.texturing.run", "texture_requested")+ if not texture_requested(spec):+ # ONE owner for "does this run texture?" (codeverse.texturing.run.texture_requested).+ # The tool is registered for every object track, so without this an agent could —+ # and did — buy a texture pass in a run whose spec says texture: false.+ raise ToolUsageError(+ "this run did not ask for texturing (spec.options.texture is false), so the "+ "texture pass is off; finish the geometry instead",+ "3dcv make ... --texture # or `3dcv texture pass <slug>` after the run")texture_pass = lazy("codeverse.texturing.run", "texture_pass")services = ctx.extra.get("texture_services") # the ONE injection point (TextureServices)rep = texture_pass(ctx.workspace, spec, plan, model_id=args.model or spec.backends.planner,diff --git a/harness/codeverse/texturing/run.py b/harness/codeverse/texturing/run.pyindex cf968f6f..44856938 100644--- a/harness/codeverse/texturing/run.py+++ b/harness/codeverse/texturing/run.py@@ -106,6 +106,24 @@ def _measure(glb: Path) -> Measurement | None:return None+#: the ONE owner of "does this run texture?". Three call sites used to answer it+#: independently — ``tracks.lifecycle.finalise`` (spec.options.texture / a "texture" tag /+#: ctx.extra), and the ``texture_pass`` spatial tool, which the coding agent could call in+#: any run because the tool is registered for every object track. That is how a+#: ``texture: false`` run still paid for a texture pass (docs/COST.md §15: the quality run's+#: ledger was +9.5% over record.total_usage because the pass ran twice, once from inside a+#: round-2 agent session). Both now ask this function.+def texture_requested(spec: Spec) -> bool:+ """True when the run asked for the derived texture pass.++ ``Spec.options.texture`` is the switch; the legacy ``texture`` tag is still+ honoured because recorded specs carry it. Nothing else may turn texturing+ on — an agent calling the ``texture_pass`` tool in a run that did not ask for+ it is refused, and ``3dcv texture pass <slug>`` is an explicit user+ instruction that does not go through here at all."""+ return bool(spec.options.texture or "texture" in (spec.tags or []))++def texture_pass(ws: Workspace,spec: Spec,diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.pyindex ff87c0d7..653adade 100644--- a/harness/codeverse/tracks/generation.py+++ b/harness/codeverse/tracks/generation.py@@ -11,15 +11,36 @@Neither path commits; the round commits once after all (possibly parallel)tasks finished. Neither path builds; see ``tracks/repair.py``.++**Turn budget — off by default, measured.** The audit's turn-≥20 share (39.3%+of the agent bill, docs/COST.md §4) looked like $14 of waste, so wave 2 capped+every session at 28 turns. A controlled A/B rejected it — same prompt, n=3 per+arm: capped $1.381 / **0.479**, uncapped $1.360 / **0.684** (docs/COST.md §17).+A session cut off mid-task leaves work the next round pays for again.++So there is **no default cap** (``DEFAULT_AGENT_MAX_TURNS`` and+``RoundPolicy.agent_max_turns`` are 0 → the backend's own ``AgentJob.max_turns``).+A caller may still set one (``GenerationTask.max_turns`` > ``generate(max_turns=…)``+> ``CV3D_AGENT_MAX_TURNS`` > ``Settings.limits.agent_max_turns``, which a cost+profile sets), and a cap that IS set stays **graceful**: the session is not killed+but asked, in a short wrap-up (``agent_wrapup_turns``), for one last build + summary.++**Every dollar is charged.** Both strategies spend through+``BudgetGuard.spend`` with their stage / label / round, so a retried session+(``<label>.a2``) and a wrap-up session are visible to the guard, the record and+the ledger even when the *next* attempt then raises."""from __future__ import annotations+import jsonimport logging+import osimport subprocessimport threadingimport timefrom collections.abc import Callable+from pathlib import Pathfrom typing import Anyfrom pydantic import BaseModel, Field@@ -42,6 +63,53 @@ log = logging.getLogger(__name__)SINGLE_SHOT_PREFIX = "single-shot:"+#: default cap on model turns per agent session: 0 = none, the session runs under the+#: backend's own ``AgentJob.max_turns``. A 28-turn default was measured and rejected+#: (A/B: no saving, −0.205 score) — see the module docstring and docs/COST.md §17.+DEFAULT_AGENT_MAX_TURNS = 0+#: turns granted to the wrap-up session that lands a final build + summary when a cap IS set+DEFAULT_WRAPUP_TURNS = 6++WRAPUP_PROMPT = (+ "You have reached the turn budget for this task, so this is your LAST session on it.\n"+ "Do NOT start new work and do NOT begin any refactor.\n"+ "1. Make sure every file you edited is written to disk and self-consistent.\n"+ "2. Run the build tool ONCE. If it fails, make the smallest edit that makes it build "+ "(or revert your last incomplete edit) and build once more.\n"+ "3. Reply with a short summary: what you changed, what still does not match the task, "+ "and the single next step you would take.\n\n"+ "The task you were working on, for reference:\n"+)+++def agent_max_turns(default: int = DEFAULT_AGENT_MAX_TURNS) -> int:+ """The turn cap a machine/profile asks for: ``CV3D_AGENT_MAX_TURNS`` >+ ``Settings.limits.agent_max_turns`` (what a cost profile sets) > ``default``.++ ``0`` means *no cap*: the caller leaves ``AgentJob.max_turns`` at the backend's+ own default. That is the measured default (docs/COST.md §17)."""+ raw = os.environ.get("CV3D_AGENT_MAX_TURNS", "").strip()+ if raw.isdigit() and int(raw) > 0:+ return int(raw)+ try:+ from codeverse.config import get_settings++ configured = int(getattr(get_settings().limits, "agent_max_turns", 0) or 0)+ except Exception as e: # noqa: BLE001 — settings must never break a generation+ log.debug("agent turn cap fell back to the default: %s", e)+ configured = 0+ return max(0, configured or int(default))+++def turn_capped(res: Any) -> bool:+ """True when the agent stopped because it ran out of turns (not money/time).++ api-agent says ``max_turns (N) reached``; claude-code reports the subtype+ ``error_max_turns`` — both surface as ``exit_reason="budget"``."""+ if getattr(res, "exit_reason", "") != "budget":+ return False+ return any("max_turns" in str(e) for e in (getattr(res, "errors", None) or []))+class GenerationTask(BaseModel):"""One unit of code-writing work (baseline, refine group, asset, zone, repair)."""@@ -57,6 +125,7 @@ class GenerationTask(BaseModel):thinking: str = "medium"max_output_tokens: int = 32000timeout_s: int | None = None+ max_turns: int = Field(default=0, description="model turns for an agent session (0 = the harness default)")write_roots: list[str] = Field(default_factory=lambda: ["src", "public"])@@ -69,6 +138,9 @@ class GenerationResult(BaseModel):text: str = Field(default="", description="model/agent final text (truncated)")transcript_path: str = ""label: str = ""+ turns: int = Field(default=0, description="turns the agent session(s) took, as the backend counts them")+ sessions: int = Field(default=0, description="agent sessions run for this task (retry / wrap-up count)")+ turn_capped: bool = Field(default=False, description="a session hit the turn budget and was wrapped up")# ----------------------------------------------------------------------------- git helpers@@ -138,12 +210,13 @@ def generate_files(temperature=task.temperature,thinking=task.thinking, # type: ignore[arg-type]max_output_tokens=task.max_output_tokens,- label=task.label,+ # the round tag is the cost ledger's only way to attribute a single-shot call:+ # it has no agent session to inherit an ambient round from (codeverse.cost.context)+ label=f"{task.label}:r{task.round:02d}",)resp = model.generate(req)usage = resp.usage- if budget is not None:- budget.charge(usage)+ _charge(budget, usage, task=task, label=task.label, outcome=str(resp.finish_reason or "ok"))if _is_truncated(resp):# cut off by max_output_tokens: the envelope is unterminated — one retry with# a doubled budget beats writing a half-file and burning repair attempts on it@@ -152,8 +225,7 @@ def generate_files(req = req.model_copy(update={"max_output_tokens": min(task.max_output_tokens * 2, 65536)})resp = model.generate(req)usage = usage + resp.usage- if budget is not None:- budget.charge(resp.usage)+ _charge(budget, resp.usage, task=task, label=f"{task.label}.retry", outcome="truncated")traj = ws.trajectory_dir(task.label.replace("/", "_"), task.round)(traj / "prompt.md").write_text(f"# system\n{system}\n\n# user\n{task.prompt}\n")(traj / "response.md").write_text(resp.text or "")@@ -185,6 +257,33 @@ def generate_files(text=(resp.text or "")[:2000], transcript_path=str(traj / "response.md"), label=task.label)+#: task kinds whose money belongs to a differently-named stage of the cost vocabulary+_STAGE_FOR_KIND = {"rebuild": "repair", "asset_fix": "assets", "asset": "assets",+ "zone": "zones", "generate": "baseline", "compose": "assemble"}+++def task_stage(task: GenerationTask) -> str:+ """The cost stage one generation task spends in (a ``codeverse.cost.types.Stage``+ value when we can name one, else the label — which ``record_call`` maps through+ ``stage_for_label`` (``asset_koi`` → assets, ``r00_baseline_repair1`` → repair))."""+ kind = task.kind or ""+ stage = _STAGE_FOR_KIND.get(kind, kind)+ try:+ from codeverse.cost.types import Stage++ return str(Stage(stage))+ except Exception: # noqa: BLE001 — unknown kind (or no cost package): let the label decide+ return task.label or stage or "other"+++def _charge(budget: Any | None, usage: Usage, *, task: GenerationTask, label: str, outcome: str = "ok") -> None:+ """Spend through the guard with the stage/label/round this task belongs to."""+ if budget is None:+ return+ budget.charge(usage, stage=task_stage(task), role="generator", label=label,+ round_index=task.round, outcome=outcome)++_TRUNCATED_FINISH = {"max_tokens", "max_output_tokens", "length"}@@ -201,10 +300,23 @@ def run_agent_task(budget: Any | None = None,events: Any | None = None,retry_silent_bail: bool = True,+ max_turns: int = 0,+ wrapup_turns: int = DEFAULT_WRAPUP_TURNS,) -> GenerationResult:- """CodingAgent path. ``ok`` = the agent changed files (envelope-vs-disk truth)."""+ """CodingAgent path. ``ok`` = the agent changed files (envelope-vs-disk truth).++ A turn cap is applied only when a caller asks for one (``task.max_turns`` >+ ``max_turns`` > ``CV3D_AGENT_MAX_TURNS`` > ``Settings.limits.agent_max_turns``+ > :data:`DEFAULT_AGENT_MAX_TURNS`, which is 0 = the backend's own+ ``AgentJob.max_turns``; a 28-turn default was measured and rejected, see the+ module docstring). Hitting whatever cap is in force is not a failure: one+ short wrap-up session is asked for a final build and a summary, so the money+ already spent leaves a buildable workspace instead of a session that was cut+ off mid-edit. Every session — first attempt, silent-bail retry, wrap-up — is+ charged to ``budget`` as it ends, so nothing is lost when a later attempt raises."""before = ws.head()timeout = task.timeout_s or (settings.limits.agent_timeout_s if settings is not None else 1800)+ turns_cap = task.max_turns or max_turns or agent_max_turns() # 0 = leave AgentJob's own default# typed job context honoured by every CodingAgent: round → trajectory dir + ToolContext,# language/track → spatial tool filtering, files_hint → per-session attribution of# files_changed when tasks run concurrently in ONE workspace (see agents/cli_common).@@ -224,30 +336,120 @@ def run_agent_task(model=getattr(agent, "model", ""), label=task.label, timeout_s=timeout,spatial_tools=True, write_roots=task.write_roots,round=task.round, kind=task.kind, language=language, track=track,- files_hint=list(task.files_hint), extra=extra)- res: AgentResult = agent.run(job)- usage = res.usage- if budget is not None:- budget.charge(res.usage)+ files_hint=list(task.files_hint), extra=extra,+ **({"max_turns": turns_cap} if turns_cap > 0 else {}))+ acc = _SessionAcc(task=task, budget=budget)+ res = acc.run(agent, job)changes = res.files_changed or _attributed_fallback(ws, task, before)- if retry_silent_bail and not changes and res.exit_reason not in ("timeout", "budget"):++ if turn_capped(res):+ # the turns are gone, the money is not wasted: ask for a landing, not more work+ if events is not None:+ # NB: not ``cost_usd`` — an event carrying that key is priced as its own+ # call by ``cost.reconstruct``, and this money is already in the session row.+ events.emit("generate.turn_cap", label=task.label, round=task.round, max_turns=job.max_turns,+ turns=acc.turns, session_usd=round(acc.usage.cost_usd, 4))+ wrap = job.model_copy(update={"prompt": WRAPUP_PROMPT + task.prompt,+ "max_turns": max(2, int(wrapup_turns))})+ res = acc.run(agent, wrap, wrapup=True, optional=True) or res+ changes = res.files_changed or changes or _attributed_fallback(ws, task, before)++ if retry_silent_bail and not changes and not acc.wrapped and res.exit_reason not in ("timeout", "budget"):if events is not None:events.emit("generate.silent_bail", label=task.label, exit_reason=res.exit_reason)job2 = job.model_copy(update={"prompt": ("Your previous attempt ended WITHOUT writing any file. You must create/edit the files ""described below and run the build tool before finishing.\n\n" + task.prompt)})- res = agent.run(job2)- usage = usage + res.usage- if budget is not None:- budget.charge(res.usage)+ res = acc.run(agent, job2, optional=True) or reschanges = res.files_changed or _attributed_fallback(ws, task, before)+notes = f"exit={res.exit_reason}" + (f"; errors={res.errors[:3]}" if res.errors else "")+ notes += ("; " + "; ".join(acc.errors)) if acc.errors else ""+ if acc.wrapped:+ notes += f"; turn cap {job.max_turns} reached → wrap-up session"if events is not None:events.emit("generate.done", label=task.label, strategy=getattr(agent, "kind", "agent"),files=[c.path for c in changes], ok=bool(changes), exit_reason=res.exit_reason,- cost_usd=round(usage.cost_usd, 4))- return GenerationResult(ok=bool(changes), usage=usage, files_changed=changes, notes=notes,- text=(res.text or "")[:2000], transcript_path=res.transcript_path, label=task.label)+ turns=acc.turns, sessions=acc.sessions, turn_capped=acc.wrapped,+ cost_usd=round(acc.usage.cost_usd, 4))+ return GenerationResult(ok=bool(changes), usage=acc.usage, files_changed=changes, notes=notes,+ text=(res.text or "")[:2000], transcript_path=res.transcript_path, label=task.label,+ turns=acc.turns, sessions=acc.sessions, turn_capped=acc.wrapped)+++class _SessionAcc:+ """Runs the agent sessions of ONE task and keeps their accounting.++ Each session is charged the moment it returns (label ``<task>``/``<task>.a2``+ for the retry, ``<task>.wrapup`` for the landing session), so a crash in a+ later session can never erase what an earlier one already spent — the ``.a2``+ hole of docs/COST.md §6."""++ def __init__(self, *, task: GenerationTask, budget: Any | None):+ self.task = task+ self.budget = budget+ self.usage = Usage()+ self.turns = 0+ self.sessions = 0+ self.wrapped = False+ self.errors: list[str] = []++ def run(self, agent: Any, job: AgentJob, *, wrapup: bool = False, optional: bool = False) -> AgentResult | None:+ """One session. ``optional`` sessions (wrap-up, silent-bail retry) never+ take the task down: a crash there leaves the earlier, already charged+ session as the result instead of discarding money that was really spent."""+ self.sessions += 1+ self.wrapped = self.wrapped or wrapup+ label = self.task.label + (".wrapup" if wrapup else (f".a{self.sessions}" if self.sessions > 1 else ""))+ try:+ res: AgentResult = agent.run(job)+ except Exception as e: # noqa: BLE001 — a budget stop still propagates (see below)+ self.sessions -= 1+ if not optional or _is_budget_stop(e):+ raise+ log.warning("optional agent session %s failed: %s: %s", label, type(e).__name__, e)+ self.errors.append(f"{label}: {type(e).__name__}: {e}")+ return None+ self.usage = self.usage + res.usage+ self.turns += session_turns(res)+ self.charge(res.usage, label=label, outcome=res.exit_reason or "ok")+ return res++ def charge(self, usage: Usage, *, label: str, outcome: str) -> None:+ if self.budget is None or not (usage.cost_usd or usage.input_tokens or usage.output_tokens):+ return+ self.budget.charge(usage, stage=task_stage(self.task), role="generator",+ label=label, round_index=self.task.round, outcome=outcome)+++def _is_budget_stop(e: BaseException) -> bool:+ from codeverse.orchestrator.budget import BudgetExceeded++ return isinstance(e, BudgetExceeded)+++def session_turns(res: Any) -> int:+ """The session's turn count as the BACKEND reports it — the ``turns`` field+ ``agents/cli_common.finish_session`` writes into ``result.json`` (0 when the+ backend does not report one).++ Backends count slightly differently (the api-agent reports transcript+ messages, roughly two per model turn), so this is a size signal for the+ ``cost.round`` event, not the number the turn cap is compared against —+ that one is enforced inside the session by ``job.max_turns``."""+ path = getattr(res, "transcript_path", "") or ""+ if not path:+ return 0+ p = Path(path)+ result = (p if p.is_dir() else p.parent) / "result.json"+ try:+ if result.is_file():+ value = json.loads(result.read_text(errors="replace")).get("turns")+ if isinstance(value, (int, float)):+ return int(value)+ except (OSError, ValueError) as e: # noqa: PERF203 - best effort telemetry+ log.debug("session turns unreadable (%s): %s", result, e)+ return 0def _attributed_fallback(ws: Workspace, task: GenerationTask, before: str) -> list[FileChange]:@@ -276,6 +478,8 @@ def generate(events: Any | None = None,model_factory: Callable[[str], Any] | None = None,agent_factory: Callable[[str], Any] | None = None,+ max_turns: int = 0,+ wrapup_turns: int = DEFAULT_WRAPUP_TURNS,) -> GenerationResult:"""Dispatch to the single-shot or the coding-agent strategy by ``agent_id``.@@ -291,7 +495,8 @@ def generate(if agent is None:factory = agent_factory or _default_agent_factoryagent = factory(agent_id)- return run_agent_task(ws, agent=agent, task=task, settings=settings, budget=budget, events=events)+ return run_agent_task(ws, agent=agent, task=task, settings=settings, budget=budget, events=events,+ max_turns=max_turns, wrapup_turns=wrapup_turns)def _default_model_factory(model_id: str) -> Any:diff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.pyindex 3b2a66ce..e9afd078 100644--- a/harness/codeverse/tracks/lifecycle.py+++ b/harness/codeverse/tracks/lifecycle.py@@ -27,11 +27,15 @@ from codeverse.contracts.spec import Specfrom codeverse.events import EventLogfrom codeverse.orchestrator.budget import BudgetExceeded, BudgetGuardfrom codeverse.orchestrator.rounds import (+ REWRITE_KIND,BestSelector,+ RefineTask,RoundPolicy,StopPolicy,StopReason,+ Strategy,TaskGroup,+ best_score,build_refine_instructions,plan_refine_groups,)@@ -67,9 +71,31 @@ log = logging.getLogger(__name__)REFERENCE_RUBRIC = "reference_v1"+#: fallback name of the run's live cost ledger when the cost package cannot place+#: one itself (``flywheel/telemetry.py`` publishes it as ``telemetry/usage.jsonl``)+LEDGER_NAME = "cost_ledger.jsonl"+++def run_ledger_path(ws: Workspace) -> Any:+ """Where this run's priced per-call rows go.++ ``codeverse.cost`` owns the run layout (``telemetry/cost.jsonl`` + the+ ``cost_ledger.jsonl`` alias); fall back to the alias name if it cannot."""+ try:+ from codeverse.cost.ledger import open_run_ledger++ return open_run_ledger(ws.root)+ except Exception as e: # noqa: BLE001 — accounting must never fail a run+ log.debug("cost ledger placement fell back to %s: %s", LEDGER_NAME, e)+ return ws.root / LEDGER_NAME++_STATUS: dict[str, RunStatus] = {"pass": RunStatus.PASSED, "plateau": RunStatus.PLATEAU, "max_rounds": RunStatus.PLATEAU,"budget": RunStatus.BUDGET,+ # the two money stops (orchestrator/rounds.py): the code is intact and the best+ # round is delivered — the run stopped because another round was not worth buying.+ "regression": RunStatus.PLATEAU, "diminishing_returns": RunStatus.PLATEAU,# judge outage (degraded/crashed verdicts even after a retry): the code is intact# and the best built round is delivered — a stop, not a failure."judge_unavailable": RunStatus.PLATEAU,@@ -191,8 +217,14 @@ class BaseTrack:raise NotImplementedError# ------------------------------------------------------------------ refine scaffold- def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord]) -> tuple[list[GenerationTask], list[str]]:- """index → rebuild guard → instructions → groups → per-track ``_refine_task``."""+ def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord],+ *, strategy: Strategy = "same") -> tuple[list[GenerationTask], list[str]]:+ """index → rebuild guard → instructions → groups → per-track ``_refine_task``.++ ``strategy="switch"`` is what the loop asks for after a regression: the same+ evidence, but ONE whole-artifact task instead of the per-part set that just+ made the score worse (docs/COST.md §5 — repeating the shape is what the 19+ regression rounds did)."""index = len(history)if last.build is None or not last.build.ok:return [self._rebuild_task(ctx, last, index)], ["rebuild: previous round did not build"]@@ -202,11 +234,15 @@ class BaseTrack:extra=self.extra_refine_tasks(ctx, last))if not tasks:return [], []- groups, parallel = plan_refine_groups(tasks, allow_fanout=self.allow_refine_fanout,- parallel_min_tasks=ctx.policy.parallel_min_tasks)+ if strategy == "switch":+ tasks = [rewrite_task(last, best_score(history))] + tasks+ groups, parallel = [TaskGroup(tasks=tasks, files=[])], False+ else:+ groups, parallel = plan_refine_groups(tasks, allow_fanout=self.allow_refine_fanout,+ parallel_min_tasks=ctx.policy.parallel_min_tasks)gen_tasks = [self._refine_task(ctx, g, last, index, parallel=parallel) for g in groups]ctx.events.emit("refine.planned", round=index, n_tasks=len(tasks), n_groups=len(groups), parallel=parallel,- targets=[g.targets for g in groups])+ strategy=strategy, targets=[g.targets for g in groups])return gen_tasks, [t.line() for t in tasks]def _rebuild_task(self, ctx: RunContext, last: RoundRecord, index: int) -> GenerationTask:@@ -268,12 +304,14 @@ class BaseTrack:def build_context(self, spec: Spec, ws: Workspace, events: EventLog, state: RunState) -> RunContext:settings = self._settings or get_settings()runtime = self._runtime or self.services.runtime(spec.language)- budget = BudgetGuard(spec.budget, soft_fraction=self.soft_budget_fraction)+ budget = BudgetGuard(spec.budget, soft_fraction=self.soft_budget_fraction,+ run=ws.root.name, ledger=run_ledger_path(ws))spent = state.extra.get("spent_usage")if spent:budget.spent = Usage.model_validate(spent)n_cand = self._resolve_candidates(spec, state, settings)- policy = (self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds)).with_candidates(n_cand)+ policy = ((self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds))+ .with_candidates(n_cand).with_judge(spec.backends.judge))# reference images: static objects are scored with reference_v1 (adds the measured silhouette# criterion); other tracks keep their rubric but the judge still sees the references.rubric = REFERENCE_RUBRIC if spec.references and self.track is Track.STATIC_OBJECT else self.rubric@@ -381,10 +419,14 @@ class BaseTrack:selector = BestSelector()rejudged: set[int] = set()while True:- decision = stop_policy.decide(rounds, budget_ok=ctx.budget.ok()) if rounds else "continue"- if decision != "continue":- ctx.events.emit("stop", reason=decision, rounds=len(rounds), best=ctx.state.best_round)- return decision+ decision = stop_policy.evaluate(rounds, budget_ok=ctx.budget.ok())+ if decision.stop:+ ctx.events.emit("stop", rounds=len(rounds), best=ctx.state.best_round, **decision.event())+ return decision.reason+ if decision.strategy == "switch":+ # the last round regressed by more than the judge's noise: one round of a+ # DIFFERENT shape (whole-artifact rewrite), never another of the same.+ ctx.events.emit("strategy.switch", round=len(rounds), reason="regression", detail=decision.detail)index = len(rounds)previous = rounds[-1].judgment if rounds else Noneif index == 0:@@ -404,8 +446,8 @@ class BaseTrack:ctx.events.emit("best.updated", round=best, score=rounds[best].score)self._save_spent(ctx)continue # decide() re-runs with the recovered score- tasks, instructions = self.refine_tasks(ctx, last, rounds)- kind = "refine"+ tasks, instructions = self.refine_tasks(ctx, last, rounds, strategy=decision.strategy)+ kind = REWRITE_KIND if decision.strategy == "switch" else "refine"if not tasks:if last.judgment is None and last.build is not None and last.build.ok:# no tasks only because the judge never scored the round: stopping@@ -422,7 +464,7 @@ class BaseTrack:else:rec = run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,previous=previous, files_hint=self.round_files_hint(ctx),- extra_notes=self.round_extra_notes(ctx))+ extra_notes=self.round_extra_notes(ctx), previous_best=best_score(rounds))except RoundFailed as e:if index == 0:raise@@ -490,7 +532,7 @@ class BaseTrack:ctx.events.emit("finalise.rebuild", round=best, ok=build.ok)except Exception as e: # noqa: BLE001 — the best round already built once; report, don't failctx.events.emit("finalise.rebuild_failed", error=f"{type(e).__name__}: {e}")- if rounds and (ctx.spec.options.texture or "texture" in ctx.spec.tags or ctx.extra.get("texture")):+ if rounds and self._texture_wanted(ctx):self._texture_pass(ctx)ctx.state.status, ctx.state.stop_reason, ctx.state.error = status, stop_reason, errorctx.state.save(ctx.ws)@@ -512,9 +554,35 @@ class BaseTrack:log.warning("changed_files failed in finalise: %s", e)return False+ @staticmethod+ def _texture_wanted(ctx: RunContext) -> bool:+ """``codeverse.texturing.run.texture_requested`` is the single owner of this+ decision (spec.options.texture / the legacy tag) — the same predicate the+ ``texture_pass`` spatial tool asks, so a ``texture: false`` run cannot be+ textured from inside an agent session either."""+ try:+ from codeverse.texturing.run import texture_requested++ return texture_requested(ctx.spec)+ except Exception as e: # noqa: BLE001 — texturing is optional; never block finalise+ log.debug("texture_requested unavailable: %s", e)+ return bool(ctx.spec.options.texture)+def _texture_pass(self, ctx: RunContext) -> None:- """Optional post-hoc texture pass (spec tag 'texture'). Additive: any failure- is logged + emitted, never fatal; usage is added to the budget totals."""+ """Optional post-hoc texture pass (``Spec.options.texture``). Additive: any+ failure is logged + emitted, never fatal.++ Its usage goes through the SAME guard as everything else (stage ``texture``,+ role ``image``) so it lands in ``record.total_usage`` and in the ledger — four+ post-hoc passes on ``furn_hard_rolltop_desk`` ($0.656) were invisible to both+ (docs/COST.md §6). It is charged without enforcement: the run is finalising,+ the tiles are already paid for, and raising here would throw away the pack."""+ if self._already_textured(ctx):+ # An agent can call the `texture_pass` spatial tool during a round; the+ # finalise hook must not buy the same pack a second time (a texture pass+ # is $0.10-0.30 and the input GLB is unchanged).+ ctx.events.emit("texture.skipped", reason="already_textured_this_artifact")+ returntry:from codeverse.texturing.run import texture_pass@@ -525,21 +593,58 @@ class BaseTrack:log.warning("texture pass failed: %s", e)ctx.events.emit("texture.failed", error=f"{type(e).__name__}: {e}")return- ctx.budget.add(trep.usage)+ ctx.budget.add(trep.usage, stage="texture", role="image", label="texture_pass")ctx.extra["texturing"] = trep.summary()+ if not ctx.budget.ok():+ # visible, but NOT `budget.exceeded`: that event means "the loop stopped",+ # and a texture pass runs after the loop has already finished.+ ctx.events.emit("budget.overrun", stage="texture",+ reason=f"the texture pass took the run past its ceiling "+ f"(${ctx.budget.spent.cost_usd:.3f} of ${ctx.budget.hard_usd:.2f})",+ spent_usd=round(ctx.budget.spent.cost_usd, 4))++ @staticmethod+ def _already_textured(ctx: RunContext) -> bool:+ """True when a texture pass already ran against the current ``object.glb``.++ The report (``artifacts/texturing.json``) records the GLB it consumed; the+ hook compares content hashes so a texture pass from an EARLIER round (whose+ GLB has since been rebuilt) does not suppress the final one."""+ import hashlib+ import json as _json++ report = ctx.ws.artifacts / "texturing.json"+ glb = ctx.ws.artifacts / "object.glb"+ if not report.is_file() or not glb.is_file():+ return False+ try:+ data = _json.loads(report.read_text())+ prev = Path(str(data.get("glb_in") or ""))+ if not prev.is_file():+ return False+ digest = lambda p: hashlib.sha256(p.read_bytes()).hexdigest() # noqa: E731+ return digest(prev) == digest(glb)+ except Exception: # noqa: BLE001 — a malformed report must not block texturing+ return Falsedef _record(self, ctx: RunContext, rounds: list[RoundRecord], status: RunStatus, *, error: str, stop_reason: str) -> RunRecord:best = ctx.state.best_roundbaseline = rounds[0].score if rounds else Nonefinal = rounds[best].score if best is not None and best < len(rounds) else Nonetotal = sum_usage(rounds)- # planner + other non-round calls are in the budget guard but not in rounds+ # planner, aborted rounds, retried sessions, the texture pass: all in the guard,+ # none of them in ``rounds`` — the guard is the honest total (docs/COST.md §6).if ctx.budget.spent.cost_usd > total.cost_usd:total = ctx.budget.spentextra: dict[str, Any] = {"stop_reason": stop_reason, "rubric": ctx.rubric, "budget": ctx.budget.summary(),- "n_candidates": ctx.policy.n_candidates, "candidates": ctx.state.extra.get("candidates")}+ "n_candidates": ctx.policy.n_candidates, "candidates": ctx.state.extra.get("candidates"),+ "cost_by_stage": ctx.budget.stage_summary()}if ctx.extra.get("texturing"):extra["texturing"] = ctx.extra["texturing"]+ if ctx.extra.get("aborted_rounds"):+ # rounds the budget (or a crash) cut after the money was spent: they are not+ # in ``rounds``, so name them here and keep their dollars in total_usage.+ extra["aborted_rounds"] = ctx.extra["aborted_rounds"]return RunRecord(spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, best_round=best,baseline_score=baseline, final_score=final, total_usage=total,@@ -551,5 +656,17 @@ class BaseTrack:)+def rewrite_task(last: RoundRecord, best: float | None) -> RefineTask:+ """The instruction that turns a repeat into a change of approach."""+ was = f"{last.score:.3f}" if last.score is not None else "no score"+ peak = f"{best:.3f}" if best is not None else "the earlier best"+ return RefineTask(+ target="overall", kind="rewrite", priority=0, source="judge",+ instruction=(f"The previous round made the artifact WORSE ({was} against {peak}) — its edits are not the "+ "way forward. Do NOT repeat them part by part: re-read the plan and the failures below, then "+ "rewrite the whole artifact coherently, keeping only what the evidence shows was already right."),+ )++def generator_label(agent_id: str) -> str:return "single-shot" if is_single_shot(agent_id) else agent_id.split(":", 1)[0]diff --git a/harness/codeverse/tracks/repair.py b/harness/codeverse/tracks/repair.pyindex 31f0299f..11000120 100644--- a/harness/codeverse/tracks/repair.py+++ b/harness/codeverse/tracks/repair.py@@ -35,6 +35,7 @@ class RepairOutcome(BaseModel):attempts: list[GenerationResult] = Field(default_factory=list)usage: Usage = Field(default_factory=Usage)repaired: bool = False+ max_attempts: int = Field(default=0, description="repair budget this outcome ran under")@propertydef ok(self) -> bool:@@ -149,7 +150,7 @@ def build_with_repair(ctx: RunContext, *, round_index: int, label: str, files_hibuild, lint = build_once(ctx)ctx.events.emit("build.done", round=round_index, ok=build.ok, lint_errors=len(lint.errors),error=build.error_message[:200], duration_ms=build.duration_ms)- outcome = RepairOutcome(build=build, lint=lint)+ outcome = RepairOutcome(build=build, lint=lint, max_attempts=max_attempts)prev_sig = ""repeats = 0attempt = 0@@ -162,7 +163,8 @@ def build_with_repair(ctx: RunContext, *, round_index: int, label: str, files_hilabel=f"{label}_repair{attempt}", files_hint=files_hint)ctx.events.emit("repair.attempt", round=round_index, attempt=attempt, repeats=repeats, signature=sig[:200])res = generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model,- settings=ctx.settings, budget=ctx.budget, events=ctx.events)+ settings=ctx.settings, budget=ctx.budget, events=ctx.events,+ max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)outcome.attempts.append(res)outcome.usage = outcome.usage + res.usageif not res.ok:diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.pyindex f2da07cb..f4a08b60 100644--- a/harness/codeverse/tracks/steps.py+++ b/harness/codeverse/tracks/steps.py@@ -4,6 +4,18 @@gates / measurement / rendering. Every step writes its artifact under theworkspace (gates/rNN, renders/rNN, judge/rNN.json, rounds/rNN.json) so a runcan be resumed and the flywheel can replay it.++Two cost rules live here (docs/COST.md §5, §6):++* **never pay for a verdict you will not use** — :func:`skip_judge_reason` drops+ the judge only where the verdict is provably never bought at all, and+ :func:`rejudge_round` re-buys one only after the judge was TRIED and failed, so+ a skip is a saving and not a deferral. Two wave-2 branches were removed for+ failing exactly that test (docs/COST.md §17).+* **every round reports what it burned** — :func:`emit_round_cost` writes one+ ``cost.round`` event ({stage → $}, judge $, agent turns, wasted flag) at the+ end of the round *and* when the round raises half-way, so the ledger never has+ to reconstruct a round the budget cut."""from __future__ import annotations@@ -20,6 +32,7 @@ from codeverse.contracts.common import Usagefrom codeverse.contracts.judgment import Judgmentfrom codeverse.contracts.plan import AcceptanceItem, Planfrom codeverse.contracts.run import RoundRecord+from codeverse.orchestrator.budget import usage_deltafrom codeverse.orchestrator.fanout import fan_outfrom codeverse.tracks.common import RunContextfrom codeverse.tracks.generation import GenerationResult, GenerationTask, generate@@ -71,7 +84,8 @@ def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> lidef _one(task: GenerationTask) -> GenerationResult:return generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model,- settings=ctx.settings, budget=ctx.budget, events=ctx.events)+ settings=ctx.settings, budget=ctx.budget, events=ctx.events,+ max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)# fan_out also for ONE task: a crashing generator (503 storm, parse error) becomes a failed# result → RoundFailed (refine rounds treat that as a plateau) instead of killing the run.@@ -105,34 +119,75 @@ def run_round(files_hint: Sequence[str] = (),extra_usage: Usage | None = None,extra_notes: Sequence[str] = (),+ previous_best: float | None = None,) -> RoundRecord:"""Execute one round and persist its record. Budget is charged as it goes.``tasks`` may be empty when the code is already in place (scene stages,best-of-N winner copied in): the round is then build → gates → render →judge only. ``extra_usage`` / ``extra_notes`` fold pre-round work- (candidate generation) into the record."""+ (candidate generation) into the record; ``previous_best`` is the best score+ before this round, used only to flag the round as wasted in ``cost.round``."""+ mark = ctx.budget.mark()+ try:+ return _run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,+ previous=previous, files_hint=files_hint, extra_usage=extra_usage,+ extra_notes=extra_notes, previous_best=previous_best)+ except BaseException as e:+ # the round died half-way (budget stop, 503 storm, RoundFailed). Whatever it+ # burned is already in the guard: report it so the round is not invisible.+ burned = usage_delta(ctx.budget.spent, mark)+ record_aborted_round(ctx, index=index, kind=kind, usage=burned, error=f"{type(e).__name__}: {e}")+ raise+++def _run_round(+ ctx: RunContext,+ *,+ index: int,+ kind: str,+ tasks: Sequence[GenerationTask],+ pipeline: RoundPipeline,+ instructions: Sequence[str] = (),+ previous: Judgment | None = None,+ files_hint: Sequence[str] = (),+ extra_usage: Usage | None = None,+ extra_notes: Sequence[str] = (),+ previous_best: float | None = None,+) -> RoundRecord:t0 = time.time()ctx.events.emit("round.start", round=index, kind=kind, n_tasks=len(tasks))rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, instructions=list(instructions),started_at=datetime.now(UTC))usage = extra_usage or Usage()notes: list[str] = list(extra_notes)+ cost: dict[str, float] = {}+ if extra_usage is not None and extra_usage.cost_usd:+ cost["candidates"] = round(extra_usage.cost_usd, 6)gens = run_generation_tasks(ctx, tasks)+ turns = 0for g in gens:usage = usage + g.usage+ turns += g.turnsif not g.ok:notes.append(f"{g.label}: {g.notes}")+ if gens:+ cost["generate"] = round(sum(g.usage.cost_usd for g in gens), 6)ctx.ws.commit(f"r{index:02d} {kind}: generated")outcome: RepairOutcome = build_with_repair(ctx, round_index=index, label=f"r{index:02d}_{kind}",files_hint=list(files_hint))usage = usage + outcome.usage+ if outcome.usage.cost_usd:+ cost["repair"] = round(outcome.usage.cost_usd, 6)rec.build = outcome.buildgates: list[GateReport] = [outcome.lint]if outcome.attempts:- notes.append(f"repair attempts: {len(outcome.attempts)} ({'fixed' if outcome.ok else 'still failing'})")+ notes.append(+ f"repair attempts: {len(outcome.attempts)}/{outcome.max_attempts} "+ f"({'fixed' if outcome.ok else 'still failing'})"+ )if outcome.build.ok:rec.measurement = pipeline.measure(ctx, outcome.build)@@ -144,18 +199,20 @@ def run_round(n_err = sum(len(g.errors) for g in gates)ctx.events.emit("gates.done", round=index, n_gates=len(gates), n_errors=n_err,tri_count=rec.measurement.tri_count if rec.measurement else None)- if ctx.judge is not None and rec.renders is not None and rec.renders.views and (n_err == 0 or ctx.policy.judge_on_gate_errors):+ skip = skip_judge_reason(ctx, gates=gates, renders=rec.renders)+ if skip:+ notes.append(f"judge skipped ({skip})")+ ctx.events.emit("judge.skipped", round=index, reason=skip)+ else:rec.judgment = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous)if rec.judgment is not None:usage = usage + rec.judgment.usage+ cost["judge"] = round(rec.judgment.usage.cost_usd, 6)# add, never charge: the verdict exists and is paid for — raising here# would drop a fully judged round before it is committed/recorded# (the loop stops at its next budget_ok check instead, AFTER best promotion).- ctx.budget.add(rec.judgment.usage)- elif ctx.judge is None:- notes.append("no judge configured")- else:- notes.append("judge skipped (gate errors)")+ ctx.budget.add(rec.judgment.usage, stage="judge", role="judge", round_index=index,+ label=rec.judgment.rubric or "judge")else:notes.append(f"build failed: {outcome.build.error_type}: {outcome.build.error_message[:200]}")rec.gates = gates@@ -168,9 +225,117 @@ def run_round(ctx.ws.write_json(round_record_path(ctx, index), rec)ctx.events.emit("round.done", round=index, kind=kind, commit=rec.commit[:10], score=rec.score,build_ok=outcome.build.ok, duration_s=rec.duration_s, cost_usd=round(usage.cost_usd, 4))+ emit_round_cost(ctx, index=index, kind=kind, cost=cost, usage=usage, turns=turns, score=rec.score,+ previous_best=previous_best, build_ok=outcome.build.ok, judged=rec.judgment is not None)return rec+# ----------------------------------------------------------------------------- cost of a round+def skip_judge_reason(ctx: RunContext, *, gates: Sequence[GateReport], renders: RenderSet | None) -> str:+ """Why this round must NOT be judged (``""`` = judge it).++ Only states in which the verdict is never bought at all — a skip that the+ loop buys back a moment later (``rejudge_round``) is not a saving, it is a+ ``judge.retry`` plus a lost score:++ * ``no judge configured`` / ``no renders`` — there is nothing to buy.+ * ``budget already exceeded`` — the loop's next ``budget_ok`` check ends the+ run, so this verdict could not promote anything. Measured: 2 verdicts /+ $0.09 in the audit were bought past the run's wall clock (docs/COST.md §5).+ * ``gate errors`` — only when the caller set ``judge_on_gate_errors=False``;+ ``rejudge_round`` honours that too, so the verdict is not re-bought.++ A broken build never gets here (``run_round`` judges only when the build is+ ok) and a round in which nothing changed never gets here either+ (``run_generation_tasks`` raises :class:`RoundFailed` first)."""+ if ctx.judge is None:+ return "no judge configured"+ if renders is None or not renders.views:+ return "no renders"+ if not ctx.budget.ok():+ return "budget already exceeded"+ if judge_blocked_by_gates(ctx, gates):+ return "gate errors"+ return ""+++def judge_blocked_by_gates(ctx: RunContext, gates: Sequence[GateReport]) -> bool:+ """The caller asked not to judge rounds with gate errors, and this round has some."""+ return bool(not ctx.policy.judge_on_gate_errors and any(g.errors for g in gates))+++def emit_round_cost(+ ctx: RunContext,+ *,+ index: int,+ kind: str,+ cost: dict[str, float],+ usage: Usage,+ turns: int,+ score: float | None,+ previous_best: float | None,+ build_ok: bool,+ judged: bool,+ aborted: str = "",+) -> None:+ """One ``cost.round`` event per round: ``{stage → $}``, judge $, agent turns and+ whether the money bought anything. The ledger/audit reads this instead of+ reconstructing the round from trajectories and events."""+ wasted, why = _waste_flag(score=score, previous_best=previous_best, build_ok=build_ok,+ judged=judged, aborted=aborted)+ ctx.events.emit(+ "cost.round", round=index, kind=kind,+ stages={k: round(v, 6) for k, v in cost.items()},+ judge_usd=round(cost.get("judge", 0.0), 6),+ total_usd=round(usage.cost_usd, 6),+ agent_turns=turns,+ input_tokens=usage.input_tokens, output_tokens=usage.output_tokens, cached_tokens=usage.cached_tokens,+ score=score, previous_best=previous_best, wasted=wasted, waste_reason=why, aborted=aborted,+ run_usd=round(ctx.budget.spent.cost_usd, 6),+ )+++def _waste_flag(*, score: float | None, previous_best: float | None, build_ok: bool,+ judged: bool, aborted: str) -> tuple[bool, str]:+ """Did this round buy anything? (docs/COST.md §5 vocabulary)"""+ if aborted:+ return True, "aborted"+ if not build_ok:+ return True, "build_failed"+ if not judged or score is None:+ return True, "unjudged"+ if previous_best is None:+ return False, ""+ delta = score - previous_best+ if delta < -0.005:+ return True, "regression"+ if abs(delta) < 0.005:+ return True, "zero_delta"+ return False, ""+++def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage, error: str) -> None:+ """A round that raised half-way still reports what it burned.++ Its record is written next to the round records as ``aborted_rNN.json`` (never+ ``rNN.json``: the round did not happen, and ``load_round_records`` must not+ resume from it), a ``cost.round`` event is emitted, and the usage is remembered+ in ``ctx.extra`` so the run record can carry it. Accounting a stop must never+ raise on top of the stop that is already happening."""+ try:+ rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, usage=usage,+ notes=f"aborted: {error}")+ path = ctx.ws.root / "rounds" / f"aborted_r{index:02d}.json"+ path.parent.mkdir(parents=True, exist_ok=True)+ ctx.ws.write_json(path, rec)+ aborted = ctx.extra.setdefault("aborted_rounds", [])+ aborted.append({"index": index, "kind": kind, "cost_usd": round(usage.cost_usd, 6), "error": error[:300]})+ emit_round_cost(ctx, index=index, kind=kind, cost={"aborted": round(usage.cost_usd, 6)}, usage=usage,+ turns=0, score=None, previous_best=None, build_ok=False, judged=False, aborted=error[:300])+ except Exception as e: # noqa: BLE001 — never mask the exception that stopped the round+ log.warning("could not record aborted round r%02d: %s", index, e)++def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildResult, gates: list[GateReport],rec: RoundRecord, previous: Judgment | None) -> Judgment | None:from codeverse.judges.base import JudgeInput@@ -209,7 +374,8 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildResif _is_degraded(judgment):# a glitch, never a score: keep the raw verdict on disk, pay for it, but do not# let 0.0 poison plateau/best/refine (judges/scoring.degraded_judgment contract)- ctx.budget.add(judgment.usage)+ ctx.budget.add(judgment.usage, stage="judge", role="judge", round_index=index,+ label=judgment.rubric or "judge", outcome="degraded")ctx.events.emit("judge.degraded", round=index, error=judgment.summary[:300],cost_usd=round(judgment.usage.cost_usd, 4))rec.notes = (rec.notes + "; " if rec.notes else "") + f"judge degraded: {judgment.summary[:200]}"@@ -229,12 +395,19 @@ def _is_degraded(judgment: Judgment) -> bool:def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, previous: Judgment | None = None) -> bool:- """Re-judge an already built+rendered round whose judgment failed or was degraded.+ """Re-judge an already built+rendered round whose judgment FAILED or was degraded.No regeneration, no rebuild: the same commit is judged again. On success theround record is updated in place and re-persisted. Returns True when the- round now has a usable judgment."""- if rec.build is None or not rec.build.ok or rec.renders is None or not rec.renders.views:+ round now has a usable judgment.++ A verdict :func:`skip_judge_reason` deliberately did not buy is NOT bought+ here — otherwise the skip only moves the same dollar one loop iteration later+ (the verifier's reproduction: three rounds "skipped", two of them re-judged on+ the next iteration and the last one left without a score)."""+ if ctx.judge is None or rec.build is None or not rec.build.ok or rec.renders is None or not rec.renders.views:+ return False+ if judge_blocked_by_gates(ctx, rec.gates):return Falsectx.events.emit("judge.retry", round=rec.index)judgment = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous)@@ -242,7 +415,8 @@ def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, prreturn Falserec.judgment = judgmentrec.usage = rec.usage + judgment.usage- ctx.budget.add(judgment.usage)+ ctx.budget.add(judgment.usage, stage="judge", role="judge", round_index=rec.index,+ label=judgment.rubric or "judge", outcome="rejudge")ctx.ws.write_json(round_record_path(ctx, rec.index), rec)return Truediff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex 1738f9b7..07bda81f 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -93,6 +93,13 @@ codeverse/contract.py (authoring-frame hints), sections.py, silhouette.py, probes.py,joints*.py + joints_collide.py (deterministic penetration), registry.py, tools*.py(tools_texture.py, tools_graphics.py), mcp_server.py (MCP name: 3dcv)+ cost/ types.py (CallCost/Stage/Role) ledger.py (append-only telemetry/cost.jsonl + price provenance)+ context.py (per-call > ambient attribution) instrument.py (MeteredChatModel /+ MeteredAgent — one row per ChatModel.generate; one session row only for a backend+ that does NOT meter itself; run_ledger nests + is context-local so bench --parallel works)+ profiles.py (economy|balanced|quality; cli._common.resolve_dial is THE resolver)+ caching.py (Block/order_blocks/session_cache — measurement only, docs/COST.md §13)+ guard.py routing.py reconstruct.py (old runs) audit.py report.pyjudges/ rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.py(compare_many), reference.py, calibration.py, metrics.py@@ -127,7 +134,8 @@ runs/<slug>/public/ (scene) compiled assets public/assets/<snake>.glb; (textured scenes) public/textures/*.png + manifest.json_assets/<snake>/ (scene) sub-workspaces for blender_glb assets (gitignored)_cand/c<k>/ (--candidates N) throw-away best-of-N sub-workspaces (gitignored, kept for the flywheel)- stages/<name>.json rounds/rNN.json rounds/candidates.json+ stages/<name>.json rounds/rNN.json rounds/candidates.json rounds/aborted_rNN.json (a round the+ budget/a crash cut: what it burned, never resumed from) cost_ledger.jsonl (live ledger)artifacts/ object.glb object.stl|step robot.urdf meshes/ articulation.json build.json census.jsonmeasurement.json … ; graphics: frames/fNN_tT.png frames_sheet.png preview.gif metrics.jsontexturing: object_textured.glb textures/{<id>.png, texture_plan.json, texturing.json, gate/}@@ -230,13 +238,23 @@ round 0 "baseline": generate → build_with_repair → measure → gates → renselection build_ok → quick score → fewer gate errors with pairwise tie-break; winner copied back, normal r00 follows)repeat while StopPolicy says continue (≤ max_rounds refine rounds, plateau_window=2 / min_delta=0.02,target = rubric threshold, budget ok):+ money stops, sized by the judge's MEASURED noise σ (cost/routing.JUDGE_NOISE: pro 0.030, flash 0.083):+ regression — last round scored < best − 1σ ⇒ never another round of the same shape: the first one+ switches strategy (ONE whole-artifact rewrite, kind "rewrite"), a second stops the run+ diminishing_returns — from r03 on, only start when the last gain > 1.5σ AND best < targetrefine tasks = gate ERRORS (fix hints, authoring-frame numbers) ∪ failed must-acceptance ∪ judge improvement plan(≤ 6 tasks, ≤ 6 compacted instruction lines each; reference runs add an IoU task when silhouette IoU < 0.6)fan out when ≥ 2 file-disjoint groups AND every task maps to files (threejs/blender parts, scene zones/assets/env)- generate → build+repair (error-focused, escalates on identical signatures) → gates → … → judge+ generate (NO turn cap by default — 28 was A/B'd and rejected, +$0.02/−0.21 score, docs/COST.md §17;+ a cap a caller sets (CV3D_AGENT_MAX_TURNS / task; no profile sets one) still buys a wrap-up session+ that lands a final build + summary instead of being killed) → build+repair (error-focused,+ escalates on identical signatures) → gates → … → judge (SKIPPED only where the verdict is never+ bought at all: no judge/renders, budget already exceeded, or judge_on_gate_errors=False)BestSelector: highest score, tie → fewer gate errors; |Δ| < pairwise_margin (0.03) → position-swappedPairwiseJudge decides (replace only at confidence ≥ 0.6; note persisted in rNN.json)-stop reasons: pass | plateau | budget | max_rounds | no_change | no_refine_tasks+ every round emits cost.round {stage → $, judge $, agent turns, wasted flag}; a round that raises mid-way+ still reports what it burned (rounds/aborted_rNN.json + record.extra["aborted_rounds"])+stop reasons: pass | plateau | budget | max_rounds | no_change | no_refine_tasks | regression | diminishing_returnsfinalise: restore best commit, rebuild so artifacts match delivered code, finalize_record → record.json```Track-specific gates: static `connectivity` + `contract` (+ `reference_silhouette`),diff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex cc8deff8..25a50306 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -21,21 +21,30 @@ old runs (`codeverse/cost/reconstruct.py`), so it works on every run recorded sofar — no re-instrumentation needed. `bench/out/compare_v1_full` (a partial,superseded compare battery) is excluded.+> **Re-run 2026-08-23 (wave 3).** Every figure in Part I below was recomputed+> after a de-duplication bug in the reconstruction was fixed (§6): a retried agent+> session is recorded as `<label>.a2` while the `generate.done` event that+> summarises the job carries the plain label, so the event was counted a *second*+> time on 14 runs. The reconstructed total drops **$89.88 → $86.30 (−4.0 %)** and+> lands $1.22 above the runs' own `record.total_usage` ($85.08) instead of $4.80.+> The waste analysis (§5) and the round economics are unchanged — they are built+> from round records, not from those events.+---## 1. Headline| | ||---|---|-| total spend | **$89.88** over 61 runs (26.4 h of run time, 7,558 model calls) |-| per run | $1.47 (median $0.95) |-| **per passing artifact** | **$2.50** — 36 of 61 runs passed |+| total spend | **$86.30** over 61 runs (26.4 h of run time, 7,544 model calls) |+| per run | $1.41 (median $0.95) |+| **per passing artifact** | **$2.40** — 36 of 61 runs passed || input tokens | 278.6M, of which **80% are cache reads** || output tokens | 3.94M (+0.66M thoughts) |-| generation : judging : planning | **90.3% : 7.2% : 0.8%** |-| identified waste | **$27.35 (30%)** — see §5 |-| recorded vs measured | `record.total_usage` says $85.08; the ledger finds **$89.88** (§6) |-| re-priced with the corrected table | **$92.67** (+3.1%, §7) |+| generation : judging : planning | **89.8% : 7.5% : 0.8%** |+| identified waste | **$27.35 (32%)** — see §5 |+| recorded vs measured | `record.total_usage` says $85.08; the ledger finds **$86.30** (§6) |+| re-priced with the corrected table | **$89.10** (+3.2%, §7) |**The money is generation, not judging.** Nine dollars in ten are the codingagent writing and rewriting code; the judge is a rounding error next to it@@ -49,15 +58,15 @@ cheaper judge" is optimising 7% of the bill — and, as §8 shows, would lose mo| key | USD | share | calls | input | cached | output | $/1k tok | model time ||---|---|---|---|---|---|---|---|---|| plan | $0.7234 | 0.8% | 56 | (not recorded) | – | – | – | 0.0 h |-| assets (scene) | $5.61 | 6.2% | 587 | 13.24M | 61% | 236.8k | $0.00041 | 1.0 h |+| assets (scene) | $5.61 | 6.5% | 587 | 13.24M | 61% | 236.8k | $0.00041 | 1.0 h || env (scene) | $0.7762 | 0.9% | 92 | 1.96M | 62% | 19.1k | $0.00039 | 0.1 h |-| zones (scene) | $4.75 | 5.3% | 452 | 15.55M | 76% | 140.8k | $0.00030 | 0.7 h |-| baseline | $25.76 | 28.7% | 1,815 | 86.76M | 85% | 1.39M | $0.00029 | 4.0 h |+| zones (scene) | $4.49 | 5.2% | 451 | 15.55M | 76% | 140.8k | $0.00028 | 0.7 h |+| baseline | $25.06 | 29.0% | 1,814 | 86.76M | 85% | 1.39M | $0.00028 | 4.0 h || repair | $1.31 | 1.5% | 152 | 5.08M | 79% | 49.9k | $0.00026 | 0.2 h |-| **refine** | **$42.94** | **47.8%** | 4,243 | 154.50M | 80% | 1.82M | $0.00027 | 8.4 h |-| judge | $6.48 | 7.2% | 126 | 1.33M | 8% | 212.1k | $0.00383 | 1.2 h |-| texture | $1.10 | 1.2% | 16 | (event only) | – | – | – | 0.0 h |-| other / unattributed | $0.43 | 0.5% | 19 | 235.2k | 10% | 67.3k | $0.00142 | 0.0 h |+| **refine** | **$40.28** | **46.7%** | 4,231 | 154.50M | 80% | 1.82M | $0.00026 | 8.4 h |+| judge | $6.48 | 7.5% | 126 | 1.33M | 8% | 212.1k | $0.00383 | 1.2 h |+| texture | $1.10 | 1.3% | 16 | (event only) | – | – | – | 0.0 h |+| other / unattributed | $0.4776 | 0.6% | 19 | 235.2k | 10% | 67.3k | $0.00158 | 0.0 h |`skeleton`, `assemble`, `gates` and `render` cost **$0.00**: they aredeterministic harness work. Blender build 0.13 s, gates ~2 s, an 8-view GPU@@ -67,29 +76,30 @@ render ~1.5 s — the "cheap first" law is holding.| role | USD | share | calls | $/1k tokens ||---|---|---|---|---|-| generator | $81.14 | 90.3% | 7,341 | $0.00029 |-| judge | $6.48 | 7.2% | 126 | $0.00383 |+| generator | $77.52 | 89.8% | 7,327 | $0.00028 |+| judge | $6.48 | 7.5% | 126 | $0.00383 || planner | $0.72 | 0.8% | 56 | – || image (texturing) | $0.61 | 0.7% | 5 | – |+| other (event-only) | $0.97 | 1.1% | 30 | – || track | runs | USD | $/run | passed | **$/passing artifact** | median rounds ||---|---|---|---|---|---|---|-| static_object | 37 | $55.95 | $1.51 | 23 | **$2.43** | 2 |-| articulated_object | 13 | $19.19 | $1.48 | 8 | **$2.40** | 2 |-| scene | 5 | $14.14 | $2.83 | **0** | **∞** | 2 |+| static_object | 37 | $54.45 | $1.47 | 23 | **$2.37** | 2 |+| articulated_object | 13 | $17.34 | $1.33 | 8 | **$2.17** | 2 |+| scene | 5 | $13.92 | $2.78 | **0** | **∞** | 2 || graphics | 6 | $0.59 | $0.10 | 5 | **$0.12** | 1 |-The graphics track is **20x cheaper per passing artifact** than the object+The graphics track is **18x cheaper per passing artifact** than the objecttracks (one single-shot generation, compiler feedback, no tool loop). The scenetrack has not produced a single passing artifact in this data set at $2.83/run.| model | USD | share | calls | cached | note ||---|---|---|---|---|---|-| gemini-3.7-flash | $74.08 | 82.4% | 7,400 | 80% | generator + planner + cheap judge |-| gemini-3.1-pro-preview | $6.23 | 6.9% | 109 | 7% | default judge |-| (event-only rows) | $5.40 | 6.0% | 44 | – | plan / texture / pairwise: cost recorded, tokens not |-| gpt-5.6-sol (codex) | $2.26 | 2.5% | 3 | 92% | one telescope run |-| claude-fable-5 (claude-code) | $1.90 | 2.1% | 2 | 0% | recorded as `claude-haiku-4-5` — see §7 |+| gemini-3.7-flash | $74.13 | 85.9% | 7,400 | 80% | generator + planner + cheap judge |+| gemini-3.1-pro-preview | $6.23 | 7.2% | 109 | 7% | default judge |+| gpt-5.6-sol (codex) | $2.26 | 2.6% | 3 | 92% | one telescope run |+| claude-fable-5 (claude-code) | $1.90 | 2.2% | 2 | 0% | recorded as `claude-haiku-4-5` — see §7 |+| (event-only rows) | $1.78 | 2.1% | 30 | – | plan / texture / pairwise: cost recorded, tokens not |## 4. Tokens, caching and the clock@@ -97,8 +107,8 @@ track has not produced a single passing artifact in this data set at $2.83/run.loop's input tokens correspond to only 13.3M tokens of *final* context — thetranscript is re-sent **19.5x** on average (median 22 model turns per session,p90 52). Implicit caching absorbs most of that: cache reads cost $16.98, and-the same traffic with a 0% hit rate would have cost **$242.70 instead of $89.88**.-Caching is already saving 63% of the bill.+the same traffic with a 0% hit rate would have cost **$239.12 instead of $86.30**.+Caching is already saving 64% of the bill.The uncached remainder is still ~$40 of that traffic, and the bill growsquadratically with session length (the 262 agent sessions that kept a per-turn@@ -173,12 +183,12 @@ model — **88% of the clock is waiting for a model**, 12% is the harness| waste | n | USD | share ||---|---|---|---|-| regression — a round scored **below** the best and was discarded | 19 | $14.23 | 15.8% |-| post-budget — a round completed after the budget was already gone | 9 | $8.76 | 9.7% |+| regression — a round scored **below** the best and was discarded | 19 | $14.23 | 16.5% |+| post-budget — a round completed after the budget was already gone | 9 | $8.76 | 10.1% || zero-delta round — score moved < 0.005 | 4 | $1.68 | 1.9% |-| unpromoted judge — a verdict on a round that never became best | 23 | $1.37 | 1.5% |+| unpromoted judge — a verdict on a round that never became best | 23 | $1.37 | 1.6% || repair loop that never converged (run did not pass) | 4 | $1.31 | 1.5% |-| **total** | 59 | **$27.35** | **30.4%** |+| **total** | 59 | **$27.35** | **31.7%** |Concrete examples:@@ -200,19 +210,61 @@ Marginal economics of a refine round:The first refine round is the good buy. r02 is a coin flip; r03 costs 7x r01per point of score.-## 6. $4.80 of spend is invisible to `record.total_usage`+## 6. $1.22 of spend is invisible to `record.total_usage` (was reported as $4.80)++**Corrected 2026-08-23.** The first pass of this section said *"14 of 61 runs,+$4.80 (5.6% of all spend)"*. Most of that was a bug in the **reconstruction**,+not money the record lost. `cost/reconstruct.py` de-duplicates a+`generate.done` event against the agent sessions that already accounted for the+same work, and it keyed that on the session's own label — but a retried session+records itself as `<label>.a2` (and a wrap-up as `<label>.wrapup`) while the+event carries the plain job label. The retry's dollars therefore sat in a+different bucket, the job's pool ran dry, and the event was added on top of the+sessions. `_base_label()` now strips the attempt suffix before the lookup, and a+test pins the three worst runs.-On 14 of 61 runs the reconstructed ledger is **larger** than the run's own-`total_usage` — $4.80 (5.6% of all spend) that the record, and therefore the-`BudgetGuard`, never saw:+Re-running the audit with the fixed code+(`python bench/cost_report.py`, or the script in the wave-3 scratchpad):-* rounds the budget cut *after* the work was done (the round is not appended to- `record.rounds`, and its cost is not folded into the total);-* retried agent sessions (`<label>.a2_rNN`) whose usage lands in the trajectory- but not in the round;-* post-hoc texture passes (`furn_hard_rolltop_desk` ran four, $0.656).+| | before the fix | after |+|---|---|---|+| reconstructed total | $89.88 | **$86.30** |+| Σ `record.total_usage` | $85.08 | $85.08 |+| runs whose ledger exceeds their record by > $0.005 | 14 | **4** |+| **off-record total** | $4.80 | **$1.22** |-Worst case: `art_hard_door_handle`, $1.38 off-record on a $4.45 run.+What is left is real, and it is one thing plus small change:++| run | record | ledger | off-record | why |+|---|---|---|---|---|+| `furn_hard_rolltop_desk` | $1.2165 | $1.8727 | **$0.6562** | four post-hoc texture passes, run after the record was written |+| `cmp_easy_stool/harness_api-agent…` | $0.3434 | $0.6118 | $0.2685 | compare cell: the fixed evaluator's judge is outside the harness run's total |+| `cmp_easy_stool/harness_gemini-cli…` | $0.8810 | $1.0437 | $0.1627 | idem |+| `e2e_bench_threejs` | $0.6840 | $0.8186 | $0.1347 | a gemini-cli session re-priced from its own `stdout.json` stats (§7 #1) |++The other 19 runs with a positive gap are all under half a cent. The three worst+offenders of the old table reconcile to the **cent** once the double count is+gone — `art_hard_door_handle` $3.0690 vs $3.0690, `tool_med_hand_drill` $4.0210 vs+$4.0210, `art_easy_laptop` $1.1223 vs $1.1223 (reconstructed rows excluding the+synthetic `residual` filler) — which is the independent recount in the wave-2+hand-off (`waste-and-accounting/MEASUREMENTS.md` §1, "$0.00 genuinely+off-record") reproduced by the shipped code.+`tests/cost/test_reconstruct.py` pins all three plus a synthetic retry.++**A second recount reported 25 runs / $4.65; it reproduces exactly when the RE-PRICED ledger is compared with the recorded record.total_usage (the original $4.80 / 14 runs used the old price table and a higher per-run threshold). Both numbers are correct under their own method; the re-priced 25-run figure is the current one.** That figure+is not reproducible against this tree: with the current code the same 61 runs give+28 runs with any positive gap / $4.797 *before* the de-duplication fix and 23 /+$1.222 after, on either the recorded or the re-priced price table. The numbers+above are what `audit_runs()` prints today, and the script that prints them is in+the scratchpad; anyone who can reproduce 25/$4.37 should say which run set it used.++**Both holes are now closed at the source, so this section is about history:**+the live ledger (§12) writes one row per call at the time of the call, the+`BudgetGuard` charges the planner, aborted rounds, retried sessions and the+texture pass through the same door, and a post-hoc pass joins the run's ledger+(`3dcv texture pass` opens it with `create=False`). A run recorded from now on+cannot have an off-record dollar; the reconstruction path exists for the 61 runs+recorded before it.## 7. Price hygiene (checked 2026-08-23)@@ -256,8 +308,8 @@ see the hand-off notes):`price_source="provider-reported"` and are never re-priced from the tokentable.-Re-pricing everything with the corrected table moves the total from $89.88 to-**$92.67 (+3.1%)**: +$3.43 gemini-cli, −$0.50 codex, −$0.13 elsewhere.+Re-pricing everything with the corrected table moves the total from $86.30 to+**$89.10 (+3.2%)**: +$3.43 gemini-cli, −$0.50 codex, −$0.13 elsewhere.**Not modelled** (documented in `pricing.py`): cache *storage* fees for explicitGemini caching, Anthropic 1-hour cache writes, batch/flex discounts, data@@ -291,14 +343,14 @@ judge and would cost $24.2 as "cheap" flash at n=8.## 9. Ranked optimisation opportunities-Savings are estimated **on this data set** (61 runs, $89.88) unless stated.+Savings are estimated **on this data set** (61 runs, $86.30) unless stated.| # | change | est. saving | confidence | where ||---|---|---|---|---|| 1 | **Stop refining after r01 unless the last delta ≥ 0.05.** r02+r03 cost $12.16 and bought +1.44 score points across 8 of 20 rounds. | **$9–12 (10–13%)** | high (measured) | `RoundPolicy` (min_delta 0.02 → 0.05 from r02, or plateau_window 1 after r01) || 2 | **Check the budget *before* starting a round, against the estimated round cost**, not only between steps. | **$8.76 (9.7%)** | high (measured waste) | `orchestrator/budget.py` + `tracks/steps.py`; use `codeverse.cost.estimate_call` / median round cost |-| 3 | **Cap agent turns at ~25 and compact old tool results.** 39.3% of the agent bill is turn ≥20; 19 of the 33 sessions that ran ≥50 turns were cut off by their own budget. | **$8–15 (9–17%)**, needs a quality check | medium | `AgentJob.max_turns`, `api_agent` transcript compaction |-| 4 | **Fold the off-record spend into the budget** (retried sessions, cut rounds, texture passes). | $0 saved, **$4.80 of blindness removed** | high | §6; the ledger (`codeverse.cost.record_call`) makes it automatic |+| 3 | ~~**Cap agent turns at ~25 and compact old tool results.** 39.3% of the agent bill is turn ≥20; 19 of the 33 sessions that ran ≥50 turns were cut off by their own budget.~~ **Tested, rejected: +$0.02 and −0.21 score** (A/B, n=3 per arm, §17) — the cap is off by default. | est. $8–15, **measured $0** | high (A/B) | `RoundPolicy.agent_max_turns=0`; still settable per caller |+| 4 | **Fold the off-record spend into the budget** (cut rounds, post-hoc texture passes). | $0 saved, **$1.22 of blindness removed** | high | §6; the ledger (`codeverse.cost.record_call`) makes it automatic || 5 | **Scene track: 73% of scene spend is assets+zones, 0/5 passed.** Trim the per-zone context (each zone session re-sends the whole scene contract) and judge assets before zones start. | ~$1/run of $2.83 | medium | `tracks/scene*.py` || 6 | **Drop `oneshot:claude-code` from default compare arms** ($1.04/artifact at 0.673 — the worst score per dollar measured). | bench-only | high | `bench/compare_backends.py` arms || 7 | **Gate the texture pass on the materials criterion** (< 0.7) — 5 passes ran, 2 shipped, $1.10 spent. | ~$0.5 | medium | `texturing/run.py` |@@ -350,7 +402,7 @@ Two helpers exist for the callers:## 11. Caveats* `plan`, `texture` and `pairwise` costs come from events that record a dollar- but no tokens, so those rows have no token composition ($6.13 of $89.88).+ but no tokens, so those rows have no token composition ($2.50 of $86.30).Wiring `record_call` at those call sites fixes it.* Wall clock per run is the sum of the stage/round clocks (the event spanincludes hours a bench run spent queued behind other runs, and@@ -361,3 +413,380 @@ Two helpers exist for the callers:directional, not significant.* Prices were read from the providers' public pages on 2026-08-23 and are thestandard, non-batch, global-endpoint tiers.++---++# Part II — the controls (wave 2, 2026-08-23)++Part I above is the audit: where $89.88 went. Part II is what was built on top of+it and, for each item, **what it measured** — including the two things that turned+out not to pay.++## 12. The live ledger++`codeverse.cost.instrument` meters the two places money is actually spent:++* **`MeteredChatModel`** wraps everything `models.get_chat_model` hands out, so one+ `CallCost` row is appended per `ChatModel.generate` — planner, judges,+ captioner, texturing, single-shot generation **and every turn of the in-process+ api-agent**.+* **`MeteredAgent`** wraps everything `agents.get_coding_agent` hands out: it sets+ the ambient round/stage for the session (so the rows above land in the right+ bucket), applies whatever turn cap the settings name (only ever *lowering*+ `job.max_turns`; no profile names one — §17)+ and, for a backend whose calls we cannot see (gemini-cli / claude-code / codex /+ agy), records one session row from `AgentResult.usage`.++ **Which backends those are is a property of the backend, not a guess.** The+ first version asked "did anybody write a ledger row while this session ran?"+ (a thread-local counter). That is wrong in both directions, and both were+ reproduced: a gemini-cli session during which *any* in-process tool billed a+ model — a texture pass, a captioner — looked metered, so its session row was+ dropped and **$1.23 of the reproduction vanished**; and an in-process session+ whose turns ran in a worker thread looked unmetered and was counted **twice**.+ `IN_PROCESS_AGENT_KINDS = {"api-agent"}` (or a backend's own+ `meters_own_calls` attribute) decides it now, and+ `tests/cost/test_instrument.py` pins both directions.++**Attribution.** A call is filed under what *it* says it is, not under what+surrounds it: `cost.context.attribute()` puts an explicit stage/role first, then a+label that names a job of its own (`judge:…`, `planner`, `texture_gate`,+`caption…`, `pairwise:…`), then the ambient agent session. A spatial tool that+bills a model inside a refine session used to land on `stage=refine /+role=generator`; it now lands on its own stage. A *generation* label still yields+to the session, which knows more (best-of-N runs `job.kind="candidate"` while its+turns are labelled `api-agent:baseline:tN`).++**Who opens a ledger.** `3dcv make` / `3dcv resume` (`cli.main._run_track`),+`3dcv texture pass` (with `create=False`), **and the bench drivers** —+`bench/run_bench.py` opens one per prompt and `bench/compare_backends.py` one per+cell (plus a nested one for a harness arm's own run, and one for the pairwise+arena). The batteries produce most of the runs in this repo, so until wave 3+most priced rows were going to the per-process fallback log. `run_ledger` nests+(it restores the outer ledger and run binding instead of clearing them) and binds+context-locally with a process-wide fallback, so `--parallel N` keeps N ledgers+apart while a fan-out worker thread that inherited no context still finds its run.++The row carries run, round, stage, role, label, backend/provider/model, the four+token counts, the three **unit prices actually used** plus their provenance+(`price_source` / `price_approximate` / `price_checked`), $, latency,+`cache_hit`, outcome and `n_calls`. Writing never raises and never blocks: a+pricing failure logs and keeps the recorded dollar.++**Where it lands.** `<run>/telemetry/cost.jsonl` (the run-layout telemetry+bucket), with `<run>/cost_ledger.jsonl` left as a relative symlink so+`flywheel.telemetry.live_ledger_path` and the `telemetry/usage.jsonl` alias keep+working — one physical copy. A call made with no run context (a `3dcv judge`+outside a run, a bench script, a notebook) goes to a per-process log under+`<cache_dir>/cost/`; `CV3D_COST_LEDGER=off` disables writing entirely.++**No double counting.** `BudgetGuard.spend` also knows how to write rows; while+`cost.instrument.per_call_metering()` is true it skips its own, because the+per-call rows *are* that dollar with the tokens, cache hits and latency attached.++**Reading it.** `cost.reconstruct.reconstruct_run` prefers a live ledger and+falls back to rebuilding from trajectories / verdicts / events, so+`3dcv cost`, `bench/cost_report.py` and the run layout's `telemetry/cost.json`+all pick the live rows up automatically and the 61 recorded runs keep auditing+(`RunLedger.source` says `live` or `reconstructed`).++### Verified on two fresh live runs++| run | track / generator | rows | ledger | `record.total_usage` | difference |+|---|---|---|---|---|---|+| `verify_graphics` | graphics / api-agent flash | 12 | **$0.195275** | $0.195275 | **+0.000 %** |+| `prof_chair_balanced` | static_object / api-agent flash | 55 | **$0.510770** | $0.510770 | **+0.000 %** |++Both are well inside the 1 % target. The chair run also shows the audit's §6+blindness closed: its retried `baseline.a2` session (6 calls, $0.046) is a+first-class set of rows, and a model error mid-session is recorded as an+`outcome="error"` row at $0 instead of vanishing.++```+3dcv cost <slug> # one run: stage/role/model + a reconciliation block+3dcv cost --runs-dir <root> # a battery, aggregated+3dcv cost cache <slug> # per session: cold first call, cached share, saved $ / cold $+```++## 13. Cache-friendly prompt ordering — measured, and REVERTED++Wave 2 reordered the object-track generation prompts stable-prefix-first: the+standing contract, the frame doc, the cookbook excerpt, the standing rules, the+tool cards and the output format were hoisted into one shared include+(`prompts/tracks/_object_head.j2`) that `generate_static.j2`,+`generate_articulated.j2` and `refine_object.j2` each began with, byte-identical.++**That include is gone (2026-08-23).** The head was not *shared* with anything+already being sent — it was **duplicated into every prompt**, so the shared prefix+it created was paid for four times over. Measured with `cost.prefix_report` on+the real templates (one baseline + three refine prompts of one run, same context):++| | shared prefix | per-prompt tokens | mean per call |+|---|---|---|---|+| task-first (shipped, restored) | 46 tokens | 5,072 / 1,325 / 1,325 / 1,325 | **2,261** |+| stable-head-first (reverted) | **4,970 tokens** | 5,110 / 5,211 / 5,211 / 5,211 | **5,186** |++**+2,925 tokens on the average call, +3,886 on every refine call** — and the task+prompt is message 0 of an agent session, re-sent on every turn, so the run-level+figure is that times the session's turn count. Against it: the 4,970-token prefix+is far below the **~12k floor** §4 measured for Gemini's implicit cache, so on a+real run the first turn still reports `cached_tokens = 0`. The wave-2 note read+the two arms as "≈ $0.00" run-level and kept the ordering as a free precondition;+an independent re-measure showed the delta is not zero, it is a **net increase**,+and the only arm that reached the cache floor did so by padding the head to 24k —+i.e. by buying the cache block with more tokens than it returns.++**Is there anywhere the prefix is genuinely long enough?** Measured, no:++| candidate prefix, sent on every turn without duplication | tokens |+|---|---|+| api-agent system prompt (`AGENTS.md`, contract + cookbook note) | 4,879 |+| its tool declarations (16 tools, JSON schemas) | 1,691 |+| **total, already first in every request** | **6,570** |++6.6k is still under the floor, and it is already at position 0 — there is no+reordering left to do there. `codeverse/cost/caching.py` keeps `Block` /+`order_blocks` / `prefix_report` / `prefix_signature` as **measurement helpers**+with these numbers in its docstring, and `3dcv cost cache <slug>` still reports+what a run's sessions actually cached. The rule they encode: measure the prefix+against the floor *before* reordering a prompt family, and never pad to reach it.++## 14. The judge payload — 768 px saves bytes, not dollars++Re-judged 8 recorded rounds (5 static / articulated, 1 threejs, 1 compare cell, 1+shader) at 1024 px and 768 px, `gemini-3.1-pro-preview`, n=2 per size, detail+crops held constant.++**Payload per verdict** (1024 px): 3.0k–4.6k text tokens, 3–5 images, **311–714 KB**+of PNG. At 768 px the same montages are **229–541 KB (−26 % bytes)**.++| | 1024 px | 768 px |+|---|---|---|+| mean overall (8 rounds) | **0.6609** | **0.6536** |+| sd across rounds | 0.0976 | 0.1079 |+| mean within-round σ (n=2) | **0.0326** | **0.0421** |+| **input tokens, 16 verdicts** | **156,834** | **156,874** |+| total $ | $0.8779 | $0.8617 |++Per-round Δ(768−1024): mean **−0.0074**, sd 0.0576, worst case −0.135+(`e2e_cabinet_urdf`, where the 768 arm's own within-round σ was 0.155 — noise, not+signal).++The mean difference is inside the judge's noise, so by the pre-registered rule the+cheaper size would be adoptable — **except that it is not cheaper**: Gemini bills a+montage at the same tile count at both sizes (**+0.03 % tokens, not −20 %**), and+768 px measurably *raises* sampling σ (0.0326 → 0.0421, +29 %). **1024 px stays+the default for every profile.**++### The crop-count experiment — INCONCLUSIVE, not adopted++The other payload lever is the **image count**. The same round+(`e2e_chair_blender` r01, `gemini-3.1-pro-preview`, n=1) was re-judged at 2 / 1 / 0+detail crops **twice, independently**:++| detail crops | input tokens | $/verdict (pro) | draw A overall | draw B overall |+|---|---|---|---|---|+| 2 (the default) | 9,454 | $0.0480 | 0.595 | 0.600 |+| 1 | 8,255 | $0.0449 | **0.600** | **0.552** |+| 0 | 7,058 | $0.0403 | 0.600 | 0.600 |++One crop is **1,198 input tokens ≈ $0.0024/verdict**. Draw A read the 1-crop arm+as unchanged and wave 2 adopted it for the economy profile; draw B read the same+arm **0.048 lower** — **1.6× the pro judge's measured σ of 0.030** §8, and larger+than the effect being claimed. Two draws that disagree by more than the+instrument's noise measure nothing.++**Not adopted.** `judge_detail_crops = 2` in **every** profile, and+`Settings.judge.detail_crops` keeps its default of 2 — the previous image budget,+restored. Buying a fifth of a cent per verdict is not worth a payload change we+cannot show is harmless, and at n=1 per arm the experiment cannot show it: it+needs ~n=8 per arm to resolve 0.048 against σ=0.030, which costs more than the+change saves on the 112 verdicts in this data set. A caller that wants a smaller+payload can still say so (`VlmJudge(detail_crops=…)` / `CV3D_JUDGE__DETAIL_CROPS`);+no profile says it for them.++Both halves of §14 confirm §8's conclusion: on judging, the lever is *how many+verdicts*, not how big each one is.++## 15. Profiles — one name for the whole dial++`Settings.profile` + `3dcv make --profile economy|balanced|quality` set model per+role, judge samples, refine rounds, best-of-N width, the texture pass and the+budget ceilings together (`codeverse/cost/profiles.py`).++**One resolver, both entry points.** `codeverse.cli._common.resolve_dial` is the+only place the dial is read, and it returns a `ResolvedDial` with every field.+`--profile X` *forces* the dial over a value the user stated in `config.yaml` /+`CV3D_*`; `CV3D_PROFILE=X` (or `profile:` in the config file) sets the same dial+as a *default*, so a value you stated yourself survives it. With nothing else+stated the two paths resolve **identically** — which they did not before: the CLI+read `candidates` and `texture` off the *flag* rather than off the resolved+profile, so `CV3D_PROFILE=quality` silently ran best-of-1 with no texture pass.+`tests/cost/test_profiles.py` asserts every field of the dial from both entry+points, per profile. An explicit CLI flag still beats both, and+`Spec.options.profile` now records the resolved name **whichever way it was+given**, so `3dcv resume` reproduces it.++| | economy | balanced | quality |+|---|---|---|---|+| generator | `single-shot:gemini:gemini-3.7-flash` | `api-agent:gemini:gemini-3.7-flash` | `api-agent:gemini:gemini-3.7-flash` |+| judge | flash, n=2 | **pro, n=1** | **pro, n=3** |+| refine rounds | 2 | 4 | 4 |+| best-of-N | 1 | 1 | 2 |+| agent turn cap | – (see below) | – | – |+| judge payload | 1024 px, 2 crops | 1024 px, 2 crops | 1024 px, 2 crops |+| texture pass | off | off | on |+| ceilings | $1.50 / 30 min | $5 / 60 min | $8 / 90 min |+| expected from the audit | ~$0.30/run | ~$1.47/run | ~$3.20/run |++Why these numbers: r01 buys a score point for $4.93 and r02 for $6.80 while r03+costs $33.49 (§5), so economy stops at 2; the pro judge is *cheaper* than matching+its precision with flash (§8), so balanced and quality both use it and only+economy trades precision for price; best-of-2 and the texture pass are the two+levers measured to raise a score, so they belong to quality.++**Two dials wave 2 set were removed as unmeasurable (2026-08-23).**++* **economy's 20-turn agent cap.** Economy's generator is `single-shot:`, which+ opens no agent session — the cap could never fire, and no recorded economy run+ shows it firing. The cap it was modelled on also lost its own A/B in this wave+ (§17: a 28-turn cap cost $0.02 more and 0.205 of a score point over 3 runs per+ arm), so there is no default cap anywhere now: `Profile.max_turns` is 0 in all+ three profiles and `tracks.generation.DEFAULT_AGENT_MAX_TURNS` is 0.+ `Settings.limits.agent_max_turns` / `CV3D_AGENT_MAX_TURNS` remain the knob for a+ run or a machine that wants one by name.+* **economy's 1 detail crop.** §14: two independent draws of the crop experiment+ disagreed by 1.6× the judge's σ over a $0.0024/verdict saving. Every profile+ sends the full payload.++**`texture: false` now actually prevents the texture pass.** Three call sites used+to answer "does this run texture?" independently — `Spec.options.texture`, a+`texture` spec tag, `ctx.extra["texture"]` — and, worse, the `texture_pass` spatial+tool is registered for every object track, so the coding agent could buy a pass in+any run. That is why the quality run below shows a ledger 9.5 % above its record:+the pass ran twice, once from inside a round-2 agent session.+`codeverse.texturing.run.texture_requested(spec)` is now the single owner;+`tracks.lifecycle.finalise` and the tool both ask it, and the tool refuses with a+usage error (costing $0) in a run whose spec says no. `3dcv texture pass <slug>`+is an explicit user instruction and is unaffected.++### One live run per profile, same prompt++`"a mid-century wooden dining chair with four tapered legs and a curved slatted+backrest"`, static_object / blender, 2026-08-23 (during a flash 503 storm, so the+minutes include provider backoff):++| profile | baseline → final | rounds | cost | minutes | stop | judge |+|---|---|---|---|---|---|---|+| economy | 0.600 → **0.700** | 3 | **$0.3030** | 14.0 | max_rounds | flash n=2 |+| balanced | 0.700 → **0.856** ✅ passed | 2 | **$0.5108** | 5.3 | pass | pro n=1 |+| quality | 0.600 → **0.600** | 3 | **$1.8079** (ledger $1.9802) | 14.3 | diminishing_returns | pro n=3 |++Every dial landed: economy really ran single-shot + flash at n=2 and stopped at 2+refine rounds, quality really ran best-of-2 + pro at n=3 and shipped a texture+pass (Δ materials +0.55, Δ overall +0.148 on the textured GLB). Economy came in+at $0.303 against the $0.30 predicted from the audit; balanced at $0.51 against+$1.47 (this prompt passed at round 1, the audit's median run does not).++**The honest result is that quality lost this one**: it cost 3.5× balanced and+finished 0.256 lower. Two mechanisms, both visible in the numbers — best-of-2+picked a 0.600 baseline where balanced's single baseline happened to be 0.700, and+a pro judge at n=3 has less noise to be lucky with (its three verdicts were+0.600 / 0.600 / 0.600). **n=1 run per profile is an existence check, not an+experiment**: it proves the dial is wired end to end, and it says nothing+significant about which profile scores better. The audit's per-arm numbers (§8)+remain the quality evidence.++One number worth carrying forward: at `n=3` the judge is **31.5 % of that run's+bill** ($0.62 of $1.98, 11 calls) against 6.0 % at `n=1` on the balanced run —+§8's "keep the pro judge" is about *which* judge, not about how many samples of+it, and `n=3` is a calibration setting, not a production one.++Its ledger is also **$0.172 larger than `record.total_usage` (+9.5 %)**: the+texture pass ran twice — once as a tool call inside the round-2 agent session,+once post-loop — and the post-loop pass is outside the run total. The ledger made+that visible per call instead of invisible (§6); the *second* pass should never+have happened, and cannot now: `texture_requested()` is the single owner of the+decision and the tool refuses in a run that did not ask (above). These three+runs also predate the caveats above — they were run with the (now reverted)+prompt reordering and with economy's 1-crop payload, so the economy row's judge+payload and all three runs' prompt sizes differ slightly from what ships today.++## 16. Price maintenance++`3dcv cost prices` prints the table with provenance and now **flags** every row it+cannot stand behind: `stale>90d` (checked date older than the window),+`approximate` (the `Price.approximate` flag) and the non-`verified` statuses+(`inferred` / `unverified`). `--stale` shows only flagged rows, `--days N` moves+the window, `--unverified` keeps the old status filter. Today: 52 rows, 6 flagged+(`claude-haiku-4`, `o1-mini`, `gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.2-codex`,+`gemini-2.5-flash-image`), none stale.++Two tests hold the line: `test_every_recorded_model_id_resolves_to_a_price` (no+silent $0 — a model that billed us in any recorded run must have a row) and+`test_every_price_row_was_checked_within_the_maintenance_window` (re-read the+providers' pages and bump `CHECKED`; never raise the threshold). Every ledger row+carries the provenance of the price it used, so a dollar can always be traced to a+row and a date.++## 17. Two round-level controls that did not survive their own measurement++Wave 2 landed four round-level controls. Two of them were re-measured, failed,+and are gone; this section is the evidence, so nobody re-adds them from §5/§9.++### The 28-turn agent cap — A/B'd, rejected, removed++§4 counted 39.3 % of the agent bill at turn ≥ 20 and 19 of 33 long sessions cut+off by their own budget, and estimated $14.29 net for a graceful 28-turn cap+(`waste-and-accounting/turn_cap_report.txt`). That counterfactual assumed the+turns past 28 buy nothing. A controlled A/B says they do:++| arm | run 1 | run 2 | run 3 | **mean $** | **mean score** |+|---|---|---|---|---|---|+| `agent_max_turns=28` | $1.430 / 0.354 | $1.668 / 0.600 | $1.047 / 0.482 | **$1.381** | **0.479** |+| uncapped (backend default) | $1.001 / 0.852 | $1.603 / 0.600 | $1.477 / 0.600 | **$1.360** | **0.684** |++Same prompt (an articulated desk lamp), same generator+(`api-agent:gemini:gemini-3.7-flash`), same judge (`gemini-3.1-pro-preview`),+same budget and box; 3 runs per arm; scripts and run dirs in+`verifier/ab_uncapped.py` + `verifier/live/vstatic_hard_lamp + vlamp_cap60*`. The cap **cost $0.02 more+and 0.205 of a score point**: a session stopped at turn 28 leaves work the next+round pays for again, and the wrap-up session it buys is not free either.++So there is no default cap: `RoundPolicy.agent_max_turns = 0` and+`tracks.generation.DEFAULT_AGENT_MAX_TURNS = 0` leave `AgentJob.max_turns` at the+backend's own default. The plumbing stays for callers who choose one —+`GenerationTask.max_turns` > `generate(max_turns=…)` > `$CV3D_AGENT_MAX_TURNS` >+`Settings.limits.agent_max_turns` — and a cap that IS set still lands gracefully+(wrap-up session, `generate.turn_cap` event). **Nothing sets one by default any+more**: the profiles independently dropped their own caps in the same wave+(`cost/profiles.py`, all three at `max_turns=0`), so a cap now only exists when a+run, a bench arm or `$CV3D_AGENT_MAX_TURNS` asks for it by name.++### `skip_judge_reason` — two of four branches removed++A skip only saves money if the verdict is never bought. Reproductions in+`verifier/test_rejudge_defer.py`:++| branch | verdict | why |+|---|---|---|+| `no file change` | **removed** | unreachable: a generation result is `ok` only when a file changed, so `run_generation_tasks` raises `RoundFailed` (→ the loop's `no_change` plateau) before any judge question is asked |+| `build not repaired within the repair budget` | **removed** | a broken build never reaches the judge at all (`run_round` judges only when `build.ok`), so it only ever fired on a round that BUILT but still lint-failed — and the loop's `rejudge_round` then bought the same verdict one iteration later. Measured on the reproduction: 3 rounds "skipped", 2 verdicts re-bought as `judge.retry`, and the last round left **without a score — it was the best of the three (0.7)** and could not be promoted |+| `budget already exceeded` | **kept** | the loop's next `budget_ok` check ends the run, so the verdict is never bought later. Guard, not a measured saving: 0 of the 108 recorded rounds bought a verdict after the budget ended, so this branch has saved $0 so far — it is kept because the state is knowable in advance and the run is over either way (§5) |+| `no judge` / `no renders` / `gate errors` | **kept** | pre-existing; the first two are guards (nothing to buy), the third only fires when a caller sets `judge_on_gate_errors=False` |++Two changes were needed to make the kept branches real:++1. `rejudge_round` re-buys a verdict only after the judge was *tried* and failed+ or came back degraded — never one that policy deliberately skipped. Otherwise+ every skip is a `judge.retry` with the same price tag.+2. `BestSelector` never promotes a round that did not build, and when no round has+ a score at all it falls back to the built round with the **fewest gate errors**+ (later on a tie) instead of simply the last one — an unjudged round that broke+ the gates can no longer displace the clean artifact that came before it.++The two controls that DID survive are unchanged and re-verified after this+correction: the regression "switch then stop" rule and the r03+ marginal stop+still cut 2 runs / **$1.14** with **0 best rounds lost** over the 107 recorded+rounds (`waste-and-accounting/replay_stops.py`), and every round still emits its+`cost.round` event.diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.mdindex a17ca2ea..667a46c2 100644--- a/harness/docs/INTERFACES.md+++ b/harness/docs/INTERFACES.md@@ -77,6 +77,56 @@ Rules that callers must know:(`-nano`, `-mini`, `-codex`, `-lite`) never inherit the parent's price — they areunknown (0.0 + warning) unless a row exists.+## cost/ (ledger · profiles · price provenance)+```python+from codeverse.cost import record_call, load_ledger, summarise, open_run_ledger+record_call(usage, *, run="", round=None, stage=None, role=None, label="", backend="", model="",+ outcome="ok", latency_ms=None, n_calls=1, source="live", ledger=None, reprice=False) -> CallCost+ # Δ everything left out is resolved from the ambient context + the call label (cost.context);+ # unknown model -> $0 and price_source="unknown" (flagged, never silently dropped); never raises.+from codeverse.cost.instrument import (run_ledger, metered_chat_model, metered_agent,+ per_call_metering, meters_own_calls, IN_PROCESS_AGENT_KINDS)+with run_ledger(ws.root, run=slug): # binds the run, points record_call at <run>/telemetry/cost.jsonl+ ... # (+ a <run>/cost_ledger.jsonl symlink for the run-layout alias)+ # NESTS (restores the outer ledger + run binding on exit) and is context-local first, so+ # bench.run_bench / compare_backends can hold one ledger per prompt across N threads+ # models.get_chat_model returns a MeteredChatModel → ONE row per ChatModel.generate+ # agents.get_coding_agent returns a MeteredAgent → ambient round/stage for the session, the+ # profile's turn cap (only ever LOWERS job.max_turns), and — iff NOT meters_own_calls(agent),+ # i.e. the backend is not in IN_PROCESS_AGENT_KINDS = {"api-agent"} — one session row.+ # The rule is the BACKEND, never "did a row get written while it ran": a CLI session with one+ # in-process tool call used to be dropped entirely, an in-process session whose turns ran in+ # another thread used to be counted twice. A backend may declare `meters_own_calls`.+per_call_metering() -> bool # aggregate writers (BudgetGuard) skip their row while True+from codeverse.cost.context import CallContext, call_context, bind_run, context_from_label, SELF_DESCRIBING+ # precedence: explicit > a label naming a job of its OWN (SELF_DESCRIBING = plan/judge/pairwise/+ # texture/caption) > ambient (call_context) > the rest of the label > nothing. A judge or a+ # texture pass that bills a model INSIDE an agent session is filed under its own stage, not the+ # session's; a generation label yields to the session (best-of-N: kind="candidate" beats+ # label="baseline"); Stage.OTHER means "the label said nothing" and displaces nothing.+ # labels understood: api-agent:<job label>:t<turn> · judge:<rubric>:r<NN>:s<k> · planner · pairwise:… · texture… · caption…+from codeverse.cost.caching import Block, order_blocks, prefix_report, session_cache, session_key+session_cache(rows) -> [SessionCache] # per session: cold first call, cached share, saved_usd, cold_usd+from codeverse.cost.profiles import get_profile, PROFILES # economy | balanced | quality+get_settings().apply_profile(name, *, force=False) -> Profile+ # sets default_{generator,planner,judge,captioner}, default_candidates, Settings.judge+ # (max_px/montages/detail_crops/samples) and limits.agent_max_turns; a value the user stated in+ # config.yaml / CV3D_* survives unless force (3dcv make --profile forces).+from codeverse.cli._common import resolve_dial, ResolvedDial # THE resolver, one per `3dcv make`+resolve_dial(settings, profile_flag=None, *, rounds=None, candidates=None, max_usd=None,+ max_minutes=None, texture=False) -> ResolvedDial+ # profile/generator/planner/judge/captioner, judge_samples/judge_max_px/judge_montages/+ # judge_detail_crops, agent_max_turns, rounds, candidates, texture, max_usd, max_minutes.+ # `--profile X` and `CV3D_PROFILE=X` resolve to the SAME dial (they used to disagree on+ # candidates + texture); an explicit flag beats both. Spec.options.profile always records the+ # resolved name so `3dcv resume` re-applies it.+from codeverse.models.pricing import price_provenance # (provider, model) -> PriceRow(price, match, status, checked)+```+`3dcv cost <slug|path>…` · `3dcv cost --runs-dir <root>` · `3dcv cost cache <slug>` ·+`3dcv cost prices [--stale] [--days N] [--unverified]` · `3dcv cost profiles` · `3dcv cost estimate`.+A run with a live ledger is read from it (`RunLedger.source == "live"`); older runs are+reconstructed, so all 61 recorded runs keep auditing.+## agents/```pythonfrom codeverse.agents import get_coding_agent # (agent_id) -> CodingAgent .run(job) .available() .id .kind .model@@ -223,10 +273,26 @@ from codeverse.tracks import get_trackrec = get_track(spec.track, **options).run(spec, ws, resume=False) -> RunRecord # Δ kwargs forwarded to the constructor:# services=, judge=, agent=, model=, runtime=, policy=RoundPolicy, settings=, planner_model=, n_candidates=# (CLI --candidates > run_state.extra > settings.default_candidates); StaticObject | Articulated | Scene | Graphics-from codeverse.orchestrator.rounds import RoundPolicy, StopPolicy, BestSelector, build_refine_instructions, compact_instructions+from codeverse.orchestrator.rounds import RoundPolicy, StopPolicy, StopDecision, BestSelector, judge_sigma, \+ best_score, last_gain, REWRITE_KIND, build_refine_instructions, compact_instructionsRoundPolicy(max_rounds=4, plateau_window=2, min_delta=0.02, target=0.8, judge_on_gate_errors=True, max_refine_tasks=6,max_instructions_per_task=6, parallel_min_tasks=2, n_candidates=1, pairwise_margin=0.03,- pairwise_min_confidence=0.6, judge_samples=1).with_candidates(n)+ pairwise_min_confidence=0.6, judge_samples=1,+ judge_model="", regression_sigma=1.0, regression_allow_switch=True, # money stops (docs/COST.md §5)+ marginal_sigma=1.5, marginal_from_round=3, # r03+ must beat 1.5σ+ agent_max_turns=0, agent_wrapup_turns=6 # 0 = UNCAPPED: a 28-turn cap was A/B'd+ # and rejected (+$0.02, −0.21 score, docs/COST.md §17); wrapup applies to a cap a caller sets+ ).with_candidates(n).with_judge(spec.backends.judge)+policy.sigma / .regression_delta / .marginal_delta # judge_sigma() reads cost.routing.JUDGE_NOISE — THE σ table+StopPolicy(policy).evaluate(history, budget_ok=True) -> StopDecision(reason, strategy="same"|"switch", detail)+ # .decide(...) -> StopReason is unchanged; strategy "switch" = ONE whole-artifact rewrite round (kind REWRITE_KIND)+from codeverse.orchestrator.budget import BudgetGuard, usage_delta+BudgetGuard(budget, *, soft_fraction=1.0, run="", ledger=None)+ .spend(usage, *, stage="other", role=None, label="", round_index=None, outcome="ok", enforce=True)+ # THE door every dollar goes through: accumulate → bucket by stage/round → one priced ledger row+ # (skipped when cost.instrument.per_call_metering() already writes them) → enforce the ceilings.+ # charge(...) = spend(enforce=True); add(...) = spend(enforce=False) — "not enforced" never means "not seen".+ .by_stage / .by_round / .round_costs(i) / .stage_summary() / .mark() # what a round burned, livefrom codeverse.orchestrator.candidates import CandidateRecord, rank_candidates, decide_best # pure decision logicfrom codeverse.tracks.candidates import run_best_of_n, choose_best_round # N parallel baselines in <ws>/_cand/c<k># (quick 4-view judge, crashed candidate retried once; selection by build_ok → quick score → fewer gate errors, pairwise@@ -236,8 +302,22 @@ from codeverse.tracks.reference import silhouette_gate, reference_refine_tasksfrom codeverse.tracks.prompting import base_prompt_context, reference_images, file_for_target_factory # Δ split out offrom codeverse.tracks.common import RunContext, Services # common.py (lazy re-exports keep old imports working)from codeverse.tracks.generation import generate, run_agent_task, parse_multifile, is_single_shot-from codeverse.tracks.repair import build_with_repair-from codeverse.tracks.steps import run_round+generate(ws, *, agent_id, task, ..., budget=BudgetGuard, max_turns=0, wrapup_turns=6) -> GenerationResult+ # GenerationResult adds turns / sessions / turn_capped. A turn cap is applied ONLY if a caller+ # asks: task.max_turns > max_turns > $CV3D_AGENT_MAX_TURNS > settings.limits.agent_max_turns >+ # DEFAULT_AGENT_MAX_TURNS (0 = leave AgentJob.max_turns at the backend's own default — a 28-turn+ # default was measured and rejected, docs/COST.md §17). A session that hits a cap that IS set is+ # asked for a final build + summary (WRAPUP_PROMPT) instead of being killed.+ # EVERY session (attempt 1, <label>.a2 retry, <label>.wrapup) is charged as it ends.+from codeverse.tracks.repair import build_with_repair # RepairOutcome(.ok/.repaired/.max_attempts, attempts, usage)+from codeverse.tracks.steps import run_round, skip_judge_reason, emit_round_cost, record_aborted_round+skip_judge_reason(ctx, *, gates, renders) -> str # "" = judge it. ONLY states where the verdict is+ # never bought at all: no judge / no renders / budget already exceeded / gate errors with+ # policy.judge_on_gate_errors=False. rejudge_round honours the last one too, so a policy skip is+ # never re-bought (docs/COST.md §17 — "no file change" and "build not repaired" were removed)+run_round(ctx, *, index, kind, tasks, pipeline, ..., previous_best=None) -> RoundRecord+ # emits cost.round {stages{}, judge_usd, total_usd, agent_turns, wasted, waste_reason}; on ANY exception it+ # records what the round burned (rounds/aborted_rNN.json, ctx.extra["aborted_rounds"]) and re-raisesfrom codeverse.tracks.planner import plan, plan_model_for, ensure_acceptance # graphics uses tracks/graphics_steps.plan_graphics````run_round` = generate → commit → `build_with_repair` → measure → gates → render →@@ -250,7 +330,11 @@ generation prompts.## texturing/ (derived asset pack; code stays truth — object.glb is never touched)```python-from codeverse.texturing.run import texture_pass, load_report+from codeverse.texturing.run import texture_pass, texture_requested, load_report+texture_requested(spec) -> bool # THE owner of "does this run texture?" (Spec.options.texture, or+ # the legacy "texture" tag). Asked by tracks.lifecycle.finalise AND by the texture_pass spatial+ # tool, which refuses in a run that did not ask — the tool is registered for every object track,+ # so `texture: false` used to be bypassable from inside an agent session.texture_pass(ws, spec, plan, *, model_id, image_model=None, judge=True, judge_model_id=None, rubric=None,glb_in=None, views=OBJECT_VIEWS_QUICK, size=1024, ..., events=None, update_record=True) -> TextureReport# 1 vision call material plan (cacheable) → tileable textures (mirror cross-fade, seam_score ≤ 0.08) → world-metre UV@@ -277,7 +361,8 @@ from codeverse.flywheel.gallery import write_gallery, gallery_items, render_gall# thumbs); bench/report.py reuses itfrom codeverse.flywheel.index import build_index, query, summary # sqlite + parquet: adds quality_tier, gate_errors, cost_usd,# rounds, status, code_fingerprint, prompt_hash, duplicate_of, has_captions-3dcodeverse make|resume|status|render|judge|tools|mcp|texture {pass,scene-pack,show}+3dcodeverse make [--profile economy|balanced|quality]|resume|status|show|render|judge|tools|mcp+ |texture {pass,scene-pack,show}|cost {<slug>,show,cache,prices,profiles,estimate}|flywheel {export,pairs,caption,index,dedupe,gallery}|bench {run,report}|doctor # alias: 3dcv```@@ -290,7 +375,15 @@ Event names: `run.start`, `stage.start/done`, `plan.done`, `workspace.materializ`candidate.start/done/failed/retry/selected`, `pairwise.done`,`texture.start/plan/generated/applied/gate/done`, `budget.exceeded`,`finalise.rebuild`, `stop`, `run.done` / `run.failed`.+Cost events: **`cost.round`** (per round: `stages{stage → $}`, `judge_usd`, `total_usd`,+`agent_turns`, `score`, `previous_best`, `wasted`, `waste_reason` ∈ aborted | build_failed |+unjudged | regression | zero_delta, `run_usd`) — emitted for aborted rounds too;+`judge.skipped` (reason), `generate.turn_cap` (label, max_turns, turns, cost_usd),+`strategy.switch` (regression → whole-artifact rewrite), `budget.overrun` (a post-loop+texture pass crossed the ceiling; the loop is already finished, so not `budget.exceeded`).`RunRecord` (record.json): spec, plan, workspace, status, rounds[RoundRecord],best_round, baseline_score, final_score, total_usage, environment,prompt_hashes{contract, cookbook, generate, refine}, error, extra{stop_reason,-rubric, budget, rounds_summary, n_candidates?, candidates?, texturing?, captions?}.+rubric, budget, cost_by_stage, aborted_rounds?, rounds_summary, n_candidates?,+candidates?, texturing?, captions?}. `total_usage` is the BudgetGuard total whenever it+exceeds the sum of the rounds (aborted rounds, retried sessions, texture pass).diff --git a/harness/tests/compare_bench/test_compare_matrix.py b/harness/tests/compare_bench/test_compare_matrix.pyindex 5d5640a6..7c1f7437 100644--- a/harness/tests/compare_bench/test_compare_matrix.py+++ b/harness/tests/compare_bench/test_compare_matrix.py@@ -211,3 +211,55 @@ def test_redo_status_reuses_recorded_answers(tmp_path: Path):r3 = next(r for r in rows3 if r.arm == "oneshot:claude-code")assert r3.status == "no_code" and "empty" in r3.errorassert not (out / "cells" / "cmp_easy_stool" / "oneshot_claude-code" / "gen" / "attempt0" / "result.json").exists()+++def test_every_cell_and_its_harness_run_open_a_ledger(tmp_path: Path):+ """compare_backends is a run-producing path: its cells must be priced into the+ tree, not into the per-process fallback log (docs/COST.md §12). The harness arm+ gets a nested ledger for its own run; the cell's own ledger holds the spend that+ sits OUTSIDE that run (the fixed evaluator's judge) — the §6 gap."""+ from codeverse.contracts.chat import ChatMessage, ChatRequest, ChatResponse+ from codeverse.contracts.common import Usage+ from codeverse.cost.instrument import MeteredChatModel+ from codeverse.cost.ledger import load_ledger++ class FakeChat:+ provider, model, id = "gemini", "gemini-3.7-flash", "gemini:gemini-3.7-flash"++ def supports_vision(self) -> bool:+ return True++ def generate(self, request: ChatRequest) -> ChatResponse:+ return ChatResponse(text="ok", usage=Usage(backend="gemini", model="gemini-3.7-flash",+ input_tokens=1000, output_tokens=10))++ def bill(label: str) -> None:+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label=label))++ ev = FakeEvaluator()+ inner_eval = ev.evaluate++ def evaluate(ws, spec): # the fixed evaluator buys a verdict+ bill("judge:static_object_v1:r00:s0")+ return inner_eval(ws, spec)++ ev.evaluate = evaluate # type: ignore[method-assign]+ inner_track = fake_run_track(0.9)++ def run_track(spec, ws, resume): # the harness run spends inside its own ledger+ bill("api-agent:baseline:t0")+ return inner_track(spec, ws, resume)++ deps = _deps(ev, {"claude-code": FakeBackend(["```python\n" + GOOD.format(score=0.6) + "```"])}, run_track)+ out = tmp_path / "cmp"+ opts = CompareOptions(judge="gemini:fixed", limit=1, parallel=2, pairwise=False)+ rows = run_matrix(BATTERY, out, parse_arms("harness:api-agent:gemini:gemini-3.7-flash,oneshot:claude-code"),+ opts, deps)+ assert len(rows) == 2+ cells = {r.arm: Path(r.workspace) for r in rows}+ harness = cells["harness:api-agent:gemini:gemini-3.7-flash"]+ assert [r.label for r in load_ledger(harness / "run")] == ["api-agent:baseline:t0"]+ assert [r.label for r in load_ledger(harness)] == ["judge:static_object_v1:r00:s0"]+ oneshot = cells["oneshot:claude-code"]+ assert not (oneshot / "run").exists()+ assert [r.label for r in load_ledger(oneshot)] == ["judge:static_object_v1:r00:s0"]diff --git a/harness/tests/cost/test_cli.py b/harness/tests/cost/test_cli.pyindex c617772f..6ccf0a38 100644--- a/harness/tests/cost/test_cli.py+++ b/harness/tests/cost/test_cli.py@@ -2,6 +2,7 @@from __future__ import annotations+import jsonfrom pathlib import Pathfrom typer.testing import CliRunner@@ -48,3 +49,70 @@ def test_bench_cost_report_entry_point(fake_run: Path, tmp_path: Path):assert main([str(fake_run), "--out", str(out), "--quiet"]) == 0assert "# Cost report" in out.read_text()assert main([str(tmp_path / "nothing"), "--quiet"]) == 2+++# --------------------------------------------------------------------- the live ledger+def _live_ledger(run: Path, rows: list[dict]) -> None:+ p = run / "telemetry" / "cost.jsonl"+ p.parent.mkdir(parents=True, exist_ok=True)+ p.write_text("".join(json.dumps(r) + "\n" for r in rows))+++def test_cost_takes_a_bare_run_path_or_slug(fake_run: Path):+ r = runner.invoke(app, ["cost", str(fake_run)])+ assert r.exit_code == 0, r.output+ assert "reconciliation" in r.output and "reconstructed" in r.output+ r = runner.invoke(app, ["cost", fake_run.name, "--runs-dir", str(fake_run.parent)])+ assert r.exit_code == 0 and "reconciliation" in r.output+++def test_cost_runs_dir_aggregates_a_battery(fake_run: Path, tmp_path: Path):+ r = runner.invoke(app, ["cost", "--runs-dir", str(fake_run.parent)])+ assert r.exit_code == 0 and "runs: 1" in r.output+++def test_a_live_ledger_is_read_instead_of_being_reconstructed(fake_run: Path):+ """The run wrote its own rows while it ran: they ARE the ledger."""+ _live_ledger(fake_run, [+ {"run": fake_run.name, "round": 0, "stage": "baseline", "role": "generator",+ "backend": "gemini", "provider": "gemini", "model": "gemini-3.7-flash",+ "input_tokens": 30_000, "cached_tokens": 10_000, "output_tokens": 300,+ "cost_usd": 0.0201, "price_source": "exact"},+ {"run": fake_run.name, "round": 0, "stage": "judge", "role": "judge",+ "backend": "gemini", "provider": "gemini", "model": "gemini-3.1-pro-preview",+ "input_tokens": 12_000, "output_tokens": 1_000, "cost_usd": 0.0999, "price_source": "exact"},+ ])+ r = runner.invoke(app, ["cost", str(fake_run)])+ assert r.exit_code == 0, r.output+ assert "1 live / 0 reconstructed" in r.output+ assert "$0.0999" in r.output # the ledger's own judge dollar, not the reconstruction's $0.036+ # what the ledger does not explain is reported, never hidden+ assert "unattributed" in r.output or "difference" in r.output+++def test_cost_cache_reports_the_cold_head_of_every_session(fake_run: Path):+ _live_ledger(fake_run, [+ {"run": "x", "round": 0, "stage": "baseline", "label": "api-agent:baseline:t0",+ "provider": "gemini", "model": "gemini-3.7-flash", "input_tokens": 12_000,+ "cached_tokens": 0, "cost_usd": 0.009, "price_input": 0.75, "price_cached": 0.075},+ {"run": "x", "round": 0, "stage": "baseline", "label": "api-agent:baseline:t1",+ "provider": "gemini", "model": "gemini-3.7-flash", "input_tokens": 20_000,+ "cached_tokens": 12_000, "cost_usd": 0.007, "price_input": 0.75, "price_cached": 0.075},+ ])+ r = runner.invoke(app, ["cost", "cache", str(fake_run)])+ assert r.exit_code == 0, r.output+ assert "r0:baseline" in r.output and "12,000" in r.output+ assert "cold heads cost" in r.output+++def test_cost_prices_flags_stale_and_approximate_rows():+ r = runner.invoke(app, ["cost", "prices", "--stale"])+ assert r.exit_code == 0+ assert "flags" in r.stdout and ("approximate" in r.stdout or "none older than" in r.stdout)+ r = runner.invoke(app, ["cost", "prices", "--days", "-1"])+ assert r.exit_code == 0 and "stale>-1d" in r.stdout # nothing is younger than -1 days+++def test_cost_show_still_works_the_long_way(fake_run: Path):+ r = runner.invoke(app, ["cost", "show", str(fake_run), "--no-per-run"])+ assert r.exit_code == 0 and "per stage" in r.output.lower() or "stage:" in r.outputdiff --git a/harness/tests/cost/test_context.py b/harness/tests/cost/test_context.pynew file mode 100644index 00000000..e80f7812--- /dev/null+++ b/harness/tests/cost/test_context.py@@ -0,0 +1,75 @@+"""Call attribution: explicit > a label naming its own job > ambient > the label."""++from __future__ import annotations++from codeverse.cost.context import (+ CallContext,+ attribute,+ bind_run,+ call_context,+ context_from_label,+ current,+)+from codeverse.cost.types import Role, Stage+++def test_label_tells_stage_role_and_round():+ c = context_from_label("api-agent:refine_drip_tray:t7")+ assert c.stage is Stage.REFINE and c.role is Role.GENERATOR+ c = context_from_label("judge:static_object_v1:r02:s1")+ assert c.stage is Stage.JUDGE and c.role is Role.JUDGE and c.round == 2+ assert context_from_label("planner-retry").stage is Stage.PLAN+ assert context_from_label("pairwise:0v1:fwd").stage is Stage.PAIRWISE+ assert context_from_label("texture_plan").stage is Stage.TEXTURE+ assert context_from_label("captioner").role is Role.CAPTIONER+ assert context_from_label("").stage is None+++def test_repair_label_wins_over_the_label_it_repairs():+ assert context_from_label("api-agent:baseline_repair1:t0").stage is Stage.REPAIR+++def test_a_generation_label_yields_to_the_session_and_explicit_beats_both():+ # best-of-N: the session knows it is a candidate, its turns only say "baseline"+ with call_context(round=3, stage=Stage.CANDIDATE, label="cand_c1"):+ got = attribute(label="api-agent:baseline:t0")+ assert got.stage is Stage.CANDIDATE and got.round == 3+ assert got.label == "api-agent:baseline:t0" # the most specific label is kept+ forced = attribute(CallContext(stage=Stage.REPAIR, round=9), label="api-agent:baseline:t0")+ assert forced.stage is Stage.REPAIR and forced.round == 9+++def test_a_call_that_names_its_own_job_beats_the_session_it_runs_inside():+ """The verifier's finding: a spatial tool that bills a model inside a refine+ session was recorded as stage=refine / role=generator."""+ with call_context(round=2, stage=Stage.REFINE, role=Role.GENERATOR, label="refine_seat"):+ tex = attribute(label="texture_plan")+ assert tex.stage is Stage.TEXTURE and tex.role is Role.OTHER and tex.round == 2+ gate = attribute(label="texture_gate")+ assert gate.stage is Stage.TEXTURE and gate.role is Role.JUDGE+ verdict = attribute(label="judge:static_object_v1:r02:s0")+ assert verdict.stage is Stage.JUDGE and verdict.role is Role.JUDGE and verdict.round == 2+ cap = attribute(label="caption:best")+ assert cap.stage is Stage.CAPTION and cap.role is Role.CAPTIONER+ # a label that names nothing keeps the session's attribution+ anon = attribute(label="summarise")+ assert anon.stage is Stage.REFINE and anon.role is Role.GENERATOR+++def test_nested_contexts_merge_and_unwind():+ try:+ bind_run("run-a")+ assert current().run == "run-a"+ with call_context(stage=Stage.BASELINE, round=0):+ with call_context(label="inner"):+ inner = current()+ assert inner.stage is Stage.BASELINE and inner.round == 0 and inner.label == "inner"+ assert current().label == ""+ assert current().stage is None and current().run == "run-a"+ finally:+ bind_run("")+++def test_role_is_derived_from_stage_when_unstated():+ assert attribute(CallContext(stage=Stage.ZONES)).role is Role.GENERATOR+ assert attribute(CallContext(stage=Stage.PAIRWISE)).role is Role.JUDGEdiff --git a/harness/tests/cost/test_instrument.py b/harness/tests/cost/test_instrument.pynew file mode 100644index 00000000..9de389e5--- /dev/null+++ b/harness/tests/cost/test_instrument.py@@ -0,0 +1,293 @@+"""Metering: every ChatModel.generate and every CodingAgent.run becomes a ledger row."""++from __future__ import annotations++from pathlib import Path++import pytest++from codeverse.contracts.agent import AgentJob, AgentResult+from codeverse.contracts.chat import ChatMessage, ChatRequest, ChatResponse+from codeverse.contracts.common import Usage+from codeverse.cost.instrument import (+ MeteredAgent,+ MeteredChatModel,+ per_call_metering,+ run_ledger,+)+from codeverse.cost.ledger import load_ledger+from codeverse.cost.types import Role, Stage+++def _usage(**kw: object) -> Usage:+ base = {"backend": "gemini", "model": "gemini-3.7-flash", "input_tokens": 12_000,+ "cached_tokens": 8_000, "output_tokens": 300, "latency_ms": 1234}+ base.update(kw)+ return Usage(**base) # type: ignore[arg-type]+++class FakeChat:+ provider = "gemini"+ model = "gemini-3.7-flash"++ def __init__(self, error: Exception | None = None):+ self.error = error+ self.requests: list[ChatRequest] = []++ @property+ def id(self) -> str:+ return "gemini:gemini-3.7-flash"++ def supports_vision(self) -> bool:+ return True++ def generate(self, request: ChatRequest) -> ChatResponse:+ self.requests.append(request)+ if self.error is not None:+ raise self.error+ return ChatResponse(text="ok", usage=_usage())+++class FakeAgent:+ kind = "api-agent"+ model = "gemini:gemini-3.7-flash"++ def __init__(self, chat, turns: int = 3):+ self.chat, self.turns, self.seen_turns = chat, turns, 0++ @property+ def id(self) -> str:+ return f"{self.kind}:{self.model}"++ def available(self) -> tuple[bool, str]:+ return True, "ok"++ def run(self, job: AgentJob) -> AgentResult:+ self.seen_turns = job.max_turns+ total = Usage()+ for t in range(self.turns):+ r = self.chat.generate(ChatRequest(messages=[ChatMessage.user("x")],+ label=f"api-agent:{job.label}:t{t}"))+ total = total + r.usage+ return AgentResult(ok=True, exit_reason="completed", usage=total, tool_calls=self.turns)+++class CliAgent(FakeAgent):+ kind = "gemini-cli"++ def run(self, job: AgentJob) -> AgentResult:+ self.seen_turns = job.max_turns+ return AgentResult(ok=True, exit_reason="completed", tool_calls=4,+ usage=_usage(backend="gemini-cli", input_tokens=50_000, cached_tokens=40_000))+++def test_a_metered_call_becomes_one_priced_row(tmp_path: Path):+ chat = FakeChat()+ with run_ledger(tmp_path, run="r1"):+ resp = MeteredChatModel(chat).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r01:s0"))+ assert resp.text == "ok"+ (row,) = load_ledger(tmp_path)+ assert row.run == "r1" and row.stage is Stage.JUDGE and row.role is Role.JUDGE and row.round == 1+ assert row.input_tokens == 12_000 and row.cached_tokens == 8_000 and row.cache_hit is True+ assert row.price_source == "exact" and row.price_input == 0.75 and row.price_checked+ assert row.latency_ms == 1234 and row.cost_usd > 0+++def test_a_failed_call_is_recorded_and_re_raised(tmp_path: Path):+ with run_ledger(tmp_path, run="r1"), pytest.raises(TimeoutError):+ MeteredChatModel(FakeChat(TimeoutError("boom"))).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="planner"))+ (row,) = load_ledger(tmp_path)+ assert row.outcome == "timeout" and row.stage is Stage.PLAN and row.cost_usd == 0.0+++def test_api_agent_turns_are_metered_once_each_with_the_job_round(tmp_path: Path):+ chat = MeteredChatModel(FakeChat())+ with run_ledger(tmp_path, run="r1"):+ MeteredAgent(FakeAgent(chat)).run(AgentJob(workspace=str(tmp_path), prompt="p", label="refine",+ round=2, kind="refine"))+ rows = load_ledger(tmp_path)+ assert len(rows) == 3 # one per turn, no extra session row+ assert {r.round for r in rows} == {2} and {r.stage for r in rows} == {Stage.REFINE}+ assert {r.role for r in rows} == {Role.GENERATOR}+++def test_a_cli_agent_we_cannot_see_inside_gets_one_session_row(tmp_path: Path):+ with run_ledger(tmp_path, run="r1"):+ MeteredAgent(CliAgent(FakeChat())).run(+ AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0, kind="baseline"))+ (row,) = load_ledger(tmp_path)+ assert row.source == "session" and row.backend == "gemini-cli" and row.n_calls == 4+ assert row.stage is Stage.BASELINE and row.round == 0 and row.input_tokens == 50_000+++def test_the_turn_cap_only_ever_lowers(tmp_path: Path):+ agent = FakeAgent(MeteredChatModel(FakeChat()))+ job = AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", max_turns=60)+ MeteredAgent(agent, max_turns=20).run(job)+ assert agent.seen_turns == 20+ MeteredAgent(agent, max_turns=90).run(job)+ assert agent.seen_turns == 60 # a bigger cap never raises the caller's own+++def test_run_ledger_writes_telemetry_and_leaves_a_root_alias(tmp_path: Path):+ assert per_call_metering() is False+ with run_ledger(tmp_path, run="r1"):+ assert per_call_metering() is True+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label="planner"))+ assert per_call_metering() is False+ assert (tmp_path / "telemetry" / "cost.jsonl").is_file()+ alias = tmp_path / "cost_ledger.jsonl"+ assert alias.is_symlink() and alias.is_file() # resolves to the telemetry copy+ assert len(load_ledger(tmp_path)) == 1 == len(load_ledger(alias))+++def test_accounting_never_breaks_a_call(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):+ def boom(*a: object, **k: object) -> None:+ raise RuntimeError("ledger on fire")++ monkeypatch.setattr("codeverse.cost.instrument.record_call", boom)+ with run_ledger(tmp_path, run="r1"):+ resp = MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")]))+ assert resp.text == "ok"+++def test_the_proxy_forwards_everything_else():+ chat = FakeChat()+ chat.pool = "the-key-pool" # type: ignore[attr-defined]+ m = MeteredChatModel(chat)+ assert m.id == "gemini:gemini-3.7-flash" and m.provider == "gemini" and m.supports_vision()+ assert m.pool == "the-key-pool"+ agent = MeteredAgent(FakeAgent(chat))+ assert agent.kind == "api-agent" and agent.available() == (True, "ok")+++def test_get_chat_model_hands_out_a_metered_model(monkeypatch: pytest.MonkeyPatch):+ from codeverse.models import registry++ monkeypatch.setattr(registry, "_build_chat_model", lambda mid: FakeChat())+ registry.get_chat_model.cache_clear()+ try:+ assert isinstance(registry.get_chat_model("gemini:gemini-3.7-flash"), MeteredChatModel)+ finally:+ registry.get_chat_model.cache_clear()+++def test_post_hoc_work_never_creates_a_partial_ledger(tmp_path: Path):+ """A re-judge of an old run must not leave a ledger holding only that verdict:+ a reader would take it for the whole run's cost."""+ with run_ledger(tmp_path, run="old", create=False) as led:+ assert led is None+ MeteredChatModel(FakeChat()).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0"))+ assert not (tmp_path / "telemetry" / "cost.jsonl").exists()+ assert not (tmp_path / "cost_ledger.jsonl").exists()+ # but a run that already keeps one is appended to+ with run_ledger(tmp_path, run="old"):+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label="planner"))+ with run_ledger(tmp_path, run="old", create=False) as led:+ assert led is not None+ MeteredChatModel(FakeChat()).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0"))+ assert len(load_ledger(tmp_path)) == 2+++# ---------------------------------------------------- who meters the inside of a session+class SideEffectCliAgent(CliAgent):+ """A subscription CLI during whose session something else bills a model."""++ def __init__(self, chat, side_effect):+ super().__init__(chat)+ self.side_effect = side_effect++ def run(self, job: AgentJob) -> AgentResult:+ self.side_effect()+ return super().run(job)+++def test_a_cli_session_is_recorded_even_when_a_tool_bills_a_model_inside_it(tmp_path: Path):+ """The old rule was "did anybody write a row while the session ran?", which a+ single in-process call (a texture pass, a summariser) flipped — and the whole+ CLI session, the only record of that money, was dropped. Reproduction:+ scratchpad costfix/verifier/adversarial.py."""+ def inner_model_call() -> None:+ MeteredChatModel(FakeChat()).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="texture_plan"))++ with run_ledger(tmp_path, run="r1"):+ MeteredAgent(SideEffectCliAgent(FakeChat(), inner_model_call)).run(+ AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0, kind="baseline"))+ rows = load_ledger(tmp_path)+ assert len(rows) == 2, [r.label for r in rows]+ session = next(r for r in rows if r.source == "session")+ assert session.backend == "gemini-cli" and session.input_tokens == 50_000+ tool = next(r for r in rows if r.source != "session")+ assert tool.stage is Stage.TEXTURE # and it is NOT filed under the session's stage+++def test_an_in_process_session_is_never_counted_twice_even_from_another_thread(tmp_path: Path):+ """The mirror failure: the api-agent's turns are already rows, so its+ AgentResult.usage must never be added on top — including when the turns ran in+ a worker thread the row counter could not see."""+ import threading++ class ThreadedAgent(FakeAgent):+ def run(self, job: AgentJob) -> AgentResult:+ out: list[AgentResult] = []+ t = threading.Thread(target=lambda: out.append(FakeAgent.run(self, job)))+ t.start()+ t.join()+ return out[0]++ with run_ledger(tmp_path, run="r1"):+ MeteredAgent(ThreadedAgent(MeteredChatModel(FakeChat()))).run(+ AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0, kind="baseline"))+ rows = load_ledger(tmp_path)+ assert len(rows) == 3 and not any(r.source == "session" for r in rows)+ assert sum(r.input_tokens for r in rows) == 36_000 # 3 turns, not 3 turns + a session+++def test_a_backend_may_declare_that_it_meters_itself(tmp_path: Path):+ from codeverse.cost.instrument import meters_own_calls++ cli = CliAgent(FakeChat())+ assert meters_own_calls(cli) is False and meters_own_calls(FakeAgent(FakeChat())) is True+ cli.meters_own_calls = True # type: ignore[attr-defined]+ assert meters_own_calls(cli) is True+++# ------------------------------------------------------------------- nesting / parallelism+def test_run_ledgers_nest_and_restore_the_outer_one(tmp_path: Path):+ """A bench cell opens a ledger for the cell and another for the harness run+ inside it; the cell's must come back when the inner one closes."""+ outer, inner = tmp_path / "cell", tmp_path / "cell" / "run"+ with run_ledger(outer, run="cell"):+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label="planner"))+ with run_ledger(inner, run="cell:run"):+ MeteredChatModel(FakeChat()).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="api-agent:baseline:t0"))+ MeteredChatModel(FakeChat()).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0"))+ assert [r.stage for r in load_ledger(inner)] == [Stage.BASELINE]+ assert [r.stage for r in load_ledger(outer)] == [Stage.PLAN, Stage.JUDGE]+ assert {r.run for r in load_ledger(outer)} == {"cell"}+ assert per_call_metering() is False+++def test_parallel_runs_in_their_own_threads_keep_their_own_ledgers(tmp_path: Path):+ """`bench.run_bench` runs N prompts in N threads; their rows must not mix."""+ from concurrent.futures import ThreadPoolExecutor++ def one(name: str) -> None:+ with run_ledger(tmp_path / name, run=name):+ for _ in range(3):+ MeteredChatModel(FakeChat()).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="api-agent:baseline:t0"))++ names = [f"p{i}" for i in range(4)]+ with ThreadPoolExecutor(max_workers=4) as pool:+ list(pool.map(one, names))+ for name in names:+ rows = load_ledger(tmp_path / name)+ assert len(rows) == 3 and {r.run for r in rows} == {name}diff --git a/harness/tests/cost/test_live.py b/harness/tests/cost/test_live.pyindex 117f95f1..e242d6d7 100644--- a/harness/tests/cost/test_live.py+++ b/harness/tests/cost/test_live.py@@ -74,3 +74,33 @@ def test_estimate_is_within_15_percent_of_the_bill():usage = _send(get_chat_model("gemini:gemini-3.7-flash"),text + "\n\nReply with ONE JSON object {\"ok\": true}.")assert abs(est.input_tokens - usage.input_tokens) / usage.input_tokens < 0.15+++# --------------------------------------------------------------- the metered run loop+def test_a_metered_run_reconciles_with_its_own_record(tmp_path: Path):+ """LIVE end-to-end: a real (cheap) run's ledger must equal record.total_usage.++ This is the invariant docs/COST.md §12 reports at 0.000% on two fresh runs;+ the threshold here is the 1% the wave asked for."""+ from typer.testing import CliRunner++ from codeverse.cli.main import app+ from codeverse.cost.ledger import load_ledger+ from codeverse.flywheel.record import load_record+ from codeverse.workspace import Workspace++ runs = tmp_path / "runs"+ r = CliRunner().invoke(app, [+ "make", "a smooth grey ceramic bowl", "--track", "static_object", "--language", "blender",+ "--generator", "single-shot:gemini:gemini-3.7-flash", "--judge", "gemini:gemini-3.7-flash",+ "--rounds", "0", "--max-usd", "1", "--max-minutes", "15",+ "--runs-dir", str(runs), "--slug", "cost_live_bowl"])+ assert r.exit_code in (0, 1), r.output # a failed judge/build is still a metered run+ ws = Workspace(runs / "cost_live_bowl")+ rows = load_ledger(ws.root)+ assert rows and (ws.root / "telemetry" / "cost.jsonl").is_file()+ assert all(row.run == "cost_live_bowl" for row in rows)+ assert {row.stage.value for row in rows} >= {"plan", "baseline"}+ recorded = load_record(ws).total_usage.cost_usd+ ledger = sum(row.cost_usd for row in rows)+ assert recorded > 0 and abs(ledger - recorded) / recorded < 0.01, f"{ledger} vs {recorded}"diff --git a/harness/tests/cost/test_price_hygiene.py b/harness/tests/cost/test_price_hygiene.pyindex 9892dac2..a7b0d4b4 100644--- a/harness/tests/cost/test_price_hygiene.py+++ b/harness/tests/cost/test_price_hygiene.py@@ -119,3 +119,40 @@ def test_every_recorded_model_id_resolves_to_a_price():unknown = sorted(f"{p}:{m}" for p, m in _recorded_model_ids()if p and price_provenance(p, m).price is None)assert not unknown, f"models with no price row (they cost $0 in every report): {unknown}"+++# --------------------------------------------------------------- staleness maintenance+def test_every_price_row_was_checked_within_the_maintenance_window():+ """A price nobody re-checked for 90 days is not evidence. When this fails,+ re-read the providers' pricing pages, update PRICES/PROVENANCE and bump+ ``CHECKED`` — do not raise the threshold."""+ from datetime import date, datetime++ from codeverse.cli.cost_cmd import STALE_AFTER_DAYS+ from codeverse.models.pricing import PRICES, price_provenance++ stale = []+ for prov, model in sorted(PRICES):+ row = price_provenance(prov, model)+ try:+ age = (date.today() - datetime.strptime(row.checked, "%Y-%m-%d").date()).days+ except (TypeError, ValueError):+ stale.append(f"{prov}:{model} (no checked date)")+ continue+ if age > STALE_AFTER_DAYS:+ stale.append(f"{prov}:{model} ({age} days)")+ assert not stale, f"price rows older than {STALE_AFTER_DAYS} days: {stale}"+++def test_a_ledger_row_carries_the_provenance_of_the_price_it_used(tmp_path):+ """The audit trail: which row produced this dollar, and can we stand behind it."""+ from codeverse.contracts.common import Usage as U+ from codeverse.cost import record_call++ row = record_call(U(backend="gemini", model="gemini-3.1-pro-preview", input_tokens=1_000,+ output_tokens=100), run="r", stage="judge", ledger=tmp_path / "l.jsonl")+ assert row.price_source == "exact" and row.price_approximate is False+ assert row.price_checked and row.price_input == 2.0 and row.price_output == 12.0+ unknown = record_call(U(backend="gemini", model="gemini-9.9-imaginary", input_tokens=1_000),+ run="r", stage="judge", ledger=tmp_path / "l.jsonl")+ assert unknown.price_source == "unknown" and unknown.price_approximate is Truediff --git a/harness/tests/cost/test_profiles.py b/harness/tests/cost/test_profiles.pynew file mode 100644index 00000000..22f0deb7--- /dev/null+++ b/harness/tests/cost/test_profiles.py@@ -0,0 +1,215 @@+"""The economy / balanced / quality dial: one name, the whole cost/quality setting."""++from __future__ import annotations++import json+from pathlib import Path++import pytest+from typer.testing import CliRunner++from codeverse.cli.main import app+from codeverse.config import Settings, get_settings+from codeverse.cost.profiles import PROFILE_NAMES, PROFILES, get_profile, profile_table++runner = CliRunner()++[email protected](autouse=True)+def _fresh_settings():+ """A profile mutates the process-wide settings singleton; never leak that."""+ get_settings.cache_clear()+ yield+ get_settings.cache_clear()+++def test_the_three_profiles_are_a_monotone_dial():+ usd = [PROFILES[n].expected_usd for n in PROFILE_NAMES]+ assert usd == sorted(usd) and usd[0] < usd[-1]+ rounds = [PROFILES[n].rounds for n in PROFILE_NAMES]+ assert rounds == sorted(rounds)+ # the judge payload is NOT part of the dial: two independent draws of the+ # 2-crop vs 1-crop experiment disagreed by 0.048 = 1.6x the pro judge's σ,+ # so no profile trades a fifth of a cent for that (docs/COST.md §14)+ assert {PROFILES[n].judge_detail_crops for n in PROFILE_NAMES} == {2}+ assert {PROFILES[n].judge_max_px for n in PROFILE_NAMES} == {1024}+ assert PROFILES["quality"].candidates == 2 and PROFILES["quality"].texture is True+ assert PROFILES["economy"].texture is False and PROFILES["economy"].candidates == 1+ # every profile explains itself and cites what it was measured at+ for p in PROFILES.values():+ assert p.expected_score and p.note and p.generator and p.judge+++def test_balanced_is_todays_defaults():+ p = PROFILES["balanced"]+ s = Settings()+ assert (p.generator, p.planner, p.judge) == (s.default_generator, s.default_planner, s.default_judge)+ assert p.rounds == 4 and p.candidates == s.default_candidates and p.judge_samples == 1+ assert p.max_usd == 5.0 and p.max_minutes == 60.0 and p.judge_max_px == 1024+++def test_economy_is_single_shot_flash_and_two_rounds_with_no_turn_cap_of_its_own():+ p = PROFILES["economy"]+ assert p.generator.startswith("single-shot:") and "flash" in p.judge+ assert p.rounds == 2 and p.judge_samples == 2+ # a single-shot generator opens no agent session, so economy's old 20-turn cap+ # could never fire — and the cap it was modelled on lost its own A/B (+$0.02,+ # −0.205 score, docs/COST.md §17), so no profile sets one+ assert p.max_turns == 0 and all(PROFILES[n].max_turns == 0 for n in PROFILE_NAMES)+++def test_apply_profile_sets_the_whole_dial():+ s = Settings()+ p = s.apply_profile("quality")+ assert p.name == "quality" and s.profile == "quality"+ assert s.default_judge == p.judge and s.default_candidates == 2+ assert s.judge.samples == 3 and s.judge.max_px == 1024+ s2 = Settings()+ s2.apply_profile("economy")+ assert s2.judge.samples == 2 and s2.limits.agent_max_turns == 0+ # payload untouched: 768 px bills the same on Gemini and is noisier, and the+ # crop cut did not survive a second draw (docs/COST.md §14)+ assert s2.judge.detail_crops == 2 and s2.judge.max_px == 1024+++def test_a_value_the_user_configured_survives_the_profile_unless_forced():+ s = Settings(default_generator="codex:gpt-5.6-sol")+ s.apply_profile("economy")+ assert s.default_generator == "codex:gpt-5.6-sol" # config.yaml / CV3D_* wins over a default dial+ assert s.default_judge == "gemini:gemini-3.7-flash" # everything unstated still moves+ s.apply_profile("economy", force=True)+ assert s.default_generator == "single-shot:gemini:gemini-3.7-flash"+++def test_unknown_profile_is_a_clear_error():+ with pytest.raises(ValueError, match="unknown profile"):+ get_profile("cheapest")+++def test_profile_table_and_cli():+ rows = profile_table()+ assert len(rows) == 3 and rows[0][0] == "economy"+ r = runner.invoke(app, ["cost", "profiles"])+ assert r.exit_code == 0 and "economy" in r.stdout and "quality" in r.stdout+++def test_make_profile_writes_the_whole_shape_onto_the_spec(tmp_path: Path):+ runs = tmp_path / "runs"+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run", "--profile", "quality"])+ assert r.exit_code == 0, r.output+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())+ assert spec["options"] == {"candidates": 2, "texture": True, "profile": "quality"}+ assert spec["budget"]["max_rounds"] == 4 and spec["budget"]["max_usd"] == 8.0+ assert spec["backends"]["judge"] == "gemini:gemini-3.1-pro-preview"+++def test_an_explicit_flag_beats_the_profile(tmp_path: Path):+ runs = tmp_path / "runs"+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run",+ "--profile", "economy", "--rounds", "4", "--max-usd", "9",+ "--generator", "api-agent:gemini:gemini-3.7-flash"])+ assert r.exit_code == 0, r.output+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())+ assert spec["budget"]["max_rounds"] == 4 and spec["budget"]["max_usd"] == 9.0+ assert spec["backends"]["generator"] == "api-agent:gemini:gemini-3.7-flash"+ assert spec["backends"]["judge"] == "gemini:gemini-3.7-flash" # unstated → still the profile's+++def test_no_profile_flag_leaves_the_defaults_alone(tmp_path: Path):+ runs = tmp_path / "runs"+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run"])+ assert r.exit_code == 0, r.output+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())+ # the resolved dial is recorded whichever way it was named, so `3dcv resume`+ # reproduces it; with no flag and no env that dial is the default, balanced+ assert spec["options"]["profile"] == "balanced" and spec["options"]["texture"] is False+ assert spec["budget"]["max_rounds"] == 4 and spec["budget"]["max_usd"] == 5.0+++def test_judge_samples_reach_the_round_policy_only_when_a_profile_asks(tmp_path: Path):+ from codeverse.cli import _common as C+ from codeverse.contracts.common import Budget, Language, Track+ from codeverse.contracts.spec import Spec++ spec = Spec(id="x", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="p",+ budget=Budget(max_rounds=3))+ s = Settings()+ assert C.round_policy_options(spec, s) == {} # n=1: the track keeps its own policy+ s.apply_profile("quality", force=True)+ opts = C.round_policy_options(spec, s)+ assert opts["policy"].judge_samples == 3 and opts["policy"].max_rounds == 3+ # the rubric threshold is bound here because injecting a policy skips BaseTrack's own binding+ assert opts["policy"].target == pytest.approx(0.72)+++# --------------------------------------------------------------- flag == env var+def _dial_from_flag(name: str):+ """The dial `3dcv make --profile <name>` resolves to."""+ from codeverse.cli import _common as C++ return C.resolve_dial(Settings(), name)+++def _dial_from_env(name: str, monkeypatch):+ """The dial `CV3D_PROFILE=<name> 3dcv make` resolves to (the real settings path)."""+ from codeverse.cli import _common as C++ monkeypatch.setenv("CV3D_PROFILE", name)+ get_settings.cache_clear()+ try:+ return C.resolve_dial(get_settings(), None)+ finally:+ get_settings.cache_clear()++[email protected]("name", PROFILE_NAMES)+def test_the_flag_and_the_env_var_resolve_to_the_same_dial(name, monkeypatch):+ """`--profile X` and `CV3D_PROFILE=X` must set the WHOLE dial, not two thirds+ of it: the verifier found the env path skipped candidates and the texture pass+ because the CLI read those off the flag instead of the resolved profile."""+ assert _dial_from_flag(name) == _dial_from_env(name, monkeypatch)+++ ("name", "expected"),+ [+ ("economy", {"generator": "single-shot:gemini:gemini-3.7-flash",+ "judge": "gemini:gemini-3.7-flash", "judge_samples": 2,+ "rounds": 2, "candidates": 1, "texture": False,+ "judge_max_px": 1024, "judge_montages": 3, "judge_detail_crops": 2,+ "agent_max_turns": 0, "max_usd": 1.50, "max_minutes": 30.0}),+ ("balanced", {"generator": "api-agent:gemini:gemini-3.7-flash",+ "judge": "gemini:gemini-3.1-pro-preview", "judge_samples": 1,+ "rounds": 4, "candidates": 1, "texture": False,+ "judge_max_px": 1024, "judge_montages": 3, "judge_detail_crops": 2,+ "agent_max_turns": 0, "max_usd": 5.0, "max_minutes": 60.0}),+ ("quality", {"generator": "api-agent:gemini:gemini-3.7-flash",+ "judge": "gemini:gemini-3.1-pro-preview", "judge_samples": 3,+ "rounds": 4, "candidates": 2, "texture": True,+ "judge_max_px": 1024, "judge_montages": 3, "judge_detail_crops": 2,+ "agent_max_turns": 0, "max_usd": 8.0, "max_minutes": 90.0}),+ ],+)+def test_each_profile_resolves_to_its_documented_dial(name, expected, monkeypatch):+ """Every dial value docs/COST.md §15 promises, from BOTH entry points."""+ for dial in (_dial_from_flag(name), _dial_from_env(name, monkeypatch)):+ assert dial.profile == name+ for field, want in expected.items():+ assert getattr(dial, field) == want, f"{name}.{field}"++[email protected]("name", PROFILE_NAMES)+def test_the_env_var_reaches_the_spec_a_make_writes(name, tmp_path: Path, monkeypatch):+ """End to end: CV3D_PROFILE alone must produce the same spec shape as --profile."""+ from codeverse.cost.profiles import PROFILES++ p = PROFILES[name]+ monkeypatch.setenv("CV3D_PROFILE", name)+ get_settings.cache_clear()+ runs = tmp_path / "runs"+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run"])+ assert r.exit_code == 0, r.output+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())+ assert spec["options"] == {"candidates": p.candidates, "texture": p.texture, "profile": name}+ assert spec["budget"]["max_rounds"] == p.rounds and spec["budget"]["max_usd"] == p.max_usd+ assert spec["backends"]["judge"] == p.judge and spec["backends"]["generator"] == p.generatordiff --git a/harness/tests/cost/test_reconstruct.py b/harness/tests/cost/test_reconstruct.pyindex e24b3a91..028396d0 100644--- a/harness/tests/cost/test_reconstruct.py+++ b/harness/tests/cost/test_reconstruct.py@@ -29,6 +29,47 @@ def test_generate_event_does_not_double_count_its_session(fake_run: Path):assert not [r for r in led.rows if r.source == "event:generate.done"]+def test_a_retried_session_does_not_double_count_its_generate_event(fake_run: Path, tmp_path: Path):+ """The de-duplication bug behind docs/COST.md §6.++ A retry records itself as ``<label>.a2`` while the ``generate.done`` event+ carries the plain job label, so keying the "already covered" pool on the+ session's own label left the retry's dollars in a different bucket, the pool+ ran dry and the event was counted a SECOND time — $3.58 of the reconstructed+ $89.88 across the 61 recorded runs."""+ import shutil++ ws = tmp_path / "retried"+ shutil.copytree(fake_run, ws)+ # split the baseline into a first attempt and a retry, as the harness records them+ first = ws / "trajectories" / "baseline_r00"+ retry = ws / "trajectories" / "baseline.a2_r00"+ shutil.copytree(first, retry)+ res = json.loads((retry / "result.json").read_text())+ res["label"] = "baseline.a2"+ for key, factor in (("input_tokens", 0.5), ("output_tokens", 0.5), ("cached_tokens", 0.5),+ ("cost_usd", 0.5)):+ res["usage"][key] = type(res["usage"][key])(res["usage"][key] * factor)+ (retry / "result.json").write_text(json.dumps(res))+ turns = [json.loads(line) for line in (retry / "transcript.jsonl").read_text().splitlines()]+ for t in turns:+ for key in ("input_tokens", "output_tokens", "cached_tokens", "cost_usd"):+ t["usage"][key] = type(t["usage"][key])(t["usage"][key] * 0.5)+ (retry / "transcript.jsonl").write_text("\n".join(json.dumps(t) for t in turns))+ # the event summarises the WHOLE job (first attempt + retry) under the job label+ events = [json.loads(line) for line in (ws / "events.jsonl").read_text().splitlines()]+ for ev in events:+ if ev.get("event") == "generate.done":+ ev["cost_usd"] = round(ev["cost_usd"] * 1.5, 6)+ (ws / "events.jsonl").write_text("\n".join(json.dumps(e) for e in events))++ led = reconstruct(ws)+ assert not [r for r in led.rows if r.source == "event:generate.done"], \+ "the retry's dollars cover the job event; nothing may be added on top"+ gen = [r for r in led.rows if r.stage is Stage.BASELINE]+ assert len(gen) == 4 and all(r.source == "transcript" for r in gen) # 2 turns x 2 attempts++def test_judge_row_carries_the_judge_model(fake_run: Path):row = next(r for r in reconstruct(fake_run).rows if r.role is Role.JUDGE)assert row.model == "gemini-3.1-pro-preview" and row.provider == "gemini"@@ -75,6 +116,23 @@ def test_reference_runs_reconcile():assert led.notes, f"{led.run}: unexplained extra ${gap:.4f}"[email protected](not (REPO / "bench" / "out" / "articulated_v1_flash").is_dir(),+ reason="the recorded bench batteries are not present")+ ("path", "record_usd"),+ [("bench/out/articulated_v1_flash/runs/art_hard_door_handle", 3.0690),+ ("bench/out/static_v1_flash/runs/tool_med_hand_drill", 4.0210),+ ("bench/out/articulated_v1_flash/runs/art_easy_laptop", 1.1223)],+)+def test_runs_with_retried_sessions_reconcile_instead_of_looking_off_record(path: str, record_usd: float):+ """The three worst rows of the old docs/COST.md §6 table ($1.38 / $0.50 / $0.31+ "off record") were the retry de-duplication bug, not lost money."""+ led = reconstruct(REPO / path)+ assert led.recorded_usd == pytest.approx(record_usd, abs=0.001)+ found = sum(r.recorded_usd for r in led.rows if r.source != "residual")+ assert found <= led.recorded_usd + 0.005, f"{path}: ${found - led.recorded_usd:.4f} counted twice"++@pytest.mark.skipif(not (REPO / "runs" / "e2e_chair_blender" / "record.json").is_file(),reason="recorded reference runs are not present")def test_gemini_cli_recheck_finds_the_stale_parse():diff --git a/harness/tests/flywheel_cli/test_bench.py b/harness/tests/flywheel_cli/test_bench.pyindex 51804067..405a4946 100644--- a/harness/tests/flywheel_cli/test_bench.py+++ b/harness/tests/flywheel_cli/test_bench.py@@ -83,3 +83,38 @@ def test_run_battery_resume_and_report(tmp_path: Path):page = (out / "report.html").read_text()assert "data:image/jpeg;base64," in page and "furn_easy_stool" in page # self-contained galleryassert "veh_easy_toy_car" in page and "boom" in page # errored prompt still gets a card+++def test_every_bench_prompt_opens_its_own_run_ledger(tmp_path: Path):+ """The batteries produce most of the runs; without a ledger their per-call rows+ went to the per-process fallback log instead of the run (docs/COST.md §12)."""+ from codeverse.contracts.chat import ChatMessage, ChatRequest, ChatResponse+ from codeverse.contracts.common import Usage+ from codeverse.cost.instrument import MeteredChatModel+ from codeverse.cost.ledger import load_ledger++ class FakeChat:+ provider, model, id = "gemini", "gemini-3.7-flash", "gemini:gemini-3.7-flash"++ def supports_vision(self) -> bool:+ return True++ def generate(self, request: ChatRequest) -> ChatResponse:+ return ChatResponse(text="ok", usage=Usage(backend="gemini", model="gemini-3.7-flash",+ input_tokens=1000, output_tokens=10))++ inner = _fake_run_fn({})++ def run(spec, ws: Workspace, resume: bool) -> RunRecord:+ MeteredChatModel(FakeChat()).generate(+ ChatRequest(messages=[ChatMessage.user("x")], label="api-agent:baseline:t0"))+ return inner(spec, ws, resume)++ battery = REPO / "bench" / "prompts" / "static_objects_v1.yaml"+ out = tmp_path / "bench_out"+ res = run_battery(battery, out, BenchOptions(parallel=2, limit=3), run_fn=run)+ assert len(res) == 3+ for r in res:+ rows = load_ledger(Path(r.workspace))+ assert len(rows) == 1, f"{r.id}: {rows}"+ assert rows[0].run == r.id and rows[0].cost_usd > 0 # and not a sibling's rowdiff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.pyindex 90951b0e..f5dc6926 100644--- a/harness/tests/judges/test_vlm_judge.py+++ b/harness/tests/judges/test_vlm_judge.py@@ -31,7 +31,7 @@ def test_full_path_single_sample(judge_input, cache_dir):assert "[verdict: overall 0.80 vs threshold 0.72 → PASS]" in j.summaryreq = model.requests[0]assert req.response_schema is not None and "Criteria" in req.response_schema["$defs"]- assert req.label == "judge:static_object_v1:s0" and req.temperature == 0.2+ assert req.label == "judge:static_object_v1:r00:s0" and req.temperature == 0.2assert len(image_parts(req)) == 3 # 2×2 montage + 2 detail cropsassert "BLIND JUDGE" in req.system and "DEFECT CHECKLIST" in req.systemassert "Defects" in req.response_schema["$defs"]@@ -163,3 +163,27 @@ def test_no_acceptance_items(judge_input, cache_dir):j = _judge(model, cache_dir=cache_dir).judge(judge_input)assert j.passed and j.acceptance_results == {}assert "acceptance" not in model.requests[0].response_schema["properties"]+++# --------------------------------------------------------------- payload size (docs/COST.md §3)+def test_judge_payload_size_comes_from_the_settings_dial(monkeypatch):+ """``Settings.judge`` (set coherently by a cost profile) sizes the verdict's+ images unless the caller states its own."""+ from codeverse.config import Settings, get_settings+ from codeverse.judges.vlm_judge import VlmJudge++ s = Settings()+ s.apply_profile("economy", force=True)+ monkeypatch.setattr("codeverse.judges.vlm_judge.get_settings", lambda: s)+ j = VlmJudge(rubric="static_object_v1", model_id="fake:fake-1")+ # no profile shrinks the payload: 768 px bills the same as 1024 on Gemini and is+ # noisier, and the 2→1 crop cut did not survive a second draw (docs/COST.md §14)+ assert j.max_px == 1024 and j.detail_crops == 2 and j.max_montages == 3+ # a caller that states a size still gets it — the dial is the default, not a cap+ explicit = VlmJudge(rubric="static_object_v1", model_id="fake:fake-1", max_px=768, detail_crops=1)+ assert explicit.max_px == 768 and explicit.detail_crops == 1+ s2 = Settings(judge={"detail_crops": 0, "max_px": 512})+ monkeypatch.setattr("codeverse.judges.vlm_judge.get_settings", lambda: s2)+ s2.apply_profile("economy") # a stated payload survives a profile+ assert VlmJudge(rubric="static_object_v1", model_id="fake:fake-1").detail_crops == 0+ get_settings.cache_clear()diff --git a/harness/tests/orchestrator_tracks/data/offrecord_runs.json b/harness/tests/orchestrator_tracks/data/offrecord_runs.jsonnew file mode 100644index 00000000..b70e0f73--- /dev/null+++ b/harness/tests/orchestrator_tracks/data/offrecord_runs.json@@ -0,0 +1,534 @@+{+ "_source": "recorded telemetry of three runs from the docs/COST.md \u00a76 off-record list (bench/out/*/runs/<slug>); read-only reference data, copied here as fixtures",+ "runs": {+ "art_hard_door_handle": {+ "status": "plateau",+ "stop_reason": "max_rounds",+ "recorded_total_usd": 3.068995,+ "recorded_total_input_tokens": 9946812,+ "rounds_in_record": [+ 0,+ 1,+ 2,+ 3+ ],+ "round_usd": {+ "0": 0.771663,+ "1": 0.474745,+ "2": 0.331447,+ "3": 0.750977+ },+ "sessions": [+ {+ "dir": "baseline.a2_r00",+ "label": "baseline.a2",+ "job_label": "baseline",+ "round": 0,+ "attempt": 2,+ "exit_reason": "completed",+ "turns": 82,+ "cost_usd": 0.663884,+ "input_tokens": 2500987,+ "output_tokens": 66902,+ "cached_tokens": 2167020,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "baseline_r00",+ "label": "baseline",+ "job_label": "baseline",+ "round": 0,+ "attempt": 1,+ "exit_reason": "error",+ "turns": 9,+ "cost_usd": 0.035494,+ "input_tokens": 53988,+ "output_tokens": 59,+ "cached_tokens": 8119,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine.a2_r01",+ "label": "refine.a2",+ "job_label": "refine",+ "round": 1,+ "attempt": 2,+ "exit_reason": "completed",+ "turns": 82,+ "cost_usd": 0.357542,+ "input_tokens": 1529537,+ "output_tokens": 15943,+ "cached_tokens": 1258366,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine.a2_r03",+ "label": "refine.a2",+ "job_label": "refine",+ "round": 3,+ "attempt": 2,+ "exit_reason": "completed",+ "turns": 84,+ "cost_usd": 0.683181,+ "input_tokens": 2226412,+ "output_tokens": 51036,+ "cached_tokens": 1745205,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_r01",+ "label": "refine",+ "job_label": "refine",+ "round": 1,+ "attempt": 1,+ "exit_reason": "error",+ "turns": 13,+ "cost_usd": 0.045647,+ "input_tokens": 103904,+ "output_tokens": 79,+ "cached_tokens": 48796,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_r02",+ "label": "refine",+ "job_label": "refine",+ "round": 2,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 48,+ "cost_usd": 0.257921,+ "input_tokens": 829760,+ "output_tokens": 20677,+ "cached_tokens": 654722,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_r03",+ "label": "refine",+ "job_label": "refine",+ "round": 3,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 96,+ "cost_usd": 0.726272,+ "input_tokens": 2653708,+ "output_tokens": 46718,+ "cached_tokens": 2132151,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ }+ ],+ "judges": [+ {+ "round": 0,+ "rubric": "articulated_v1",+ "cost_usd": 0.072284,+ "input_tokens": 11668,+ "output_tokens": 2335+ },+ {+ "round": 1,+ "rubric": "articulated_v1",+ "cost_usd": 0.071556,+ "input_tokens": 12102,+ "output_tokens": 2260+ },+ {+ "round": 2,+ "rubric": "articulated_v1",+ "cost_usd": 0.073526,+ "input_tokens": 10681,+ "output_tokens": 1649+ },+ {+ "round": 3,+ "rubric": "articulated_v1",+ "cost_usd": 0.067796,+ "input_tokens": 10558,+ "output_tokens": 1868+ }+ ],+ "post_hoc_texture_passes": []+ },+ "tool_med_hand_drill": {+ "status": "budget",+ "stop_reason": "budget",+ "recorded_total_usd": 4.021042,+ "recorded_total_input_tokens": 16463068,+ "rounds_in_record": [+ 0,+ 1+ ],+ "round_usd": {+ "0": 0.624707,+ "1": 1.095167+ },+ "sessions": [+ {+ "dir": "baseline_r00",+ "label": "baseline",+ "job_label": "baseline",+ "round": 0,+ "attempt": 1,+ "exit_reason": "budget",+ "turns": 121,+ "cost_usd": 0.580225,+ "input_tokens": 3012928,+ "output_tokens": 39110,+ "cached_tokens": 2705383,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_assembly_r02",+ "label": "refine_assembly",+ "job_label": "refine_assembly",+ "round": 2,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 50,+ "cost_usd": 0.181432,+ "input_tokens": 637080,+ "output_tokens": 5733,+ "cached_tokens": 470929,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_drill_bit.a2_r01",+ "label": "refine_drill_bit.a2",+ "job_label": "refine_drill_bit",+ "round": 1,+ "attempt": 2,+ "exit_reason": "completed",+ "turns": 16,+ "cost_usd": 0.082922,+ "input_tokens": 110928,+ "output_tokens": 657,+ "cached_tokens": 4056,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_drill_bit_r01",+ "label": "refine_drill_bit",+ "job_label": "refine_drill_bit",+ "round": 1,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 20,+ "cost_usd": 0.097178,+ "input_tokens": 144813,+ "output_tokens": 597,+ "cached_tokens": 20253,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_drill_bit_r02",+ "label": "refine_drill_bit",+ "job_label": "refine_drill_bit",+ "round": 2,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 24,+ "cost_usd": 0.07962,+ "input_tokens": 208685,+ "output_tokens": 676,+ "cached_tokens": 117672,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_main_housing.a2_r01",+ "label": "refine_main_housing.a2",+ "job_label": "refine_main_housing",+ "round": 1,+ "attempt": 2,+ "exit_reason": "completed",+ "turns": 32,+ "cost_usd": 0.120601,+ "input_tokens": 330206,+ "output_tokens": 3374,+ "cached_tokens": 206972,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_main_housing_r01",+ "label": "refine_main_housing",+ "job_label": "refine_main_housing",+ "round": 1,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 40,+ "cost_usd": 0.20125,+ "input_tokens": 411341,+ "output_tokens": 2065,+ "cached_tokens": 170369,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_main_housing_r02",+ "label": "refine_main_housing",+ "job_label": "refine_main_housing",+ "round": 2,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 20,+ "cost_usd": 0.107131,+ "input_tokens": 191298,+ "output_tokens": 2745,+ "cached_tokens": 69091,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_overall.a2_r01",+ "label": "refine_overall.a2",+ "job_label": "refine_overall",+ "round": 1,+ "attempt": 2,+ "exit_reason": "completed",+ "turns": 64,+ "cost_usd": 0.300438,+ "input_tokens": 1045197,+ "output_tokens": 7017,+ "cached_tokens": 755220,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_overall_r01",+ "label": "refine_overall",+ "job_label": "refine_overall",+ "round": 1,+ "attempt": 1,+ "exit_reason": "budget",+ "turns": 121,+ "cost_usd": 0.680978,+ "input_tokens": 3165991,+ "output_tokens": 49693,+ "cached_tokens": 2784983,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_rubber_overmold.a2_r01",+ "label": "refine_rubber_overmold.a2",+ "job_label": "refine_rubber_overmold",+ "round": 1,+ "attempt": 2,+ "exit_reason": "completed",+ "turns": 106,+ "cost_usd": 0.48986,+ "input_tokens": 2358423,+ "output_tokens": 22979,+ "cached_tokens": 2022413,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_rubber_overmold_r01",+ "label": "refine_rubber_overmold",+ "job_label": "refine_rubber_overmold",+ "round": 1,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 92,+ "cost_usd": 0.444199,+ "input_tokens": 2197274,+ "output_tokens": 25188,+ "cached_tokens": 1923276,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_rubber_overmold_r02",+ "label": "refine_rubber_overmold",+ "job_label": "refine_rubber_overmold",+ "round": 2,+ "attempt": 1,+ "exit_reason": "budget",+ "turns": 121,+ "cost_usd": 0.496051,+ "input_tokens": 2612152,+ "output_tokens": 23083,+ "cached_tokens": 2295740,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ }+ ],+ "judges": [+ {+ "round": 0,+ "rubric": "static_object_v1",+ "cost_usd": 0.044482,+ "input_tokens": 10802,+ "output_tokens": 1956+ },+ {+ "round": 1,+ "rubric": "static_object_v1",+ "cost_usd": 0.062174,+ "input_tokens": 10957,+ "output_tokens": 2020+ }+ ],+ "post_hoc_texture_passes": []+ },+ "furn_hard_rolltop_desk": {+ "status": "passed",+ "stop_reason": "pass",+ "recorded_total_usd": 1.216518,+ "recorded_total_input_tokens": 3381927,+ "rounds_in_record": [+ 0,+ 1,+ 2+ ],+ "round_usd": {+ "0": 0.479199,+ "1": 0.327666,+ "2": 0.398044+ },+ "sessions": [+ {+ "dir": "baseline_r00",+ "label": "baseline",+ "job_label": "baseline",+ "round": 0,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 108,+ "cost_usd": 0.425861,+ "input_tokens": 2144051,+ "output_tokens": 21382,+ "cached_tokens": 1870162,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_desk_top_r02",+ "label": "refine_desk_top",+ "job_label": "refine_desk_top",+ "round": 2,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 16,+ "cost_usd": 0.079404,+ "input_tokens": 99297,+ "output_tokens": 1315,+ "cached_tokens": 0,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_drawer_front_r02",+ "label": "refine_drawer_front",+ "job_label": "refine_drawer_front",+ "round": 2,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 44,+ "cost_usd": 0.151169,+ "input_tokens": 471835,+ "output_tokens": 3576,+ "cached_tokens": 320173,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_leather_inlay_r01",+ "label": "refine_leather_inlay",+ "job_label": "refine_leather_inlay",+ "round": 1,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 24,+ "cost_usd": 0.121015,+ "input_tokens": 172309,+ "output_tokens": 1460,+ "cached_tokens": 20284,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_overall_r01",+ "label": "refine_overall",+ "job_label": "refine_overall",+ "round": 1,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 40,+ "cost_usd": 0.144453,+ "input_tokens": 309989,+ "output_tokens": 504,+ "cached_tokens": 133228,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ },+ {+ "dir": "refine_overall_r02",+ "label": "refine_overall",+ "job_label": "refine_overall",+ "round": 2,+ "attempt": 1,+ "exit_reason": "completed",+ "turns": 22,+ "cost_usd": 0.110066,+ "input_tokens": 148847,+ "output_tokens": 309,+ "cached_tokens": 4041,+ "backend": "api-agent",+ "model": "gemini:gemini-3.7-flash"+ }+ ],+ "judges": [+ {+ "round": 0,+ "rubric": "static_object_v1",+ "cost_usd": 0.053338,+ "input_tokens": 10601,+ "output_tokens": 1639+ },+ {+ "round": 1,+ "rubric": "static_object_v1",+ "cost_usd": 0.062198,+ "input_tokens": 10705,+ "output_tokens": 1721+ },+ {+ "round": 2,+ "rubric": "static_object_v1",+ "cost_usd": 0.057404,+ "input_tokens": 10750,+ "output_tokens": 1679+ }+ ],+ "post_hoc_texture_passes": [+ {+ "cost_usd": 0.294,+ "shipped": false+ },+ {+ "cost_usd": 0.0919,+ "shipped": false+ },+ {+ "cost_usd": 0.2703,+ "shipped": true+ }+ ]+ }+ }+}diff --git a/harness/tests/orchestrator_tracks/test_cost_accounting.py b/harness/tests/orchestrator_tracks/test_cost_accounting.pynew file mode 100644index 00000000..c969369a--- /dev/null+++ b/harness/tests/orchestrator_tracks/test_cost_accounting.py@@ -0,0 +1,146 @@+"""Closing the accounting holes docs/COST.md §6 found, and the money stops of §5.++The audit measured $4.80 of real spend that never reached ``record.total_usage``+(and therefore never reached ``BudgetGuard``): rounds the budget cut after the+work was done, retried ``.a2`` agent sessions, post-hoc texture passes. These+tests replay the *recorded* usage of three of those runs+(``data/offrecord_runs.json``, taken from the read-only bench runs) through the+production accounting objects and assert that every dollar now lands in the+guard, in one ledger row, and in a round bucket.+"""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from codeverse.contracts.agent import AgentJob, AgentResult+from codeverse.contracts.common import Budget, Usage+from codeverse.cost.ledger import load_ledger+from codeverse.orchestrator.budget import BudgetExceeded, BudgetGuard, usage_delta+from codeverse.tracks.generation import GenerationTask, _SessionAcc++FIXTURES = json.loads((Path(__file__).parent / "data" / "offrecord_runs.json").read_text())["runs"]+++def _usage(row: dict) -> Usage:+ return Usage(backend=row.get("backend") or "api-agent", model=row.get("model") or "gemini:gemini-3.7-flash",+ cost_usd=row["cost_usd"], input_tokens=row["input_tokens"], output_tokens=row["output_tokens"],+ cached_tokens=row.get("cached_tokens", 0))+++class _RecordedAgent:+ """Replays the recorded sessions of one task, attempt by attempt."""++ kind = "api-agent"+ model = "gemini:gemini-3.7-flash"++ def __init__(self, sessions: list[dict]):+ self.sessions = list(sessions)++ def run(self, job: AgentJob) -> AgentResult:+ s = self.sessions.pop(0)+ return AgentResult(ok=True, exit_reason=s["exit_reason"] or "completed", usage=_usage(s))+++def replay(name: str, ledger: Path | None, *, texture: bool = True) -> BudgetGuard:+ """Spend one recorded run's money through today's accounting path."""+ fx = FIXTURES[name]+ guard = BudgetGuard(Budget(max_usd=10_000, max_minutes=10_000), run=name, ledger=ledger)+ tasks: dict[tuple[str, int], list[dict]] = {}+ for s in fx["sessions"]:+ tasks.setdefault((s["job_label"] or s["label"], int(s["round"] or 0)), []).append(s)+ for (label, rnd), sessions in tasks.items():+ sessions.sort(key=lambda s: s["attempt"])+ task = GenerationTask(label=label, prompt="p", round=rnd,+ kind="baseline" if label.startswith("baseline") else "refine")+ acc = _SessionAcc(task=task, budget=guard)+ agent = _RecordedAgent(sessions)+ for _ in sessions: # attempt 1, then the .a2 retry — exactly as run_agent_task does+ acc.run(agent, AgentJob(workspace="/tmp", prompt="p", label=label, round=rnd))+ for j in fx["judges"]: # steps.run_round+ guard.add(Usage(cost_usd=j["cost_usd"], input_tokens=j["input_tokens"], output_tokens=j["output_tokens"]),+ stage="judge", role="judge", round_index=j["round"], label=j["rubric"])+ if texture:+ for t in fx["post_hoc_texture_passes"]: # lifecycle._texture_pass+ guard.add(Usage(cost_usd=t["cost_usd"]), stage="texture", role="image", label="texture_pass")+ return guard++[email protected]("name", sorted(FIXTURES))+def test_every_recorded_dollar_reaches_the_guard_and_the_ledger(name, tmp_path):+ fx = FIXTURES[name]+ ledger = tmp_path / "cost_ledger.jsonl"+ guard = replay(name, ledger)+ sessions = sum(s["cost_usd"] for s in fx["sessions"])+ judges = sum(j["cost_usd"] for j in fx["judges"])+ texture = sum(t["cost_usd"] for t in fx["post_hoc_texture_passes"])+ # nothing is lost between the sessions and the guard — retries included+ assert guard.spent.cost_usd == pytest.approx(sessions + judges + texture, abs=1e-6)+ # and the ledger the run ships is the same number, one row per call+ rows = load_ledger(ledger)+ assert len(rows) == len(fx["sessions"]) + len(fx["judges"]) + len(fx["post_hoc_texture_passes"])+ assert sum(r.cost_usd for r in rows) == pytest.approx(guard.spent.cost_usd, abs=1e-6)+ # a retried session is its OWN row, labelled .a2 — the reconstructor had to guess+ # this from a trajectory directory name, and double-counted when it guessed wrong+ retried = [s for s in fx["sessions"] if s["attempt"] > 1]+ assert sum(1 for r in rows if r.label.endswith(".a2")) == len(retried)+++def test_the_round_the_budget_cut_is_still_reported():+ """tool_med_hand_drill spent $0.86 on r02 and then stopped: the round is not in+ ``record.rounds`` at all, so its dollars used to be attributable to nothing."""+ fx = FIXTURES["tool_med_hand_drill"]+ assert fx["rounds_in_record"] == [0, 1]+ cut = [s for s in fx["sessions"] if s["round"] == 2]+ assert cut, "fixture must contain the round the budget cut"+ guard = replay("tool_med_hand_drill", None)+ burned = sum(s["cost_usd"] for s in cut)+ assert guard.round_costs(2)["refine"] == pytest.approx(burned, abs=1e-6)+ assert burned > 0.8 # $0.86 that record.rounds never mentioned+++def test_post_hoc_texture_passes_are_inside_the_total_now():+ """furn_hard_rolltop_desk ran four texture passes for $0.656; ``record.total_usage``+ said $1.2165. Charged through the guard the run reports $1.87."""+ fx = FIXTURES["furn_hard_rolltop_desk"]+ with_tex = replay("furn_hard_rolltop_desk", None)+ without = replay("furn_hard_rolltop_desk", None, texture=False)+ off_record = with_tex.spent.cost_usd - without.spent.cost_usd+ assert off_record == pytest.approx(0.6562, abs=1e-3)+ assert without.spent.cost_usd == pytest.approx(fx["recorded_total_usd"], abs=0.02)+ assert with_tex.by_stage["texture"] == pytest.approx(0.6562, abs=1e-3)+++# ----------------------------------------------------------------------------- guard mechanics+def test_spend_buckets_by_stage_and_round_and_only_charge_enforces():+ g = BudgetGuard(Budget(max_usd=1.0, max_minutes=60))+ g.charge(Usage(cost_usd=0.4), stage="baseline", round_index=0, label="baseline")+ g.add(Usage(cost_usd=0.05), stage="judge", role="judge", round_index=0)+ assert g.by_stage == {"baseline": pytest.approx(0.4), "judge": pytest.approx(0.05)}+ assert g.round_costs(0)["judge"] == pytest.approx(0.05) and g.round_costs(9) == {}+ g.add(Usage(cost_usd=0.9), stage="texture", role="image") # over the ceiling, but never raises+ assert not g.ok() and g.spent.cost_usd == pytest.approx(1.35)+ with pytest.raises(BudgetExceeded):+ g.charge(Usage(cost_usd=0.01), stage="refine")+ assert g.spent.cost_usd == pytest.approx(1.36) # the money is counted even when it raises+ assert list(g.stage_summary())[0] == "texture" # biggest bucket first+++def test_usage_delta_reports_what_a_round_burned():+ before = Usage(cost_usd=1.0, input_tokens=100, output_tokens=10)+ after = Usage(cost_usd=1.75, input_tokens=350, output_tokens=40)+ d = usage_delta(after, before)+ assert d.cost_usd == pytest.approx(0.75) and d.input_tokens == 250 and d.output_tokens == 30+ assert usage_delta(before, after).cost_usd == 0.0 # never negative+++def test_the_ledger_is_optional_and_never_breaks_a_run(tmp_path):+ g = BudgetGuard(Budget(max_usd=1.0, max_minutes=60), ledger=tmp_path / "sub" / "dir" / "l.jsonl")+ g.charge(Usage(cost_usd=0.1), stage="plan", role="planner")+ assert len(load_ledger(tmp_path / "sub" / "dir" / "l.jsonl")) == 1+ quiet = BudgetGuard(Budget(max_usd=1.0, max_minutes=60)) # no ledger configured+ quiet.charge(Usage(cost_usd=0.1))+ assert quiet.spent.cost_usd == pytest.approx(0.1)diff --git a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py b/harness/tests/orchestrator_tracks/test_generation_planner_repair.pyindex 44610ecf..adaebf7d 100644--- a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py+++ b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py@@ -92,7 +92,8 @@ def test_generate_single_shot_writes_files_and_charges(tmp_ws):res = generate(tmp_ws, agent_id="single-shot:gemini:x", task=task, model=model, budget=budget, events=events)assert res.ok and [c.path for c in res.files_changed] == ["src/model.py"] and (tmp_ws.src / "model.py").read_text() == "import bpy\n"assert budget.spent.cost_usd == pytest.approx(0.002)- assert "=== FILE:" in model.requests[0].system and model.requests[0].label == "baseline"+ # the label carries the round so the cost ledger can attribute a single-shot call+ assert "=== FILE:" in model.requests[0].system and model.requests[0].label == "baseline:r00"assert (tmp_ws.trajectories / "baseline_r00" / "response.md").is_file()# parse failure → ok=False, no raisebad = FakeChatModel(lambda req: "nope")diff --git a/harness/tests/orchestrator_tracks/test_orchestrator.py b/harness/tests/orchestrator_tracks/test_orchestrator.pyindex 44a4ea74..2dd4ffca 100644--- a/harness/tests/orchestrator_tracks/test_orchestrator.py+++ b/harness/tests/orchestrator_tracks/test_orchestrator.py@@ -124,7 +124,11 @@ def test_stop_policy_decisions():assert sp.decide([_round(0, 0.5)]) == "continue"assert sp.decide([_round(0, 0.5)], budget_ok=False) == "budget"assert sp.decide([_round(0, 0.5), _round(1, 0.85)]) == "pass"- assert sp.decide([_round(0, 0.5), _round(1, 0.51), _round(2, 0.515)]) == "plateau"+ # three flat rounds: from r03 on the marginal-value stop answers first (both stop;+ # "diminishing_returns" is the more precise reason — see test_round_economics.py)+ assert sp.decide([_round(0, 0.5), _round(1, 0.51), _round(2, 0.515)]) == "diminishing_returns"+ flat = StopPolicy(RoundPolicy(max_rounds=3, plateau_window=2, min_delta=0.02, target=0.8, marginal_from_round=99))+ assert flat.decide([_round(0, 0.5), _round(1, 0.51), _round(2, 0.515)]) == "plateau"assert sp.decide([_round(0, 0.5), _round(1, 0.6), _round(2, 0.7)]) == "continue"assert sp.decide([_round(0, 0.5), _round(1, 0.6), _round(2, 0.7), _round(3, 0.75)]) == "max_rounds"# unscored rounds (build failed) do not count as plateau evidencediff --git a/harness/tests/orchestrator_tracks/test_round_economics.py b/harness/tests/orchestrator_tracks/test_round_economics.pynew file mode 100644index 00000000..a29e8005--- /dev/null+++ b/harness/tests/orchestrator_tracks/test_round_economics.py@@ -0,0 +1,510 @@+"""The round-level cost controls — the ones that survived measurement, and the+proof for the ones that did not.++* a round that scores below the best by more than the judge's noise does not+ buy another round of the same shape (``regression`` / strategy switch);+* r03+ only starts when the previous round gained more than 1.5 σ+ (``diminishing_returns``);+* agent sessions are **uncapped by default** — a 28-turn cap was A/B-tested and+ rejected ($1.381 / 0.479 capped vs $1.360 / 0.684 uncapped, n=3 per arm,+ docs/COST.md §17) — but an explicitly requested cap still lands gracefully;+* the judge is skipped only where the verdict is provably never bought at all,+ and every round emits one ``cost.round`` event with what it burned.+"""++from __future__ import annotations++import json++import pytest++from codeverse.contracts.agent import AgentJob, AgentResult+from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Severity+from codeverse.contracts.common import Language, Usage+from codeverse.contracts.judgment import Judgment+from codeverse.contracts.run import RoundRecord, RunStatus+from codeverse.events import EventLog+from codeverse.orchestrator.rounds import (+ DEFAULT_JUDGE_SIGMA,+ REWRITE_KIND,+ RoundPolicy,+ StopPolicy,+ best_score,+ judge_sigma,+ last_gain,+)+from codeverse.tracks.generation import (+ DEFAULT_AGENT_MAX_TURNS,+ GenerationTask,+ agent_max_turns,+ run_agent_task,+ turn_capped,+)+from codeverse.tracks.static_object import StaticObjectTrack+from codeverse.workspace import Workspace+from tests.orchestrator_tracks.conftest import make_spec+from tests.orchestrator_tracks.fakes import (+ FakeAgent,+ FakeChatModel,+ FakeJudge,+ FakeRuntime,+ FakeServices,+)+++def _round(i: int, score: float | None, kind: str = "refine") -> RoundRecord:+ j = Judgment(rubric="r", scores={}, overall=score, passed=bool(score and score >= 0.8)) if score is not None else None+ return RoundRecord(index=i, kind="baseline" if i == 0 else kind, judgment=j,+ build=BuildResult(ok=True, language="l"), commit=f"c{i}")+++# ----------------------------------------------------------------------------- judge noise table+def test_judge_sigma_comes_from_the_one_measured_table():+ from codeverse.cost.routing import JUDGE_NOISE++ assert judge_sigma("gemini:gemini-3.1-pro-preview") == pytest.approx(JUDGE_NOISE["gemini-3.1-pro-preview"][0])+ assert judge_sigma("gemini:gemini-3.7-flash") == pytest.approx(0.083)+ assert judge_sigma("gemini:gemini-3.7-flash-002") == pytest.approx(0.083) # version suffix+ assert judge_sigma("") == DEFAULT_JUDGE_SIGMA and judge_sigma("who:knows") == DEFAULT_JUDGE_SIGMA+ pol = RoundPolicy(judge_model="gemini:gemini-3.7-flash")+ assert pol.sigma == pytest.approx(0.083)+ assert pol.regression_delta == pytest.approx(0.083) and pol.marginal_delta == pytest.approx(0.1245)+ assert RoundPolicy().with_judge("gemini:gemini-3.7-flash").sigma == pytest.approx(0.083)+ assert RoundPolicy().with_judge("") is not None+++def test_best_score_and_last_gain():+ h = [_round(0, 0.5), _round(1, 0.62), _round(2, 0.55)]+ assert best_score(h) == pytest.approx(0.62) and last_gain(h) == pytest.approx(-0.07)+ assert best_score([]) is None and last_gain([_round(0, 0.5)]) is None+ assert last_gain([_round(0, 0.5), _round(1, None)]) is None+++# ----------------------------------------------------------------------------- regression+def test_a_regression_buys_a_change_of_shape_then_stops():+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, judge_model="gemini:gemini-3.1-pro-preview",+ marginal_from_round=99))+ inside = sp.evaluate([_round(0, 0.60), _round(1, 0.58)]) # −0.02 < σ 0.030 → judge noise+ assert inside.reason == "continue" and inside.strategy == "same"+ first = sp.evaluate([_round(0, 0.60), _round(1, 0.52)]) # −0.08: a real regression+ assert first.reason == "continue" and first.strategy == "switch" and "0.520" in first.detail+ again = sp.evaluate([_round(0, 0.60), _round(1, 0.52), _round(2, 0.50, kind=REWRITE_KIND)])+ assert again.reason == "regression" and "changed strategy" in again.detail+ twice = sp.evaluate([_round(0, 0.60), _round(1, 0.52), _round(2, 0.55)]) # two regressions vs best+ assert twice.reason == "regression"+++def test_regression_can_be_configured_to_stop_outright():+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99, regression_allow_switch=False))+ d = sp.evaluate([_round(0, 0.60), _round(1, 0.52)])+ assert d.reason == "regression" and d.strategy == "same"+++def test_a_noisy_judge_widens_the_regression_band():+ flash = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99,+ judge_model="gemini:gemini-3.7-flash"))+ # −0.06 is a regression for pro (σ 0.030) but inside flash's own noise (σ 0.083)+ assert flash.evaluate([_round(0, 0.60), _round(1, 0.54)]).strategy == "same"+ pro = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99,+ judge_model="gemini:gemini-3.1-pro-preview"))+ assert pro.evaluate([_round(0, 0.60), _round(1, 0.54)]).strategy == "switch"+++# ----------------------------------------------------------------------------- diminishing returns+def test_r03_only_runs_when_the_last_round_paid_for_itself():+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, judge_model="gemini:gemini-3.1-pro-preview"))+ # r02 gained +0.10 ≫ 1.5σ = 0.045 → r03 is worth starting+ assert sp.evaluate([_round(0, 0.4), _round(1, 0.5), _round(2, 0.6)]).reason == "continue"+ # r02 gained +0.03 < 0.045 → not worth another round+ d = sp.evaluate([_round(0, 0.40), _round(1, 0.55), _round(2, 0.58)])+ assert d.reason == "diminishing_returns" and "1.5 × judge σ" in d.detail+ # ... and the rule does not apply to r01/r02, which the audit shows are the good buys+ assert sp.evaluate([_round(0, 0.40), _round(1, 0.42)]).reason == "continue"+ # scoring at/above target without passing (a failed must-item) is not worth+ # another refine round either — the score is not what is missing+ above = RoundRecord(index=2, kind="refine", build=BuildResult(ok=True, language="l"),+ judgment=Judgment(rubric="r", scores={}, overall=0.95, passed=False))+ at_target = sp.evaluate([_round(0, 0.4), _round(1, 0.6), above])+ assert at_target.reason == "diminishing_returns" and "target" in at_target.detail+++def test_the_marginal_round_is_configurable():+ """The audit's recommendation #1 (stop after r01) is this knob, not a new rule."""+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=2,+ judge_model="gemini:gemini-3.1-pro-preview"))+ assert sp.evaluate([_round(0, 0.40), _round(1, 0.42)]).reason == "diminishing_returns"+++def test_the_stop_order_is_exhausted_regression_then_marginal_then_switch_then_plateau():+ """One case per branch, each one where the NEXT branch would answer differently —+ the order in ``StopPolicy.evaluate`` is the economics, so it is pinned here.++ (Replayed over the 107 recorded rounds this shipped default cuts 2 runs /+ $1.14 with 0 best rounds lost — waste-and-accounting/replay_stops.py.)"""+ pro = dict(max_rounds=4, target=0.9, judge_model="gemini:gemini-3.1-pro-preview")+ sp = StopPolicy(RoundPolicy(**pro))+ # 1. the switch is already spent (a rewrite regressed again) — outranks everything,+ # even though the marginal test would also stop here (last gain −0.10)+ spent = [_round(0, 0.60), _round(1, 0.52), _round(2, 0.50, kind=REWRITE_KIND)]+ assert sp.evaluate(spent).reason == "regression" and "changed strategy" in sp.evaluate(spent).detail+ # 2. at r03 the marginal test beats a FIRST regression: no shape is worth $0.5 there+ marginal = [_round(0, 0.60), _round(1, 0.65), _round(2, 0.55)]+ assert sp.evaluate(marginal).reason == "diminishing_returns"+ # 3. before r03 the same first regression buys exactly one change of shape+ switch = sp.evaluate([_round(0, 0.60), _round(1, 0.52)])+ assert (switch.reason, switch.strategy, switch.stop) == ("continue", "switch", False)+ # 4. plateau is the fallback: no regression, marginal test not in force, no gain+ flat = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99, plateau_window=2,+ min_delta=0.02, judge_model="gemini:gemini-3.1-pro-preview"))+ assert flat.evaluate([_round(0, 0.50), _round(1, 0.505), _round(2, 0.51)]).reason == "plateau"+ # ... and a run that is still climbing is never stopped by any of the four+ assert flat.evaluate([_round(0, 0.50), _round(1, 0.60), _round(2, 0.70)]).reason == "continue"+++# ----------------------------------------------------------------------------- turn budget+class _TurnAgent:+ """Burns turns; reports the api-agent's own 'max_turns (N) reached' when it runs out."""++ kind = "api-agent"+ model = "gemini:gemini-3.7-flash"++ def __init__(self, *, writes_on: int = 2, cost: float = 0.2):+ self.jobs: list[AgentJob] = []+ self.writes_on = writes_on+ self.cost = cost++ def run(self, job: AgentJob) -> AgentResult:+ self.jobs.append(job)+ ws = Workspace(job.workspace)+ wrote = []+ if len(self.jobs) >= self.writes_on:+ (ws.src).mkdir(parents=True, exist_ok=True)+ (ws.src / "object.js").write_text(f"// session {len(self.jobs)}\n")+ wrote = [{"path": "src/object.js", "status": "modified"}]+ exhausted = len(self.jobs) == 1+ return AgentResult(ok=bool(wrote), exit_reason="budget" if exhausted else "completed",+ files_changed=wrote, errors=[f"max_turns ({job.max_turns}) reached"] if exhausted else [],+ usage=Usage(backend="api-agent", cost_usd=self.cost, input_tokens=1000))+++def test_agent_sessions_are_uncapped_by_default(tmp_ws, monkeypatch):+ """Measured (docs/COST.md §17): cap 28 cost $1.381 at score 0.479, uncapped+ $1.360 at 0.684 — n=3 per arm, same prompt/generator/judge/budget. The cap+ saved nothing and lost 0.205 of a score point, so nothing sets one by default."""+ monkeypatch.delenv("CV3D_AGENT_MAX_TURNS", raising=False)+ assert DEFAULT_AGENT_MAX_TURNS == 0 and RoundPolicy().agent_max_turns == 0+ agent = _TurnAgent(writes_on=1)+ task = GenerationTask(label="baseline", prompt="p", round=0, kind="baseline")+ run_agent_task(tmp_ws, agent=agent, task=task)+ assert agent.jobs[0].max_turns == AgentJob(workspace="w", prompt="p").max_turns # the backend's own default+ assert agent_max_turns() == 0 # "no cap asked for"+ # ... and every explicit way of asking for one still works+ run_agent_task(tmp_ws, agent=agent, task=task, max_turns=12)+ assert agent.jobs[-1].max_turns == 12+ run_agent_task(tmp_ws, agent=agent, task=task.model_copy(update={"max_turns": 7}), max_turns=12)+ assert agent.jobs[-1].max_turns == 7 # the task wins over the policy+ monkeypatch.setenv("CV3D_AGENT_MAX_TURNS", "9")+ assert agent_max_turns() == 9+ run_agent_task(tmp_ws, agent=agent, task=task)+ assert agent.jobs[-1].max_turns == 9+ monkeypatch.setenv("CV3D_AGENT_MAX_TURNS", "nonsense")+ assert agent_max_turns() == DEFAULT_AGENT_MAX_TURNS == 0+++def test_the_policy_cap_is_plumbed_through_the_round(tmp_path, spec, settings):+ """A caller that DOES set one (a cost profile, a bench arm) still reaches the job."""+ from codeverse.tracks.steps import run_generation_tasks++ agent = _TurnAgent(writes_on=1)+ ctx = _ctx(tmp_path, spec, settings, policy=RoundPolicy(agent_max_turns=15), agent=agent)+ run_generation_tasks(ctx, [GenerationTask(label="baseline", prompt="p", round=0, kind="baseline")])+ assert agent.jobs[0].max_turns == 15+ assert agent.jobs[1].max_turns == RoundPolicy().agent_wrapup_turns # ... and the landing session+++def test_hitting_the_cap_asks_for_a_landing_instead_of_killing_the_session(tmp_ws):+ events = EventLog(tmp_ws.events_path)+ agent = _TurnAgent(writes_on=2) # session 1 burns its turns and writes nothing+ guard = _guard()+ res = run_agent_task(tmp_ws, agent=agent, task=GenerationTask(label="refine", prompt="do the work", round=1,+ kind="refine", max_turns=28),+ budget=guard, events=events, wrapup_turns=5)+ assert len(agent.jobs) == 2 and turn_capped(_result_of(agent, 0))+ wrap = agent.jobs[1]+ assert wrap.max_turns == 5 and "LAST session" in wrap.prompt and "do the work" in wrap.prompt+ assert res.ok and res.turn_capped and res.sessions == 2+ # both sessions are paid for and both are in the guard, under their own labels+ assert res.usage.cost_usd == pytest.approx(0.4) and guard.spent.cost_usd == pytest.approx(0.4)+ assert guard.round_costs(1)["refine"] == pytest.approx(0.4)+ kinds = [e["event"] for e in events.read()]+ assert "generate.turn_cap" in kinds+ done = next(e for e in events.read() if e["event"] == "generate.done")+ assert done["turn_capped"] is True and done["sessions"] == 2+++def test_a_crashing_second_session_never_erases_what_the_first_one_spent(tmp_ws):+ class _Flaky(_TurnAgent):+ def run(self, job: AgentJob) -> AgentResult:+ if self.jobs:+ self.jobs.append(job)+ raise RuntimeError("503 storm")+ return super().run(job)++ guard = _guard()+ agent = _Flaky(writes_on=99)+ res = run_agent_task(tmp_ws, agent=agent, task=GenerationTask(label="refine", prompt="p", round=2, kind="refine"),+ budget=guard)+ assert len(agent.jobs) == 2 and not res.ok+ assert guard.spent.cost_usd == pytest.approx(0.2) and res.usage.cost_usd == pytest.approx(0.2)+ assert "503 storm" in res.notes and guard.round_costs(2)["refine"] == pytest.approx(0.2)+++def _result_of(agent, i):+ return AgentResult(ok=False, exit_reason="budget", errors=[f"max_turns ({agent.jobs[i].max_turns}) reached"])+++def _guard():+ from codeverse.contracts.common import Budget+ from codeverse.orchestrator.budget import BudgetGuard++ return BudgetGuard(Budget(max_usd=100, max_minutes=100))+++def test_task_stage_names_the_cost_bucket_a_task_spends_in():+ from codeverse.tracks.generation import task_stage++ def t(kind: str, label: str = "x") -> str:+ return task_stage(GenerationTask(label=label, prompt="p", kind=kind))++ assert t("baseline") == "baseline" and t("refine") == "refine" and t("repair") == "repair"+ assert t("rebuild") == "repair" # regenerating after a failed build IS repair money+ assert t("zone") == "zones" and t("asset") == "assets" and t("asset_fix") == "assets"+ assert t("compose") == "assemble" and t("env") == "env"+ # an unknown kind falls back to the label, which record_call maps by prefix+ assert t("something_new", "asset_koi") == "asset_koi"+++# ----------------------------------------------------------------------------- judge skipping+def _renders():+ from codeverse.contracts.artifacts import RenderSet, RenderView++ return RenderSet(views=[RenderView(name="front", path="x.png")], renderer="fake")+++def _gates(errors: int = 0):+ return [GateReport(gate="lint", passed=not errors,+ findings=[GateFinding(gate="lint", severity=Severity.ERROR, message="e")] * errors)]+++def test_skip_judge_reasons_are_only_states_where_the_verdict_is_never_bought(tmp_path, spec, settings):+ """Every branch left in ``skip_judge_reason`` must save a REAL verdict.++ The two wave-2 branches that did not are gone: "no file change" was+ unreachable and "build not repaired within the repair budget" was bought back+ by ``rejudge_round`` on the next loop iteration (both reproduced below)."""+ from codeverse.tracks.steps import skip_judge_reason++ ctx = _ctx(tmp_path, spec, settings)+ assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == ""+ # a round with gate errors IS judged by default — the gates say what is broken,+ # the verdict says whether the shape is right, and only the verdict can promote it+ assert skip_judge_reason(ctx, gates=_gates(3), renders=_renders()) == ""+ assert skip_judge_reason(ctx, gates=_gates(), renders=None) == "no renders"+ # the caller can switch that off; then the SAME rule holds in the rejudge path+ strict = _ctx(tmp_path, spec, settings, policy=RoundPolicy(judge_on_gate_errors=False), name="strict")+ assert skip_judge_reason(strict, gates=_gates(3), renders=_renders()) == "gate errors"+ assert skip_judge_reason(strict, gates=_gates(0), renders=_renders()) == ""+ # ... the budget/clock stop: the loop's next budget_ok check ends the run, so this+ # verdict cannot promote anything (audit: 2 verdicts / $0.09 bought past the clock)+ ctx.budget.add(Usage(cost_usd=99.0), stage="refine")+ assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == "budget already exceeded"+ assert StopPolicy(ctx.policy).evaluate([_round(0, 0.5)], budget_ok=ctx.budget.ok()).reason == "budget"+ ctx.judge = None+ assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == "no judge configured"+++def test_a_round_that_changed_no_file_never_reaches_the_judge_question(tmp_path, spec, settings):+ """Proof the deleted "no file change" branch was unreachable: a task that writes+ nothing is not ``ok``, and ``run_generation_tasks`` raises before any verdict."""+ from codeverse.tracks.steps import RoundFailed, run_generation_tasks++ class _Idle:+ kind, model = "api-agent", "m"++ def run(self, job: AgentJob) -> AgentResult:+ return AgentResult(ok=True, exit_reason="completed", files_changed=[], usage=Usage(cost_usd=0.01))++ ctx = _ctx(tmp_path, spec, settings, agent=_Idle(), name="nochange")+ with pytest.raises(RoundFailed):+ run_generation_tasks(ctx, [GenerationTask(label="refine", prompt="p", round=1, kind="refine")])+++def test_a_skipped_verdict_is_not_bought_back_by_the_rejudge_path(tmp_path, spec, settings):+ """``rejudge_round`` exists for a judge that FAILED, not for one we chose not to+ call: without this the "gate errors" skip is a ``judge.retry``, not a saving."""+ from codeverse.tracks.steps import rejudge_round++ ctx = _ctx(tmp_path, spec, settings, policy=RoundPolicy(judge_on_gate_errors=False), name="norebuy")+ rec = RoundRecord(index=1, kind="refine", build=BuildResult(ok=True, language="l"),+ gates=_gates(2), renders=_renders())+ assert rejudge_round(ctx, _pipeline(), rec) is False+ assert ctx.judge.calls == [] and ctx.budget.spent.cost_usd == 0.0+ # the same round with clean gates (an outage, not a policy skip) IS re-judged+ rec2 = RoundRecord(index=2, kind="refine", build=BuildResult(ok=True, language="l"),+ gates=_gates(0), renders=_renders())+ assert rejudge_round(ctx, _pipeline(), rec2) is True and len(ctx.judge.calls) == 1+++def test_a_round_that_broke_the_gates_does_not_displace_a_clean_one():+ """The unscored fallback: never promote a round that does not build, and never+ let a gate-breaking round displace the previous artifact just by being last."""+ from codeverse.orchestrator.rounds import BestSelector++ def r(i, *, build_ok=True, errors=0):+ return RoundRecord(index=i, kind="refine", build=BuildResult(ok=build_ok, language="l"),+ gates=_gates(errors), commit=f"c{i}")++ assert BestSelector().pick([r(0), r(1, build_ok=False)]) == 0 # broken build never wins+ assert BestSelector().pick([r(0), r(1, errors=4)]) == 0 # clean r00 keeps the crown+ assert BestSelector().pick([r(0, errors=4), r(1)]) == 1+ assert BestSelector().pick([r(0), r(1)]) == 1 # tie → the later one+ assert BestSelector().pick([r(0, build_ok=False)]) is None+++class _Pipeline:+ """The minimum ``RoundPipeline`` the judge path touches."""++ def measure(self, ctx, build): return None+ def gates(self, ctx, i, build, m): return []+ def render(self, ctx, i, build, m): return _renders()+ def plan_summary(self, ctx): return "plan"+ def judge_context(self, ws, plan, i, build, gates): return ""+++def _pipeline():+ return _Pipeline()+++def _ctx(tmp_path, spec, settings, *, policy: RoundPolicy | None = None, agent=None, name: str = "skip"):+ from codeverse.contracts.common import Budget+ from codeverse.orchestrator.budget import BudgetGuard+ from codeverse.orchestrator.state import RunState+ from codeverse.tracks.common import RunContext++ ws = Workspace(tmp_path / "runs" / name)+ ws.create()+ return RunContext(spec=spec, ws=ws, events=EventLog(ws.events_path), settings=settings,+ budget=BudgetGuard(Budget(max_usd=1.0, max_minutes=60)), runtime=FakeRuntime(Language.THREEJS),+ services=FakeServices(), state=RunState(), policy=policy or RoundPolicy(), track=spec.track,+ rubric="static_object_v1", agent_id="fake:fake-model", judge=FakeJudge(), agent=agent)+++# ----------------------------------------------------------------------------- end to end+def _planner(plan_dict):+ return FakeChatModel(lambda req: plan_dict)+++def _writer(job, ws):+ return {"src/object.js": f"// {job.label} r{job.round}\nexport function build(THREE) {{}}\n"}+++def test_the_loop_switches_shape_after_a_regression_and_emits_cost_rounds(tmp_path, chair_plan, settings):+ """r01 regresses hard → r02 is ONE whole-object rewrite, not another per-part fan-out."""+ spec = make_spec(max_rounds=3)+ ws = Workspace(tmp_path / "runs" / "regress")+ agent = FakeAgent(_writer, cost=0.02)+ judge = FakeJudge(scores=(0.60, 0.30, 0.35))+ track = StaticObjectTrack(services=FakeServices(contract_errors=2), judge=judge, agent=agent,+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,+ runtime=FakeRuntime(Language.THREEJS),+ policy=RoundPolicy(max_rounds=3, target=0.8, judge_model="gemini:gemini-3.1-pro-preview"))+ rec = track.run(spec, ws)+ assert [r.kind for r in rec.rounds] == ["baseline", "refine", REWRITE_KIND]+ assert rec.extra["stop_reason"] == "regression" and rec.status is RunStatus.PLATEAU+ events = EventLog(ws.events_path).read()+ switch = next(e for e in events if e["event"] == "strategy.switch")+ assert switch["reason"] == "regression"+ rewrite = [j for j in agent.jobs if "rewrite" in j.prompt.lower() or "made the artifact WORSE" in j.prompt]+ assert rewrite, "the rewrite round must tell the agent the last round made it worse"+ assert len([j for j in agent.jobs if j.round == 2]) == 1 # ONE task, not a per-part fan-out+ # every round reports what it burned+ costs = [e for e in events if e["event"] == "cost.round"]+ assert len(costs) == 3 and all("generate" in c["stages"] for c in costs)+ assert costs[0]["agent_turns"] == 0 # the fake agent keeps no transcript+ assert costs[1]["wasted"] is True and costs[1]["waste_reason"] == "regression"+ assert costs[2]["judge_usd"] > 0 and costs[2]["run_usd"] >= costs[1]["run_usd"]+ assert rec.extra["cost_by_stage"]["judge"] > 0 and rec.extra["cost_by_stage"]["refine"] > 0+ # ... and the run ships its own priced ledger+ from codeverse.cost.ledger import load_ledger++ rows = load_ledger(ws.root / "cost_ledger.jsonl")+ assert {str(r.stage) for r in rows} >= {"baseline", "refine", "judge"}+ assert sum(r.cost_usd for r in rows) == pytest.approx(rec.total_usage.cost_usd, abs=1e-6)+++def test_a_lint_stuck_run_keeps_every_score_instead_of_deferring_the_verdict(tmp_path, chair_plan, settings):+ """The verifier's reproduction of the removed "build not repaired" skip.++ Every round here builds but leaves a lint error the repair budget cannot clear.+ Wave 2 skipped the judge on all three rounds and then bought two of the verdicts+ back one loop iteration later (``judge.retry``) — no money saved — while the LAST+ round, the best one (0.7), was left without a score and could not be promoted.+ Now each round is judged once, in the round, and r02 wins."""+ def _writer_lint(job, ws):+ return {"src/object.js": f"// {job.label} r{job.round}\nLINT_ERROR\nexport function build(THREE) {{}}\n"}++ spec = make_spec(max_rounds=2)+ ws = Workspace(tmp_path / "runs" / "lintstuck")+ judge = FakeJudge(scores=(0.5, 0.6, 0.7))+ track = StaticObjectTrack(services=FakeServices(), judge=judge, agent=FakeAgent(_writer_lint),+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,+ runtime=FakeRuntime(Language.THREEJS),+ policy=RoundPolicy(max_rounds=2, target=0.9))+ rec = track.run(spec, ws)+ events = EventLog(ws.events_path).read()+ assert [e for e in events if e["event"] == "judge.skipped"] == []+ assert [e for e in events if e["event"] == "judge.retry"] == []+ assert [r.score for r in rec.rounds] == [0.5, 0.6, 0.7]+ assert rec.best_round == 2 and rec.final_score == pytest.approx(0.7)+ # one verdict per round, bought once+ assert len([e for e in events if e["event"] == "judge.done"]) == len(rec.rounds) == len(judge.calls)+++def test_a_round_that_raises_still_reports_what_it_burned(tmp_path, chair_plan, settings):+ spec = make_spec(max_rounds=2, max_usd=0.05)+ ws = Workspace(tmp_path / "runs" / "cut")++ class _Expensive(FakeAgent):+ def run(self, job):+ res = super().run(job)+ return res.model_copy(update={"usage": Usage(backend="fake", cost_usd=0.04, input_tokens=1000)})++ track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.5, 0.6)), agent=_Expensive(_writer),+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,+ runtime=FakeRuntime(Language.THREEJS), policy=RoundPolicy(max_rounds=2, target=0.9))+ rec = track.run(spec, ws)+ assert rec.status is RunStatus.BUDGET+ aborted = json.loads((ws.root / "rounds" / "aborted_r01.json").read_text())+ assert aborted["index"] == 1 and aborted["usage"]["cost_usd"] > 0 and "BudgetExceeded" in aborted["notes"]+ assert rec.extra["aborted_rounds"][0]["index"] == 1+ # the aborted round is NOT resumable state, and its money is in the total+ assert [r.index for r in rec.rounds] == [0]+ assert rec.total_usage.cost_usd >= sum(r.usage.cost_usd for r in rec.rounds)+ cut = next(e for e in EventLog(ws.events_path).read() if e["event"] == "cost.round" and e["round"] == 1)+ assert cut["wasted"] is True and cut["waste_reason"] == "aborted" and cut["total_usd"] > 0+++def test_fan_out_workers_inherit_the_callers_context():+ """Ledger attribution rides on contextvars: a parallel judge sample, a+ best-of-N candidate or a bench cell must not fall back to the process+ default just because it ran on a pool thread."""+ import contextvars++ from codeverse.fanout import fan_out++ var: contextvars.ContextVar[str] = contextvars.ContextVar("attr", default="process-default")+ var.set("run-42")+ seen = fan_out(range(6), lambda _i: var.get(), max_workers=4, label="attr")+ assert set(seen) == {"run-42"}+ # the caller's own context is untouched by the workers+ assert var.get() == "run-42"diff --git a/harness/tests/orchestrator_tracks/test_tracks.py b/harness/tests/orchestrator_tracks/test_tracks.pyindex 7f37f3e6..3b8c6db3 100644--- a/harness/tests/orchestrator_tracks/test_tracks.py+++ b/harness/tests/orchestrator_tracks/test_tracks.py@@ -91,7 +91,7 @@ def test_static_track_single_shot_with_repair_and_budget_stop(tmp_path, chair_pldef responder(req):n["gen"] += 1- if req.label == "baseline": # first build fails, the repair fixes it+ if req.label.startswith("baseline"): # first build fails, the repair fixes itreturn f"=== FILE: src/object.js ===\n// {FAIL_MARK}\nexport function build(){{}}\n=== END FILE ==="return "=== FILE: src/object.js ===\nexport function build(THREE) { return new THREE.Group(); }\n=== END FILE ==="@@ -99,7 +99,7 @@ def test_static_track_single_shot_with_repair_and_budget_stop(tmp_path, chair_pltrack = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.5, 0.55, 0.6, 0.62)), model=model,planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings, runtime=FakeRuntime(Language.THREEJS))rec = track.run(spec, ws)- assert rec.status is RunStatus.BUDGET and rec.rounds[0].build.ok and "repair attempts: 1 (fixed)" in rec.rounds[0].notes+ assert rec.status is RunStatus.BUDGET and rec.rounds[0].build.ok and "repair attempts: 1/2 (fixed)" in rec.rounds[0].notesassert rec.rounds[0].score == pytest.approx(0.5)assert any(r.label.startswith("r00_baseline_repair") for r in model.requests) or any("Repair" in r.messages[0].text for r in model.requests)assert rec.total_usage.cost_usd > 0.03 and ws.record_path.is_file()diff --git a/harness/tests/texturing/test_tools_cli.py b/harness/tests/texturing/test_tools_cli.pyindex 5df0cfd8..3c1c74d6 100644--- a/harness/tests/texturing/test_tools_cli.py+++ b/harness/tests/texturing/test_tools_cli.py@@ -14,9 +14,12 @@ from codeverse.workspace import Workspacefrom tests.texturing.conftest import FakeJudge, fake_render-def _ws(tmp_path: Path, chair_glb, chair_spec, chair_plan) -> Workspace:+def _ws(tmp_path: Path, chair_glb, chair_spec, chair_plan, *, texture: bool = True) -> Workspace:ws = Workspace(tmp_path / "run").create()- ws.write_json(ws.spec_path, chair_spec)+ # texturing.run.texture_requested is the ONE owner of "does this run texture?" —+ # the tool refuses in a run whose spec says no, so the spec has to say yes here+ spec = chair_spec.model_copy(update={"options": chair_spec.options.model_copy(update={"texture": texture})})+ ws.write_json(ws.spec_path, spec)ws.write_json(ws.plan_path, chair_plan)shutil.copy(chair_glb, ws.artifacts / "object.glb")return ws@@ -54,6 +57,21 @@ def test_texture_pass_tool_without_glb_is_usage_error(tmp_path, chair_spec, chaiassert not obs.ok and "build" in obs.text+def test_texture_pass_tool_refuses_when_the_run_did_not_ask_for_texturing(+ tmp_path, chair_glb, chair_spec, chair_plan):+ """`texture: false` must actually prevent the pass. The tool is registered for every+ object track, so an agent used to be able to buy a texture pass inside any run+ (docs/COST.md §15: the quality run paid for two)."""+ ws = _ws(tmp_path, chair_glb, chair_spec, chair_plan, texture=False)+ services = TextureServices(image_model=FakeImageModel(), judge_obj=FakeJudge([(0.7, {})]),+ render=fake_render, cache_dir=tmp_path / "c")+ obs = get_tool("texture_pass").call(+ ToolContext(workspace=ws, track="static_object", language="blender",+ extra={"texture_services": services}), {})+ assert not obs.ok and "did not ask for texturing" in obs.text+ assert not (ws.artifacts / "object_textured.glb").exists()++def test_cli_texture_show_and_help(tmp_path, chair_glb, chair_spec, chair_plan):from codeverse.cli.main import appfrom codeverse.texturing.run import texture_pass