Skip to content

Official Codebase of 3DCodeVerse

Python

cost controls, corrected by measurement: revert the agent turn cap (A/B: +$0.02, -0.21 score) and the prompt reordering (net +tokens below the cache floor); keep the replay-verified regression/marginal stops (2 runs cut, $1.14, 0 best rounds lost); repair the judge-skip branches; fan_out propagates contextvars so parallel ledger rows attribute correctly; finalise texture pass is idempotent; honest profiles; 1245 tests, ruff clean

gaoypeng515f36f2d1e8
diff --git a/harness/CLAUDE.md b/harness/CLAUDE.md
index a04721a5..2199fab4 100644
--- a/harness/CLAUDE.md
+++ b/harness/CLAUDE.md
@@ -48,6 +48,8 @@ calibration) before changing anything.
range). Judge default `gemini-3.1-pro-preview` (`Settings.default_judge`; tracks use
n_samples=1); flash + `--n 3` is the cheap fallback, pro for calibration/eval.
- Live runs under `runs/` (`e2e_*`): read-only reference material; never modify.
+- Every model call is priced into the run's `telemetry/cost.jsonl` (`codeverse.cost.instrument`;
+ `CV3D_COST_LEDGER=off` to disable). `docs/COST.md` Part II has the measured cost controls.
## Commands
```
@@ -55,6 +57,7 @@ pip install -e /home/yipeng/3dcodeverse/harness # once (entry points: 3dcodeve
cd /home/yipeng/3dcodeverse/harness
3dcodeverse doctor [--live] [--no-gpu] [--json]
3dcodeverse make "a mid-century wooden dining chair" --track static_object --language blender
+3dcodeverse make "..." --profile economy|balanced|quality # one dial: models, judge n, rounds, candidates, texture, ceilings (--profile == CV3D_PROFILE)
3dcodeverse make "..." --track static_object --language threejs --generator gemini-cli:gemini-3.7-flash
3dcodeverse make "..." --track articulated_object --language urdf_blender
3dcodeverse make "..." --track scene --language scene_threejs --rounds 2 --max-usd 3
@@ -63,6 +66,8 @@ cd /home/yipeng/3dcodeverse/harness
3dcv resume <slug> · 3dcv status <slug> · 3dcv render <slug> [--mode wire] · 3dcv judge <slug> [--model ... --n 3]
3dcv texture pass <slug> [--no-judge] · 3dcv texture scene-pack <slug> · 3dcv texture show <slug>
3dcv tools list · 3dcv tools measure --workspace runs/<slug> · 3dcv mcp --workspace runs/<slug>
+3dcv cost <slug> · 3dcv cost --runs-dir bench/out/<battery> · 3dcv cost cache <slug>
+3dcv cost prices [--stale] · 3dcv cost profiles · 3dcv cost estimate gemini:gemini-3.1-pro-preview --in 12000
3dcv flywheel export runs/ dataset/ [--pack --drop-duplicates --captions-dir d/] · 3dcv flywheel pairs runs/ pairs.jsonl
3dcv flywheel caption <slug> [--out dir] · 3dcv flywheel gallery runs/ gallery.html
3dcv bench run bench/prompts/static_objects_v1.yaml --generator ... --judge gemini:gemini-3.1-pro-preview
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index c481a6d8..be51bb0c 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -76,6 +76,7 @@ from codeverse.config import get_settings # noqa: E402
from codeverse.contracts.artifacts import RenderSet # noqa: E402
from codeverse.contracts.run import RunRecord # noqa: E402
from codeverse.contracts.spec import Spec # noqa: E402
+from codeverse.cost import run_ledger # noqa: E402
from codeverse.tracks.generation import MultiFileParseError # noqa: E402
from codeverse.workspace import Workspace # noqa: E402
@@ -208,7 +209,11 @@ def _run_harness(spec: Spec, cell: Path, eval_ws: Workspace, deps: CompareDeps,
if not resume:
run_ws.create()
run_ws.write_json(run_ws.spec_path, spec)
- rec = deps.run_track(spec, run_ws, resume)
+ # the harness arm is a real run: give it its own ledger, nested inside the cell's
+ # (run_ledger restores the outer one on the way out, so the fixed evaluation that
+ # follows keeps landing in the cell ledger)
+ with run_ledger(run_ws.root, run=f"{spec.id}:{run_ws.root.parent.name}"):
+ rec = deps.run_track(spec, run_ws, resume)
res.gen_cost_usd = rec.total_usage.cost_usd
res.tool_calls = rec.total_usage.tool_calls
res.harness_status, res.harness_rounds, res.harness_loop_score = rec.status.value, len(rec.rounds), rec.final_score
@@ -229,16 +234,20 @@ def run_cell(battery: Battery, item: BenchPrompt, arm: Arm, out: Path, opts: Com
eval_ws = _fresh_ws(cell / "eval")
eval_ws.write_json(eval_ws.spec_path, spec)
try:
- if arm.kind == "harness":
- _run_harness(spec, cell, eval_ws, deps, res)
- else:
- _generate_oneshot(arm, spec, cell, eval_ws, opts, deps, res)
- if (eval_ws.root / MODEL_FILE).is_file():
- outcome = deps.evaluator.evaluate(eval_ws, spec)
- eval_ws.write_json(eval_ws.root / "eval.json", outcome)
- _fill_from_outcome(res, outcome)
- else:
- res.status, res.score, res.passed, res.build_ok = "no_code", 0.0, False, False
+ # one ledger per cell: generation (one-shot arms) AND the fixed evaluation's
+ # judge calls are priced into cells/<prompt>/<arm>/telemetry/cost.jsonl instead
+ # of the per-process fallback log. Context-local, so --parallel keeps cells apart.
+ with run_ledger(cell, run=f"{item.id}:{arm.slug}"):
+ if arm.kind == "harness":
+ _run_harness(spec, cell, eval_ws, deps, res)
+ else:
+ _generate_oneshot(arm, spec, cell, eval_ws, opts, deps, res)
+ if (eval_ws.root / MODEL_FILE).is_file():
+ outcome = deps.evaluator.evaluate(eval_ws, spec)
+ eval_ws.write_json(eval_ws.root / "eval.json", outcome)
+ _fill_from_outcome(res, outcome)
+ else:
+ res.status, res.score, res.passed, res.build_ok = "no_code", 0.0, False, False
except Exception as e: # noqa: BLE001 — one cell must never kill the matrix
res.status = "error"
res.error = (res.error + "; " if res.error else "") + f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1200:]}"
@@ -343,7 +352,9 @@ def run_matrix(battery_path: Path | str, out_dir: Path | str, arms: Sequence[Arm
if on_result:
on_result(r)
if opts.pairwise:
- run_pairwise(battery, done, arms, out, opts, deps)
+ # the arena is battery-level spend: its own ledger at <out>/telemetry/cost.jsonl
+ with run_ledger(out, run=f"{battery.name}:pairwise"):
+ run_pairwise(battery, done, arms, out, opts, deps)
build_compare_report(out)
return list(done.values())
diff --git a/harness/bench/run_bench.py b/harness/bench/run_bench.py
index bb37c950..b5a35703 100644
--- a/harness/bench/run_bench.py
+++ b/harness/bench/run_bench.py
@@ -9,6 +9,12 @@ Layout of ``out_dir``::
Every prompt becomes a ``Spec`` with the battery's track/language, a FIXED judge
model (methodology: paired runs share the judge), and the generator under test.
+
+Each prompt runs inside its own ``codeverse.cost.run_ledger``, exactly like a
+``3dcv make``: the batteries are where most runs come from, so without it the
+priced per-call rows of a whole battery went to the per-process fallback log and
+``3dcv cost --runs-dir <out>/runs`` had to reconstruct them from trajectories.
+The binding is context-local, so ``--parallel N`` keeps N ledgers apart.
"""
from __future__ import annotations
@@ -30,6 +36,7 @@ from codeverse.config import get_settings
from codeverse.contracts.common import Backends, Budget, Language, Track
from codeverse.contracts.run import RunRecord
from codeverse.contracts.spec import Constraints, Spec
+from codeverse.cost import run_ledger
from codeverse.workspace import Workspace
RESULT_FIELDS = ("id", "tier", "category", "score_baseline", "score_final", "passed", "rounds", "cost_usd",
@@ -193,7 +200,8 @@ def run_battery(
ws.write_json(ws.spec_path, spec)
t0 = time.time()
try:
- rec = run_fn(spec, ws, resume)
+ with run_ledger(ws.root, run=item.id):
+ rec = run_fn(spec, ws, resume)
except Exception as e: # one failing prompt must not kill the battery
return BenchItemResult(id=item.id, tier=item.tier, category=item.category, status="error",
errors=f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}",
diff --git a/harness/codeverse/cli/_common.py b/harness/codeverse/cli/_common.py
index 8ad59d64..521ec8ca 100644
--- a/harness/codeverse/cli/_common.py
+++ b/harness/codeverse/cli/_common.py
@@ -7,6 +7,7 @@ import hashlib
import importlib
import shutil
import sys
+from dataclasses import dataclass
from pathlib import Path
from typing import Any
@@ -113,3 +114,118 @@ def echo_json(obj: Any) -> None:
import json
console.print_json(json.dumps(obj, default=str, ensure_ascii=False))
+
+
+# --------------------------------------------------------------------------- cost profile
+def active_profile(settings: Any | None = None) -> Any:
+ """The :class:`~codeverse.cost.profiles.Profile` the current settings name."""
+ from codeverse.cost.profiles import get_profile
+
+ return get_profile((settings or get_settings()).profile)
+
+
+@dataclass(frozen=True)
+class ResolvedDial:
+ """Every profile-controlled value, after the profile and the user's own
+ settings/flags have been folded together. This is what a run actually gets."""
+
+ profile: str
+ generator: str
+ planner: str
+ judge: str
+ captioner: str
+ judge_samples: int
+ judge_max_px: int
+ judge_montages: int
+ judge_detail_crops: int
+ agent_max_turns: int
+ rounds: int
+ candidates: int
+ texture: bool
+ max_usd: float
+ max_minutes: float
+
+
+def resolve_dial(
+ settings: Any | None = None,
+ profile_flag: str | None = None,
+ *,
+ rounds: int | None = None,
+ candidates: int | None = None,
+ max_usd: float | None = None,
+ max_minutes: float | None = None,
+ texture: bool = False,
+) -> ResolvedDial:
+ """The single resolver for the cost dial — used by ``3dcv make`` and by the tests.
+
+ ``--profile X`` (``profile_flag``) *forces* the dial over anything the user
+ stated in ``config.yaml`` / ``CV3D_*``; ``CV3D_PROFILE=X`` set the same dial
+ as a *default* when :func:`codeverse.config.get_settings` built the settings.
+ With nothing else stated the two paths therefore resolve **identically** —
+ that equality is what ``tests/cost/test_profiles.py`` asserts, and it
+ is the bug this function exists to prevent: the run shape (candidates, the
+ texture pass) used to be read off the flag and so applied to only one path.
+
+ An explicit CLI flag (``--rounds`` / ``--candidates`` / ``--max-usd`` /
+ ``--max-minutes`` / ``--texture``) always wins over both.
+ """
+ settings = settings or get_settings()
+ prof = settings.apply_profile(profile_flag, force=True) if profile_flag else active_profile(settings)
+ return ResolvedDial(
+ profile=prof.name,
+ generator=settings.default_generator,
+ planner=settings.default_planner,
+ judge=settings.default_judge,
+ captioner=settings.default_captioner,
+ judge_samples=int(settings.judge.samples),
+ judge_max_px=int(settings.judge.max_px),
+ judge_montages=int(settings.judge.montages),
+ judge_detail_crops=int(settings.judge.detail_crops),
+ agent_max_turns=int(settings.limits.agent_max_turns),
+ rounds=prof.rounds if rounds is None else int(rounds),
+ candidates=int(settings.default_candidates if candidates is None else candidates),
+ texture=bool(texture or prof.texture),
+ max_usd=prof.max_usd if max_usd is None else float(max_usd),
+ max_minutes=prof.max_minutes if max_minutes is None else float(max_minutes),
+ )
+
+
+def round_policy_options(spec: Spec, settings: Any | None = None) -> dict[str, Any]:
+ """``get_track(...)`` options the active profile implies.
+
+ Only the judge sample count needs a ``RoundPolicy`` (rounds travel on
+ ``spec.budget``, best-of-N on ``spec.options``), so a run at the default
+ ``n=1`` gets **no** policy and keeps the track's own — including the
+ rubric-derived stop target that ``BaseTrack.after_plan`` binds when no policy
+ was injected. When a profile does ask for more samples we bind that target
+ here instead, from the same rubric the track will use."""
+ settings = settings or get_settings()
+ samples = int(getattr(settings.judge, "samples", 1) or 1)
+ if samples <= 1:
+ return {}
+ from dataclasses import replace
+
+ from codeverse.orchestrator.rounds import RoundPolicy
+
+ policy = RoundPolicy(max_rounds=spec.budget.max_rounds, judge_samples=samples)
+ threshold = rubric_threshold(spec)
+ if threshold is not None:
+ policy = replace(policy, target=float(threshold))
+ return {"policy": policy}
+
+
+def rubric_threshold(spec: Spec) -> float | None:
+ """Pass threshold of the rubric this spec will be judged with (``None`` when
+ the rubric cannot be loaded — the caller then keeps the policy default)."""
+ try:
+ from codeverse.contracts.common import Track
+ from codeverse.judges.rubrics import load_rubric
+ from codeverse.tracks import get_track
+ from codeverse.tracks.lifecycle import REFERENCE_RUBRIC
+
+ # same selection rule as BaseTrack.build_context
+ name = (REFERENCE_RUBRIC if spec.references and spec.track is Track.STATIC_OBJECT
+ else get_track(spec.track).rubric)
+ return float(load_rubric(name).pass_threshold)
+ except Exception: # noqa: BLE001 - a missing rubric must not stop a run
+ return None
diff --git a/harness/codeverse/cli/cost_cmd.py b/harness/codeverse/cli/cost_cmd.py
index aa2d5341..d68548e1 100644
--- a/harness/codeverse/cli/cost_cmd.py
+++ b/harness/codeverse/cli/cost_cmd.py
@@ -1,22 +1,65 @@
"""``3dcv cost`` — what the runs cost and where the money went.
- 3dcv cost runs/ # audit a tree of runs
- 3dcv cost runs/e2e_chair_blender # one run, stage by stage
- 3dcv cost bench/out/static_v1_flash --md report.md
+ 3dcv cost <slug> # one run, from its own ledger
+ 3dcv cost --runs-dir bench/out/x # a whole battery, aggregated
+ 3dcv cost runs/ bench/out/* # any mix of run dirs / trees
+ 3dcv cost show runs/ --md report.md # the same, explicit + full markdown
3dcv cost runs/ --recheck # re-price with today's table (drift vs what was billed)
- 3dcv cost prices # the price table + provenance
+ 3dcv cost prices [--stale] [--unverified]
+ 3dcv cost profiles # the economy / balanced / quality dial
+ 3dcv cost estimate gemini:… --in 12000 --out 800
+
+A run that wrote a live ledger (``telemetry/cost.jsonl``) is read from it — one
+priced row per real model call. Older runs are reconstructed from their
+trajectories, verdicts and events, so all 61 reference runs still audit.
"""
from __future__ import annotations
+from datetime import date, datetime
from pathlib import Path
from typing import Annotated
+import click
import typer
+from typer.core import TyperGroup
from codeverse.cli._fmt import console, ok, warn
-cost_app = typer.Typer(no_args_is_help=True)
+#: a price row older than this needs re-checking against the provider's page
+STALE_AFTER_DAYS = 90
+
+class _CostGroup(TyperGroup):
+ """``3dcv cost <slug|path> ...`` — a first argument that is not a subcommand
+ is forwarded to ``show`` instead of failing with "No such command"."""
+
+ def resolve_command(self, ctx: click.Context, args: list[str]): # type: ignore[override]
+ try:
+ return super().resolve_command(ctx, args)
+ except click.UsageError:
+ return "show", self.get_command(ctx, "show"), args
+
+
+cost_app = typer.Typer(no_args_is_help=True, invoke_without_command=True, cls=_CostGroup)
+
+
+@cost_app.callback()
+def cost(
+ ctx: typer.Context,
+ runs_dir: Annotated[Path | None, typer.Option("--runs-dir", help="audit every run under this root")] = None,
+ recheck: Annotated[bool, typer.Option("--recheck", help="re-price every call with today's table")] = False,
+ md: Annotated[Path | None, typer.Option("--md", help="also write the full markdown report here")] = None,
+) -> None:
+ """Cost breakdown: per stage, per role, per model, waste, $ per passing artifact.
+
+ ``3dcv cost <slug>`` reports one run from its own ledger; ``--runs-dir <dir>``
+ aggregates every run under a root (a bench battery)."""
+ if ctx.invoked_subcommand is not None:
+ return
+ if runs_dir is None:
+ console.print(ctx.get_help())
+ raise typer.Exit()
+ _report([Path(runs_dir)], md=md, recheck=recheck, limit=20, per_run=True)
@cost_app.command("show")
@@ -26,8 +69,23 @@ def show(
recheck: Annotated[bool, typer.Option("--recheck", help="re-price every call with today's table")] = False,
limit: Annotated[int, typer.Option("--limit", help="rows in the per-run table")] = 20,
per_run: Annotated[bool, typer.Option("--per-run/--no-per-run")] = True,
+ runs_dir: Annotated[Path | None, typer.Option("--runs-dir", help="runs root a bare slug is resolved against")] = None,
) -> None:
- """Cost breakdown: per stage, per role, per model, waste, $ per passing artifact."""
+ """Cost breakdown for runs named by slug or path (``3dcv cost <slug>`` lands here)."""
+ _report([_resolve(str(p), runs_dir) for p in paths], md=md, recheck=recheck, limit=limit, per_run=per_run)
+
+
+def _resolve(arg: str, runs_dir: Path | None) -> Path:
+ """A bare argument is a path when it exists, else a slug under the runs root."""
+ from codeverse.cli import _common as C
+
+ p = Path(arg)
+ if p.exists():
+ return p
+ return C.runs_root(runs_dir) / arg
+
+
+def _report(paths: list[Path], *, md: Path | None, recheck: bool, limit: int, per_run: bool) -> None:
from codeverse.cost import audit_runs
from codeverse.cost.report import console as text_report
from codeverse.cost.report import markdown, runs_table
@@ -36,9 +94,13 @@ def show(
if not audit.runs:
raise typer.BadParameter(f"no runs with a record.json under {', '.join(str(p) for p in paths)}")
console.print(text_report(audit), soft_wrap=True)
- if per_run:
+ if len(audit.runs) == 1:
+ _single_run_lines(audit.runs[0])
+ elif per_run:
console.print("")
console.print(runs_table(audit, limit=limit), soft_wrap=True)
+ live = sum(1 for r in audit.runs if r.source == "live")
+ console.print(f"[dim]ledger source: {live} live / {len(audit.runs) - live} reconstructed[/dim]")
if recheck:
drift = audit.total_usd - audit.recorded_usd
warn(f"re-priced total ${audit.total_usd:.4f} vs recorded ${audit.recorded_usd:.4f} ({drift:+.4f})")
@@ -51,33 +113,150 @@ def show(
ok(f"wrote {md}")
+def _single_run_lines(run: object) -> None:
+ """Reconciliation for one run: the ledger against what the record was billed."""
+ from codeverse.cli._fmt import kv_table
+
+ ledger = float(getattr(run, "ledger_usd", 0.0))
+ recorded = float(getattr(run, "recorded_usd", 0.0))
+ drift = ledger - recorded
+ pct = (100.0 * drift / recorded) if recorded else 0.0
+ rows = {
+ "source": getattr(run, "source", "?"),
+ "ledger": f"${ledger:.4f} over {len(getattr(run, 'rows', []))} calls",
+ "record.total_usage": f"${recorded:.4f}",
+ "difference": f"${drift:+.4f} ({pct:+.2f}%)",
+ "status": f"{getattr(run, 'status', '')} ({getattr(run, 'stop_reason', '')})",
+ "budget": f"${float(getattr(run, 'budget_usd', 0.0)):.2f}",
+ }
+ console.print("")
+ console.print(kv_table("reconciliation", rows))
+ for note in getattr(run, "notes", []):
+ warn(note)
+
+
+# --------------------------------------------------------------------------- prices
@cost_app.command("prices")
def prices(
provider: Annotated[str | None, typer.Option("--provider", help="gemini | anthropic | openai")] = None,
unverified: Annotated[bool, typer.Option("--unverified", help="only rows that are not verified")] = False,
+ stale: Annotated[bool, typer.Option("--stale", help=f"only rows older than {STALE_AFTER_DAYS} days or approximate")] = False,
+ days: Annotated[int, typer.Option("--days", help="staleness threshold in days")] = STALE_AFTER_DAYS,
) -> None:
- """The USD price table with its provenance (source + checked date + status)."""
+ """The USD price table with its provenance (source + checked date + status).
+
+ A row is flagged when it was last checked more than ``--days`` ago, when the
+ price is marked approximate, or when its status is not ``verified`` — those
+ are the dollars an audit cannot stand behind."""
from codeverse.models.pricing import PRICES, price_provenance
- rows = []
+ rows, flagged = [], 0
for (prov, model) in sorted(PRICES):
if provider and prov != provider:
continue
row = price_provenance(prov, model)
if unverified and row.status == "verified":
continue
+ age = _age_days(row.checked)
+ problems = _price_flags(row, age, days)
+ if stale and not problems:
+ continue
+ flagged += bool(problems)
p = row.price
assert p is not None
rows.append((f"{prov}:{model}", f"{p.input:g}", f"{p.cached:g}", f"{p.output:g}",
f"{p.image_usd:g}" if p.image_usd else "-", row.status, row.checked,
- (row.provenance.note if row.provenance else "")[:60]))
- header = ("model", "in/M", "cached/M", "out/M", "$/img", "status", "checked", "note")
- widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))]
+ "-" if age is None else str(age), ",".join(problems) or "-",
+ (row.provenance.note if row.provenance else "")[:52]))
+ header = ("model", "in/M", "cached/M", "out/M", "$/img", "status", "checked", "age_d", "flags", "note")
+ widths = [max(len(str(r[i])) for r in [header, *rows]) for i in range(len(header))] if rows else \
+ [len(h) for h in header]
console.print(" ".join(h.ljust(w) for h, w in zip(header, widths, strict=True)), soft_wrap=True)
for r in rows:
console.print(" ".join(str(c).ljust(w) for c, w in zip(r, widths, strict=True)), soft_wrap=True)
+ if flagged:
+ warn(f"{flagged} of {len(rows)} row(s) flagged: re-check them against the provider's pricing page")
+ elif rows:
+ ok(f"{len(rows)} price row(s), none older than {days} days or approximate")
+
+
+def _price_flags(row: object, age: int | None, days: int) -> list[str]:
+ flags = []
+ if age is not None and age > days:
+ flags.append(f"stale>{days}d")
+ price = getattr(row, "price", None)
+ if price is not None and getattr(price, "approximate", False):
+ flags.append("approximate")
+ status = getattr(row, "status", "")
+ if status not in ("verified", ""):
+ flags.append(status)
+ return flags
+
+
+def _age_days(checked: str) -> int | None:
+ try:
+ return (date.today() - datetime.strptime(checked, "%Y-%m-%d").date()).days
+ except (TypeError, ValueError):
+ return None
+
+
+# --------------------------------------------------------------------------- cache
+@cost_app.command("cache")
+def cache(
+ paths: Annotated[list[Path], typer.Argument(help="run slugs or run dirs")],
+ runs_dir: Annotated[Path | None, typer.Option("--runs-dir")] = None,
+) -> None:
+ """Did prompt caching actually happen? Per session: the cold first call, the
+ cached share, the dollars the cache saved and the dollars the cold head cost.
+
+ Only a live ledger has per-call ``cached_tokens``; reconstructed runs show
+ what their transcripts recorded."""
+ from codeverse.cost.caching import session_cache
+ from codeverse.cost.ledger import load_ledger
+ from codeverse.cost.reconstruct import reconstruct_run
+
+ for arg in paths:
+ root = _resolve(str(arg), runs_dir)
+ rows = load_ledger(root) or reconstruct_run(root).rows
+ sessions = session_cache(rows)
+ if not sessions:
+ warn(f"{root}: no priced calls found")
+ continue
+ console.print(f"[bold]{root.name}[/bold] ({len(rows)} calls, {len(sessions)} sessions)")
+ header = ("session", "calls", "first in", "first cached", "cached", "saved $", "cold $")
+ table = [header] + [
+ (s.key, str(s.n_calls), f"{s.first_input:,}", f"{s.first_cached:,}",
+ f"{s.cached_fraction:.0%}", f"{s.saved_usd:.4f}", f"{s.cold_usd:.4f}") for s in sessions]
+ widths = [max(len(r[i]) for r in table) for i in range(len(header))]
+ for r in table:
+ console.print(" " + " ".join(c.rjust(w) if i else c.ljust(w)
+ for i, (c, w) in enumerate(zip(r, widths, strict=True))),
+ soft_wrap=True)
+ tot_in = sum(s.input_tokens for s in sessions)
+ tot_cached = sum(s.cached_tokens for s in sessions)
+ console.print(f" → {tot_cached:,} of {tot_in:,} input tokens cached ({tot_cached / tot_in if tot_in else 0:.0%}), "
+ f"saved ${sum(s.saved_usd for s in sessions):.4f}, "
+ f"cold heads cost ${sum(s.cold_usd for s in sessions):.4f}\n")
+
+
+# --------------------------------------------------------------------------- profiles
+@cost_app.command("profiles")
+def profiles() -> None:
+ """The economy / balanced / quality dial and what each is measured to cost."""
+ from codeverse.config import get_settings
+ from codeverse.cost.profiles import profile_table
+
+ header = ("profile", "generator", "judge", "shape", "payload", "$/run", "measured quality")
+ rows = [tuple(str(c) for c in r) for r in profile_table()]
+ widths = [max(len(r[i]) for r in [header, *rows]) for i in range(6)]
+ for r in [header, *rows]:
+ console.print(" ".join(c.ljust(w) for c, w in zip(r[:6], widths, strict=True)) + " " + r[6],
+ soft_wrap=True)
+ console.print(f"\n[dim]active profile: {get_settings().profile} · "
+ f"`3dcv make ... --profile quality` to switch one run[/dim]")
+# --------------------------------------------------------------------------- estimate
@cost_app.command("estimate")
def estimate(
model: Annotated[str, typer.Argument(help="model id, e.g. gemini:gemini-3.1-pro-preview")],
diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.py
index aad46fa6..bdc6e81e 100644
--- a/harness/codeverse/cli/main.py
+++ b/harness/codeverse/cli/main.py
@@ -62,9 +62,12 @@ def make(
judge: Annotated[str | None, typer.Option()] = None,
captioner: Annotated[str | None, typer.Option()] = None,
image: Annotated[list[Path] | None, typer.Option("--image", help="reference image(s)")] = None,
- rounds: Annotated[int, typer.Option("--rounds", min=0)] = 4,
- max_usd: Annotated[float, typer.Option("--max-usd")] = 5.0,
- max_minutes: Annotated[float, typer.Option("--max-minutes")] = 60.0,
+ profile: Annotated[str | None, typer.Option("--profile", help="cost/quality dial: economy | balanced | quality "
+ "(sets models, judge samples, rounds, candidates, turn cap, "
+ "montage size and the texture pass together)")] = None,
+ rounds: Annotated[int | None, typer.Option("--rounds", min=0, help="refine rounds after the baseline (default: the profile's)")] = None,
+ max_usd: Annotated[float | None, typer.Option("--max-usd")] = None,
+ max_minutes: Annotated[float | None, typer.Option("--max-minutes")] = None,
candidates: Annotated[int | None, typer.Option("--candidates", min=1, help="best-of-N baseline: N parallel candidates, keep the best (default: settings.default_candidates or 1)")] = None,
slug: Annotated[str | None, typer.Option("--slug")] = None,
runs_dir: RunsDirOpt = None,
@@ -86,7 +89,14 @@ def make(
if language is None:
language = TRACK_LANGUAGES[track][0]
run_slug = C.make_slug(prompt, track.value, language.value, slug)
- backends = get_settings().backends(generator=generator, planner=planner, judge=judge, captioner=captioner)
+ settings = get_settings()
+ # ONE resolver for the whole dial, so `--profile X` and `CV3D_PROFILE=X` land the same
+ # values (they used to disagree on candidates + texture); an explicit flag beats both.
+ dial = C.resolve_dial(settings, profile, rounds=rounds, candidates=candidates,
+ max_usd=max_usd, max_minutes=max_minutes, texture=texture)
+ rounds, max_usd, max_minutes = dial.rounds, dial.max_usd, dial.max_minutes
+ candidates, texture = dial.candidates, dial.texture
+ backends = settings.backends(generator=generator, planner=planner, judge=judge, captioner=captioner)
# validate the whole Spec BEFORE touching the filesystem: an invalid
# track/language combination must not leave an orphan run directory behind.
try:
@@ -96,7 +106,10 @@ def make(
constraints=Constraints(dimensions_m=C.parse_kv_floats(dim, "--dim") or None, must_have=must,
must_not=must_not, style=style),
budget=Budget(max_rounds=rounds, max_usd=max_usd, max_minutes=max_minutes),
- backends=backends, options=RunOptions(candidates=candidates, texture=texture),
+ # options.profile records the dial this run resolved to, whichever way it was
+ # named (flag, CV3D_PROFILE, config.yaml), so `3dcv resume` re-applies it
+ backends=backends, options=RunOptions(candidates=candidates, texture=texture,
+ profile=dial.profile),
seed=seed, tags=tag,
)
except ValueError as e:
@@ -106,7 +119,11 @@ def make(
ws.commit("spec")
console.print(kv_table("run", {"slug": run_slug, "workspace": ws.root, "track": track.value,
"language": language.value, "generator": backends.generator,
- "candidates": candidates or "default"}))
+ "profile": f"{dial.profile} ({C.active_profile(settings).expectation()})",
+ "judge": f"{backends.judge} n={dial.judge_samples} "
+ f"({dial.judge_max_px}px/{dial.judge_detail_crops}crop)",
+ "rounds": rounds, "max_usd": max_usd,
+ "candidates": candidates, "texture": texture}))
if no_run:
ok(f"spec written: {ws.spec_path} (not run; `3dcv resume {run_slug}` to start)")
return
@@ -114,10 +131,15 @@ def make(
def _run_track(spec: Spec, ws, *, resume: bool, candidates: int | None = None) -> None:
+ from codeverse.cost.instrument import run_ledger
+
get_track = C.lazy("codeverse.tracks", "get_track")
- options = {"n_candidates": candidates} if candidates else {}
+ options: dict = {"n_candidates": candidates} if candidates else {}
+ options.update(C.round_policy_options(spec))
try:
- record = get_track(spec.track, **options).run(spec, ws, resume=resume)
+ # every model call and agent session of this run lands in telemetry/cost.jsonl
+ with run_ledger(ws.root, run=ws.root.name):
+ record = get_track(spec.track, **options).run(spec, ws, resume=resume)
except KeyboardInterrupt:
raise C.CliError(f"interrupted; resume with `3dcv resume {ws.root.name}`", code=130) from None
except Exception as e: # the track failed outside its own error handling
@@ -143,6 +165,8 @@ def resume(
first — the only way to continue a BUDGET-stopped run."""
ws = C.open_workspace(slug, runs_dir)
spec = C.load_spec(ws)
+ if spec.options.profile: # the dial the run was created with (judge samples, montage px, turn cap)
+ get_settings().apply_profile(spec.options.profile, force=True)
raised = {k: v for k, v in {"max_usd": max_usd, "max_minutes": max_minutes, "max_rounds": rounds}.items() if v is not None}
if raised:
from codeverse.events import EventLog
@@ -279,8 +303,13 @@ def judge(
inp = J.build_judge_input(ws, rec, rnd)
judge_obj = J.make_judge(rec, rubric_name, model or rec.spec.backends.judge, n)
n_images = J.count_prompt_images(inp, rubric_name)
+ from codeverse.cost.instrument import run_ledger
+
try:
- verdict = judge_obj.judge(inp)
+ # a re-judge joins the run's ledger when it has one; otherwise the per-process log
+ # (a ledger holding only this verdict would be read as the whole run's cost)
+ with run_ledger(ws.root, run=ws.root.name, create=False):
+ verdict = judge_obj.judge(inp)
except ValueError as e: # e.g. a measured rubric fed to a judge that computes nothing
raise C.CliError(f"judge failed: {e}") from e
except Exception as e:
diff --git a/harness/codeverse/cli/texture_cmd.py b/harness/codeverse/cli/texture_cmd.py
index 69267eb1..5b12044d 100644
--- a/harness/codeverse/cli/texture_cmd.py
+++ b/harness/codeverse/cli/texture_cmd.py
@@ -45,8 +45,13 @@ def pass_(
load_plan = C.lazy("codeverse.spatial.tool_common", "load_plan")
plan = load_plan(ws.plan_path)
texture_pass = C.lazy("codeverse.texturing.run", "texture_pass")
- rep = texture_pass(ws, spec, plan, model_id=model or spec.backends.planner, image_model=_image_model(image_model),
- judge=judge, judge_model_id=judge_model, size=size)
+ from codeverse.cost.instrument import run_ledger
+
+ # a post-hoc pass joins the run's ledger when it has one, else the per-process log
+ with run_ledger(ws.root, run=ws.root.name, create=False):
+ rep = texture_pass(ws, spec, plan, model_id=model or spec.backends.planner,
+ image_model=_image_model(image_model), judge=judge,
+ judge_model_id=judge_model, size=size)
_print_report(rep)
diff --git a/harness/codeverse/config.py b/harness/codeverse/config.py
index 3f23f969..71de10a8 100644
--- a/harness/codeverse/config.py
+++ b/harness/codeverse/config.py
@@ -15,6 +15,7 @@ import re
import shutil
from functools import lru_cache
from pathlib import Path
+from typing import Any
import yaml
from pydantic import BaseModel, Field
@@ -53,6 +54,21 @@ class Limits(BaseModel):
bpy_rlimit_gb: int = 12
max_parallel_agents: int = 6
max_parallel_builds: int = 3
+ agent_max_turns: int = Field(
+ default=0,
+ description="hard cap on model turns per agent session (0 = the backend's own default, "
+ "which is what ships: a 28-turn cap cost $0.02 more and 0.205 of a score point in its "
+ "own A/B — docs/COST.md §17 — so no profile sets one; name it here if you want one).",
+ )
+
+
+class Judge(BaseModel):
+ """Judge payload size — what one verdict is allowed to send (docs/COST.md §3)."""
+
+ max_px: int = Field(default=1024, description="longest edge of a montage / crop sent to the judge")
+ montages: int = Field(default=3, description="max 2x2 montages per verdict")
+ detail_crops: int = Field(default=2, description="max zoomed detail crops per verdict")
+ samples: int = Field(default=1, description="default VLM samples per verdict")
class Settings(BaseSettings):
@@ -63,6 +79,7 @@ class Settings(BaseSettings):
binaries: Binaries = Field(default_factory=Binaries)
render: Render = Field(default_factory=Render)
limits: Limits = Field(default_factory=Limits)
+ judge: Judge = Field(default_factory=Judge)
gemini_api_keys: list[str] = Field(default_factory=list)
anthropic_api_key: str = ""
@@ -77,6 +94,12 @@ class Settings(BaseSettings):
default_captioner: str = Field(default_factory=lambda: Backends().captioner)
default_candidates: int = Field(default=1, description="best-of-N baseline candidates (tracks read it)")
+ # Cost dial: one name that sets model-per-role, judge samples, rounds, candidates,
+ # turn cap, montage size and the texture pass together (codeverse/cost/profiles.py).
+ profile: str = Field(default="balanced", description="economy | balanced | quality")
+ cost_ledger: bool = Field(default=True, description="append one priced row per model call "
+ "to the run's telemetry/cost.jsonl (or a per-process log)")
+
# ------------------------------------------------------------------ helpers
def backends(self, **overrides: str | None) -> Backends:
"""The ``Backends`` for a new Spec from the settings defaults. Keyword
@@ -96,6 +119,37 @@ class Settings(BaseSettings):
base[role] = value
return Backends(**base)
+ def apply_profile(self, name: str | None = None, *, force: bool = False) -> Any:
+ """Set every profile-controlled default on this Settings object and return
+ the :class:`~codeverse.cost.profiles.Profile`.
+
+ A field the user stated themselves (config file / ``CV3D_*`` env) is left
+ alone unless ``force`` — so a profile is a *default* dial, while
+ ``3dcv make --profile X`` (which forces) is an instruction. The CLI's own
+ flags are applied after this and always win."""
+ from codeverse.cost.profiles import get_profile
+
+ p = get_profile(name or self.profile)
+ stated = set() if force else set(self.model_fields_set)
+
+ def put(field: str, value: Any) -> None:
+ if field not in stated and value not in (None, ""):
+ setattr(self, field, value)
+
+ self.profile = p.name
+ put("default_generator", p.generator)
+ put("default_planner", p.planner)
+ put("default_judge", p.judge)
+ put("default_captioner", p.captioner)
+ put("default_candidates", p.candidates)
+ if "judge" not in stated:
+ self.judge = self.judge.model_copy(update={
+ "max_px": p.judge_max_px, "montages": p.judge_montages,
+ "detail_crops": p.judge_detail_crops, "samples": p.judge_samples})
+ if "limits" not in stated:
+ self.limits = self.limits.model_copy(update={"agent_max_turns": p.max_turns})
+ return p
+
def resolve_blender(self) -> str:
if self.binaries.blender and Path(self.binaries.blender).exists():
return self.binaries.blender
@@ -152,4 +206,8 @@ def get_settings() -> Settings:
# de-dupe keys preserving order
seen: set[str] = set()
s.gemini_api_keys = [k for k in s.gemini_api_keys if not (k in seen or seen.add(k))]
+ # the dial named in config.yaml / CV3D_PROFILE, applied as a *default*: a value the
+ # user stated themselves survives it. `3dcv make --profile X` forces the same dial
+ # (codeverse.cli._common.resolve_dial is the one resolver both paths go through).
+ s.apply_profile(s.profile)
return s
diff --git a/harness/codeverse/contracts/spec.py b/harness/codeverse/contracts/spec.py
index f1bcddae..d5281a3d 100644
--- a/harness/codeverse/contracts/spec.py
+++ b/harness/codeverse/contracts/spec.py
@@ -37,6 +37,9 @@ class RunOptions(BaseModel):
candidates: int | None = Field(default=None, ge=1, description="best-of-N baseline candidates")
texture: bool = Field(default=False, description="run the derived texture pass after the loop")
+ profile: str = Field(default="", description="cost/quality dial this run was created with "
+ "(codeverse/cost/profiles.py); '' = the settings default, and a resume "
+ "re-applies it so judge samples / montage size / turn cap survive")
class Spec(BaseModel):
diff --git a/harness/codeverse/cost/__init__.py b/harness/codeverse/cost/__init__.py
index 536ce667..a6ef3d72 100644
--- a/harness/codeverse/cost/__init__.py
+++ b/harness/codeverse/cost/__init__.py
@@ -1,7 +1,8 @@
"""Cost accounting: a typed ledger of every model call, a price-provenance layer
-on top of ``models/pricing.py``, an audit over recorded runs, and the two cheap
-helpers that keep the bill down (cache-friendly prompt ordering, a pre-send cost
-guard).
+on top of ``models/pricing.py``, an audit over recorded runs, a pre-send cost
+guard, and the prompt-prefix measurement helpers (``Block`` / ``order_blocks`` /
+``prefix_report`` — measurement only: reordering the generation prompts for the
+cache was measured and reverted, see ``docs/COST.md`` §13).
Public API::
@@ -26,6 +27,7 @@ from codeverse.cost.caching import (
prefix_signature,
render_blocks,
)
+from codeverse.cost.context import CallContext, bind_run, call_context
from codeverse.cost.guard import (
CostEstimate,
CostGuard,
@@ -33,15 +35,25 @@ from codeverse.cost.guard import (
estimate_call,
text_tokens,
)
+from codeverse.cost.instrument import (
+ MeteredAgent,
+ MeteredChatModel,
+ metered_agent,
+ metered_chat_model,
+ per_call_metering,
+ run_ledger,
+)
from codeverse.cost.ledger import (
CostLedger,
ledger_path,
load_ledger,
+ open_run_ledger,
price_call,
record_call,
set_default_ledger,
summarise,
)
+from codeverse.cost.profiles import PROFILES, Profile, get_profile
from codeverse.cost.reconstruct import RunLedger, find_runs, reconstruct, reconstruct_run
from codeverse.cost.report import console, markdown
from codeverse.cost.routing import (
@@ -54,11 +66,56 @@ from codeverse.cost.routing import (
from codeverse.cost.types import CallCost, CostBucket, Role, Stage, Summary
__all__ = [
- "ROUTES", "Audit", "Block", "CallCost", "CostBucket", "CostEstimate", "CostGuard", "CostLedger",
- "PrefixReport", "Role", "Route", "RunLedger", "Stage", "Summary", "WasteItem", "audit_dir",
- "audit_runs", "cache_efficiency", "cheapest_affordable", "console", "default_route",
- "estimate_call", "find_runs", "ledger_path", "load_ledger", "markdown", "order_blocks",
- "prefix_report", "prefix_signature", "price_call", "pro_break_even", "reconstruct",
- "reconstruct_run", "record_call", "render_blocks", "samples_for_precision",
- "set_default_ledger", "summarise", "text_tokens",
+ "Audit",
+ "Block",
+ "CallContext",
+ "CallCost",
+ "CostBucket",
+ "CostEstimate",
+ "CostGuard",
+ "CostLedger",
+ "MeteredAgent",
+ "MeteredChatModel",
+ "PROFILES",
+ "PrefixReport",
+ "Profile",
+ "ROUTES",
+ "Role",
+ "Route",
+ "RunLedger",
+ "Stage",
+ "Summary",
+ "WasteItem",
+ "audit_dir",
+ "audit_runs",
+ "bind_run",
+ "cache_efficiency",
+ "call_context",
+ "cheapest_affordable",
+ "console",
+ "default_route",
+ "estimate_call",
+ "find_runs",
+ "get_profile",
+ "ledger_path",
+ "load_ledger",
+ "markdown",
+ "metered_agent",
+ "metered_chat_model",
+ "open_run_ledger",
+ "order_blocks",
+ "per_call_metering",
+ "prefix_report",
+ "prefix_signature",
+ "price_call",
+ "pro_break_even",
+ "reconstruct",
+ "reconstruct_run",
+ "record_call",
+ "render_blocks",
+ "run_ledger",
+ "samples_for_precision",
+ "set_default_ledger",
+ "summarise",
+ "text_tokens",
]
diff --git a/harness/codeverse/cost/caching.py b/harness/codeverse/cost/caching.py
index 67ec46d0..9c5b8117 100644
--- a/harness/codeverse/cost/caching.py
+++ b/harness/codeverse/cost/caching.py
@@ -17,6 +17,32 @@ Usage::
:func:`prefix_report` measures a real sequence of prompts: how many tokens they
share, and how many tokens each call pays full price for because of divergence.
+
+**Measured, and NOT applied to the generation prompts (2026-08-23).** Wave 2
+reordered ``generate_static`` / ``generate_articulated`` / ``refine_object``
+stable-prefix-first, by hoisting a shared head into every one of them. Measured
+on the real templates (``prefix_report`` over one baseline + three refines):
+
+=============================== ============================== ===============
+ per-prompt tokens mean/call
+=============================== ============================== ===============
+task-first (shipped, restored) 5,072 / 1,325 / 1,325 / 1,325 2,261
+stable-head-first (reverted) 5,110 / 5,211 / 5,211 / 5,211 5,186
+=============================== ============================== ===============
+
+The head was **duplicated** into every prompt rather than shared with anything
+already being sent, so the run-level change is +2,925 tokens on the average call
+(+3,886 on every refine) — and the prompt is re-sent on every turn of the
+session, so it is that figure times the turn count. What it bought was a 4,970
+token shared prefix, below the ~12k floor §4 of ``docs/COST.md`` measured for
+Gemini's implicit cache, so there is no cache read to set against it. The one
+prefix that is genuinely shared without duplication — the api-agent's system
+prompt (``AGENTS.md``, 4,879 tokens) plus its tool declarations (1,691) — already
+sits first in every request and totals **6,570 tokens**, also below the floor.
+
+So: keep these helpers for measuring and for any future prompt family that is
+big enough, but do not reorder a prompt for the cache without measuring the
+prefix against the floor first.
"""
from __future__ import annotations
@@ -123,3 +149,77 @@ def cache_efficiency(usages: Sequence[object]) -> tuple[int, int, float]:
cached = sum(int(getattr(u, "cached_tokens", 0)) for u in usages)
total = sum(int(getattr(u, "input_tokens", 0)) for u in usages)
return cached, total, (cached / total if total else 0.0)
+
+
+# --------------------------------------------------------------------------- ledger view
+@dataclass
+class SessionCache:
+ """Cache behaviour of ONE agent session (or one-shot call) as the ledger saw it."""
+
+ key: str
+ n_calls: int = 0
+ first_input: int = 0
+ first_cached: int = 0
+ input_tokens: int = 0
+ cached_tokens: int = 0
+ cost_usd: float = 0.0
+ price_input: float = 0.0
+ price_cached: float = 0.0
+
+ @property
+ def cached_fraction(self) -> float:
+ return self.cached_tokens / self.input_tokens if self.input_tokens else 0.0
+
+ @property
+ def first_cached_fraction(self) -> float:
+ return self.first_cached / self.first_input if self.first_input else 0.0
+
+ @property
+ def saved_usd(self) -> float:
+ """USD the cache reads saved against paying full input price for them."""
+ return self.cached_tokens * max(0.0, self.price_input - self.price_cached) / 1e6
+
+ @property
+ def cold_usd(self) -> float:
+ """USD still paid at full price for the uncached head of the session."""
+ return self.first_input * self.price_input / 1e6 if not self.first_cached else 0.0
+
+
+def session_key(label: str) -> str:
+ """Collapse a per-call label to its session: ``api-agent:refine_legs:t7`` →
+ ``refine_legs``, ``judge:static_object_v1:r02:s1`` → ``judge:static_object_v1``.
+
+ A session is what shares a prompt prefix: every turn of one agent session,
+ every sample of one verdict."""
+ parts = [p for p in (label or "").split(":") if p]
+ if parts and parts[0] == "api-agent":
+ parts = parts[1:]
+ while parts and len(parts[-1]) > 1 and parts[-1][0] in "tsr" and parts[-1][1:].isdigit():
+ parts = parts[:-1]
+ return ":".join(parts)
+
+
+def session_cache(rows: Sequence[object], *, key: str = "label") -> list[SessionCache]:
+ """Per-session cache behaviour from ledger rows (``codeverse.cost.CallCost``).
+
+ Sessions are ``(round, session_key(<key>))`` groups in ledger order, so the first row of a
+ group is that session's cold turn — the one the audit shows pays full price
+ (docs/COST.md §4). Use it to check whether a prompt-ordering change actually
+ moved ``cached_tokens``, not just the prompt."""
+ out: dict[tuple[object, str], SessionCache] = {}
+ order: list[tuple[object, str]] = []
+ for row in rows:
+ k = (getattr(row, "round", None), session_key(str(getattr(row, key, "") or "")))
+ s = out.get(k)
+ if s is None:
+ s = out[k] = SessionCache(key=f"r{k[0] if k[0] is not None else '-'}:{k[1]}",
+ first_input=int(getattr(row, "input_tokens", 0)),
+ first_cached=int(getattr(row, "cached_tokens", 0)),
+ price_input=float(getattr(row, "price_input", 0.0)),
+ price_cached=float(getattr(row, "price_cached", 0.0)))
+ order.append(k)
+ s.n_calls += 1
+ s.input_tokens += int(getattr(row, "input_tokens", 0))
+ s.cached_tokens += min(int(getattr(row, "cached_tokens", 0)), int(getattr(row, "input_tokens", 0)))
+ s.cost_usd += float(getattr(row, "cost_usd", 0.0))
+ return [out[k] for k in order]
diff --git a/harness/codeverse/cost/context.py b/harness/codeverse/cost/context.py
new file mode 100644
index 00000000..b56f3cdd
--- /dev/null
+++ b/harness/codeverse/cost/context.py
@@ -0,0 +1,230 @@
+"""Where a call sits in a run — the attribution the ledger needs.
+
+A :class:`CallContext` answers "which run, which round, which stage, which
+role" for the call that is about to happen. Precedence:
+
+1. what the caller passed explicitly (a judge knows it is judging) — always wins;
+2. a label that names a **job of its own** (``judge:…``, ``planner``,
+ ``texture_gate``, ``caption…``, ``pairwise:…`` — :data:`SELF_DESCRIBING`).
+ Such a call is not the work of whatever agent session happens to surround it,
+ so it outranks the ambient context. This is the fix for a measured
+ mis-attribution: an in-process tool that bills a model *inside* an agent
+ session (a texture pass, a captioner, a judge invoked as a tool) was filed
+ under the session's ``stage=refine / role=generator``;
+3. the ambient context (:func:`call_context`) — set by
+ :mod:`codeverse.cost.instrument` around an agent session. It beats a
+ *generation* label because the session knows more than one of its turns does:
+ a best-of-N candidate runs with ``job.kind="candidate"`` while its turns are
+ labelled ``api-agent:baseline:t3``;
+4. what is left of the label (a generation stage, the round tag) — which also
+ survives a thread hop that a ``ContextVar`` does not.
+
+A label that names nothing (:data:`~codeverse.cost.types.Stage.OTHER`) states
+nothing and never displaces anything.
+
+The **run binding** (:func:`bind_run`, the run slug) is a ``ContextVar`` with a
+process-wide fallback: a nested run (a bench worker that opens its own ledger in
+its own thread) gets its own binding, while a plain worker thread that inherits
+nothing — ``codeverse.fanout`` does not copy context — still sees the process's
+one run.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, replace
+
+from codeverse.cost.types import Role, Stage, role_for_stage, stage_for_label
+
+
+@dataclass(frozen=True)
+class CallContext:
+ """Attribution for one call (all fields optional; ``None`` = "not stated")."""
+
+ run: str = ""
+ round: int | None = None
+ stage: Stage | None = None
+ role: Role | None = None
+ label: str = ""
+
+ def merge(self, other: CallContext) -> CallContext:
+ """``other`` wins field by field where it says something."""
+ return CallContext(
+ run=other.run or self.run,
+ round=self.round if other.round is None else other.round,
+ stage=other.stage or self.stage,
+ role=other.role or self.role,
+ label=other.label or self.label,
+ )
+
+ def resolved(self) -> CallContext:
+ """Fill role from stage (and stage from label) so the row is never ``other``
+ when something better can be derived."""
+ stage = self.stage or (stage_for_label(self.label) if self.label else None)
+ role = self.role or (role_for_stage(stage) if stage else None)
+ return replace(self, stage=stage, role=role)
+
+
+_EMPTY = CallContext()
+_ctx: ContextVar[CallContext] = ContextVar("cv3d_cost_ctx", default=_EMPTY)
+
+#: the run this execution context belongs to ("" = nothing bound here)
+_run_var: ContextVar[str] = ContextVar("cv3d_cost_run", default="")
+
+_binding_lock = threading.Lock()
+_run_binding: CallContext = _EMPTY
+
+
+def bind_run(run: str) -> None:
+ """Name the run every subsequent row belongs to.
+
+ Sets both the context-local binding (so a bench worker that runs its own
+ prompt in its own thread cannot have its rows stolen by a sibling) and the
+ process-wide fallback (so a worker thread that inherited no context — the
+ fan-out pools — still finds the run)."""
+ global _run_binding
+ _run_var.set(run or "")
+ with _binding_lock:
+ _run_binding = CallContext(run=run)
+
+
+def run_binding() -> CallContext:
+ """The context-local run if this thread bound one, else the process's."""
+ local = _run_var.get()
+ if local:
+ return CallContext(run=local)
+ with _binding_lock:
+ return _run_binding
+
+
+def current() -> CallContext:
+ """Run binding merged with the ambient (thread-local) context."""
+ return run_binding().merge(_ctx.get())
+
+
+@contextmanager
+def call_context(
+ *,
+ run: str = "",
+ round: int | None = None, # noqa: A002 - matches the record/round vocabulary
+ stage: Stage | str | None = None,
+ role: Role | str | None = None,
+ label: str = "",
+) -> Iterator[CallContext]:
+ """Set the ambient attribution for the duration of the block (this thread).
+
+ Used by :mod:`codeverse.cost.instrument` around an agent session and by the
+ judges around a verdict; nesting merges, so an inner block only has to name
+ what it changes."""
+ ctx = _ctx.get().merge(CallContext(run=run, round=round, stage=_stage(stage), role=_role(role), label=label))
+ token = _ctx.set(ctx)
+ try:
+ yield ctx
+ finally:
+ _ctx.reset(token)
+
+
+def _stage(value: Stage | str | None) -> Stage | None:
+ if value is None or isinstance(value, Stage):
+ return value
+ try:
+ return Stage(value)
+ except ValueError:
+ return stage_for_label(str(value))
+
+
+def _role(value: Role | str | None) -> Role | None:
+ if value is None or isinstance(value, Role):
+ return value
+ try:
+ return Role(value)
+ except ValueError:
+ return None
+
+
+# --------------------------------------------------------------------------- labels
+#: request-label prefix → (stage, role). Longest prefix wins; anything else is
+#: routed through ``stage_for_label`` (which understands generation labels).
+_LABEL_HINTS: tuple[tuple[str, Stage, Role], ...] = (
+ ("planner", Stage.PLAN, Role.PLANNER),
+ ("plan", Stage.PLAN, Role.PLANNER),
+ ("judge", Stage.JUDGE, Role.JUDGE),
+ ("pairwise", Stage.PAIRWISE, Role.JUDGE),
+ ("texture_gate", Stage.TEXTURE, Role.JUDGE),
+ ("texture", Stage.TEXTURE, Role.OTHER),
+ ("scene_texture", Stage.TEXTURE, Role.OTHER),
+ ("caption", Stage.CAPTION, Role.CAPTIONER),
+ ("calibrate", Stage.JUDGE, Role.JUDGE),
+)
+
+
+def context_from_label(label: str) -> CallContext:
+ """Best-effort attribution from a ``ChatRequest.label``.
+
+ Understands the api-agent's ``api-agent:<job label>:t<turn>`` form (the job
+ label carries the stage) and the judge's ``…:r<NN>:s<k>`` round tag."""
+ low = (label or "").strip()
+ if not low:
+ return _EMPTY
+ round_index = _round_from_label(low)
+ body = low
+ if low.startswith("api-agent:"):
+ parts = low.split(":")
+ body = parts[1] if len(parts) > 1 else ""
+ return CallContext(round=round_index, stage=_stated(stage_for_label(body)),
+ role=Role.GENERATOR, label=low)
+ head = body.split(":", 1)[0].lower()
+ best: tuple[int, Stage, Role] | None = None
+ for prefix, stage, role in _LABEL_HINTS:
+ if head.startswith(prefix) and (best is None or len(prefix) > best[0]):
+ best = (len(prefix), stage, role)
+ if best is not None:
+ return CallContext(round=round_index, stage=best[1], role=best[2], label=low)
+ return CallContext(round=round_index, stage=_stated(stage_for_label(head)), label=low)
+
+
+def _stated(stage: Stage | None) -> Stage | None:
+ """``Stage.OTHER`` is "the label told us nothing" — not a statement, so it must
+ not outrank the ambient session context."""
+ return None if stage is Stage.OTHER else stage
+
+
+def _round_from_label(label: str) -> int | None:
+ """``judge:static_object_v1:r02:s1`` → 2 (``None`` when the label has no round tag)."""
+ for token in label.replace("/", ":").split(":"):
+ t = token.strip()
+ if len(t) >= 2 and t[0] == "r" and t[1:].isdigit():
+ return int(t[1:])
+ return None
+
+
+#: Stages a label can name that describe a job of their own. A call whose label
+#: says "judge" / "plan" / "texture" / "caption" / "pairwise" is doing that job
+#: even when an agent session surrounds it, so it outranks the ambient context;
+#: a generation stage in a label does not (see the module docstring).
+SELF_DESCRIBING: frozenset[Stage] = frozenset(
+ {Stage.PLAN, Stage.JUDGE, Stage.PAIRWISE, Stage.TEXTURE, Stage.CAPTION})
+
+
+def attribute(explicit: CallContext | None = None, *, label: str = "") -> CallContext:
+ """The final attribution for one call.
+
+ Precedence (see the module docstring): explicit > a label that names a job of
+ its own > the ambient session context > the rest of the label. The row's
+ ``label`` is always the most specific one available."""
+ ctx = current()
+ if label:
+ from_label = context_from_label(label)
+ ctx = ctx.merge(from_label) if from_label.stage in SELF_DESCRIBING else from_label.merge(ctx)
+ if explicit is not None:
+ ctx = ctx.merge(explicit)
+ if label and not (explicit is not None and explicit.label):
+ ctx = replace(ctx, label=label)
+ return ctx.resolved()
+
+
+__all__ = ["CallContext", "attribute", "bind_run", "call_context", "context_from_label", "current",
+ "run_binding"]
diff --git a/harness/codeverse/cost/instrument.py b/harness/codeverse/cost/instrument.py
new file mode 100644
index 00000000..35cf35e4
--- /dev/null
+++ b/harness/codeverse/cost/instrument.py
@@ -0,0 +1,348 @@
+"""Meter every model call and every agent session into the ledger.
+
+Two thin proxies do the whole job:
+
+* :class:`MeteredChatModel` wraps a :class:`~codeverse.models.base.ChatModel`.
+ Every ``generate`` appends one :class:`~codeverse.cost.types.CallCost` row
+ (tokens, the unit prices actually used + their provenance, $, latency,
+ cache hit, outcome). Because ``models.registry.get_chat_model`` returns the
+ proxy, this covers the planner, the judges, the captioner, single-shot
+ generation **and every turn of the in-process api-agent**.
+* :class:`MeteredAgent` wraps a :class:`~codeverse.agents.base.CodingAgent`.
+ It sets the ambient attribution (round / stage / label) for the session so
+ the model rows above land in the right bucket, applies the profile's turn cap
+ and, for backends whose calls we cannot see (the subscription CLIs — see
+ :data:`IN_PROCESS_AGENT_KINDS`), records one session row from
+ ``AgentResult.usage``.
+
+Accounting is never allowed to fail a run: every hook is wrapped, and an
+exception in the ledger is logged and swallowed. The proxies forward every
+other attribute, so a caller that reaches for ``model.pool`` or ``agent.kind``
+sees the real object's.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from pathlib import Path
+from typing import Any
+
+from codeverse.contracts.agent import AgentJob, AgentResult
+from codeverse.contracts.chat import ChatRequest, ChatResponse
+from codeverse.contracts.common import Usage
+from codeverse.cost.context import bind_run, call_context, run_binding
+from codeverse.cost.ledger import (
+ CostLedger,
+ default_ledger_path,
+ existing_ledger_path,
+ open_run_ledger,
+ record_call,
+ set_default_ledger,
+)
+from codeverse.cost.types import Role, Stage, stage_for_label
+
+log = logging.getLogger(__name__)
+
+#: True while a run is metered call-by-call (:func:`run_ledger`). Aggregate
+#: writers — ``orchestrator.budget.BudgetGuard``, which appends one row per
+#: *charge* — must not also write, or every dollar lands in the ledger twice.
+#:
+#: Context-local first, with a process-wide **count** of the open run ledgers as
+#: the fallback for a worker thread that inherited no context. A plain global
+#: boolean cannot survive ``bench.run_bench --parallel N``: two threads entering
+#: and leaving their own ledgers interleave, and the second one restores "on"
+#: after the first turned it off.
+_per_call_var: ContextVar[bool | None] = ContextVar("cv3d_cost_per_call", default=None)
+_active_lock = threading.Lock()
+_active_ledgers = 0
+
+
+def per_call_metering() -> bool:
+ """True when every model call of the current run is already written to the
+ ledger one by one, so a coarser writer should skip its own row."""
+ local = _per_call_var.get()
+ if local is not None:
+ return local
+ with _active_lock:
+ return _active_ledgers > 0
+
+
+#: Agent kinds whose model calls go through ``models.get_chat_model`` and are
+#: therefore ALREADY one ledger row each (``MeteredChatModel`` wraps that
+#: factory). Every other kind is an opaque subscription CLI: we never see its
+#: turns, it hands back one ``AgentResult.usage`` for the whole session, and that
+#: session row is the only record of the money.
+#:
+#: This used to be decided by a thread-local "did anybody write a row while the
+#: session ran?" counter, which is wrong in both directions and was reproduced by
+#: the verifier (``adversarial.py``): a CLI session during which any in-process
+#: tool billed a model (a texture pass, a summariser) looked metered and its whole
+#: session — $1.23 in the reproduction — was silently dropped; and an in-process
+#: session whose turns ran in another thread looked unmetered and was counted
+#: twice. The backend either meters itself or it does not; that is a property of
+#: the backend, not of what happened to run alongside it.
+IN_PROCESS_AGENT_KINDS = frozenset({"api-agent"})
+
+
+def meters_own_calls(agent: Any) -> bool:
+ """True when this backend's individual model calls are already on the ledger.
+
+ A backend may state it with a ``meters_own_calls`` attribute; otherwise the
+ kind decides (:data:`IN_PROCESS_AGENT_KINDS`)."""
+ declared = getattr(agent, "meters_own_calls", None)
+ if declared is not None:
+ return bool(declared)
+ return str(getattr(agent, "kind", "")) in IN_PROCESS_AGENT_KINDS
+
+
+class MeteredChatModel:
+ """ChatModel proxy that appends one ledger row per call."""
+
+ def __init__(self, inner: Any):
+ self._inner = inner
+
+ # -- protocol ---------------------------------------------------------
+ @property
+ def provider(self) -> str:
+ return getattr(self._inner, "provider", "")
+
+ @property
+ def model(self) -> str:
+ return getattr(self._inner, "model", "")
+
+ @property
+ def id(self) -> str:
+ return str(self._inner.id)
+
+ def supports_vision(self) -> bool:
+ return bool(self._inner.supports_vision())
+
+ def __getattr__(self, name: str) -> Any: # everything else is the real model's
+ return getattr(self._inner, name)
+
+ def __repr__(self) -> str: # pragma: no cover - debugging aid
+ return f"MeteredChatModel({self._inner!r})"
+
+ # -- the meter --------------------------------------------------------
+ def generate(self, request: ChatRequest) -> ChatResponse:
+ t0 = time.perf_counter()
+ try:
+ resp = self._inner.generate(request)
+ except BaseException as exc: # noqa: BLE001 - record the attempt, then re-raise
+ self._record(Usage(backend=self.provider, model=self.model), request,
+ outcome=_outcome(exc), ms=int((time.perf_counter() - t0) * 1000))
+ raise
+ self._record(resp.usage, request, outcome="ok",
+ ms=int((time.perf_counter() - t0) * 1000))
+ return resp
+
+ def _record(self, usage: Usage, request: ChatRequest, *, outcome: str, ms: int) -> None:
+ try:
+ record_call(usage, label=request.label, backend=usage.backend or self.provider,
+ model=usage.model or self.model, outcome=outcome,
+ latency_ms=usage.latency_ms or ms)
+ except Exception as e: # pragma: no cover - accounting must never break a call
+ log.debug("cost: could not record %s: %s", request.label, e)
+
+
+def _outcome(exc: BaseException) -> str:
+ name = type(exc).__name__.lower()
+ if "timeout" in name:
+ return "timeout"
+ return "error"
+
+
+class MeteredAgent:
+ """CodingAgent proxy: attribution for the session + a turn-cap backstop.
+
+ ``tracks.generation.agent_max_turns`` is where the cap is *decided* (it reads
+ ``Settings.limits.agent_max_turns`` and handles the wrap-up continuation).
+ There is no default cap — a 28-turn one lost its A/B, ``docs/COST.md`` §17 —
+ so this is only a backstop for a caller that names one: a bench script, a
+ test, ``$CV3D_AGENT_MAX_TURNS``. It only ever *lowers* ``job.max_turns``, so
+ it composes with whatever the caller set."""
+
+ def __init__(self, inner: Any, *, max_turns: int | None = None):
+ self._inner = inner
+ self.max_turns = max_turns
+
+ @property
+ def kind(self) -> str:
+ return str(getattr(self._inner, "kind", ""))
+
+ @property
+ def model(self) -> str:
+ return str(getattr(self._inner, "model", ""))
+
+ @property
+ def id(self) -> str:
+ return str(self._inner.id)
+
+ def available(self) -> tuple[bool, str]:
+ return self._inner.available()
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._inner, name)
+
+ def __repr__(self) -> str: # pragma: no cover - debugging aid
+ return f"MeteredAgent({self._inner!r})"
+
+ def run(self, job: AgentJob) -> AgentResult:
+ job = self._capped(job)
+ stage = stage_for_label(job.kind or job.label)
+ t0 = time.perf_counter()
+ with call_context(round=job.round, stage=stage, role=Role.GENERATOR, label=job.label):
+ result = self._inner.run(job)
+ if not meters_own_calls(self._inner): # an opaque CLI: its session row is the only record
+ self._record_session(job, result, stage, int((time.perf_counter() - t0) * 1000))
+ return result
+
+ def _capped(self, job: AgentJob) -> AgentJob:
+ cap = self.max_turns if self.max_turns is not None else _settings_turn_cap()
+ if cap and 0 < cap < job.max_turns:
+ return job.model_copy(update={"max_turns": cap})
+ return job
+
+ def _record_session(self, job: AgentJob, result: AgentResult, stage: Stage, ms: int) -> None:
+ usage = result.usage
+ if not (usage.input_tokens or usage.output_tokens or usage.cost_usd):
+ return
+ try:
+ record_call(usage, round=job.round, stage=stage, role=Role.GENERATOR, label=job.label,
+ backend=usage.backend or self.kind, model=usage.model or self.model,
+ outcome="ok" if result.ok else (result.exit_reason or "error"),
+ latency_ms=usage.latency_ms or ms,
+ n_calls=max(1, int(result.tool_calls or 1)), source="session")
+ except Exception as e: # pragma: no cover
+ log.debug("cost: could not record agent session %s: %s", job.label, e)
+
+
+def _settings_turn_cap() -> int:
+ try:
+ from codeverse.config import get_settings
+
+ return int(get_settings().limits.agent_max_turns)
+ except Exception: # pragma: no cover - settings must never break a run
+ return 0
+
+
+# --------------------------------------------------------------------------- factories
+def metered_chat_model(model: Any) -> Any:
+ """Wrap ``model`` unless it is already metered or metering is off."""
+ if model is None or isinstance(model, MeteredChatModel) or not metering_enabled():
+ return model
+ return MeteredChatModel(model)
+
+
+def metered_agent(agent: Any) -> Any:
+ if agent is None or isinstance(agent, MeteredAgent) or not metering_enabled():
+ return agent
+ return MeteredAgent(agent)
+
+
+def metering_enabled() -> bool:
+ """``Settings.cost_ledger`` (env ``CV3D_COST_LEDGER=off`` also disables it)."""
+ try:
+ from codeverse.config import get_settings
+
+ return bool(get_settings().cost_ledger)
+ except Exception: # pragma: no cover
+ return True
+
+
+# --------------------------------------------------------------------------- agent seam
+_install_lock = threading.Lock()
+_installed = False
+
+
+def install_agent_metering() -> bool:
+ """Make ``get_coding_agent`` hand out metered agents (idempotent).
+
+ The agent package builds its backends through one factory, so wrapping that
+ factory is the whole integration; when ``agents/registry.py`` starts calling
+ :func:`metered_agent` itself this seam becomes a no-op."""
+ global _installed
+ with _install_lock:
+ if _installed:
+ return False
+ try:
+ import codeverse.agents as agents_pkg
+ from codeverse.agents import registry as agents_registry
+ except Exception as e: # pragma: no cover - agents are optional for cost-only use
+ log.debug("cost: agent metering unavailable: %s", e)
+ return False
+ inner = agents_registry.get_coding_agent
+ if getattr(inner, "_cv3d_metered", False):
+ _installed = True
+ return False
+
+ def get_coding_agent(agent_id: str) -> Any:
+ return metered_agent(inner(agent_id))
+
+ get_coding_agent._cv3d_metered = True # type: ignore[attr-defined]
+ get_coding_agent.__doc__ = inner.__doc__
+ agents_registry.get_coding_agent = get_coding_agent # type: ignore[assignment]
+ agents_pkg.get_coding_agent = get_coding_agent # type: ignore[attr-defined]
+ _installed = True
+ return True
+
+
+# --------------------------------------------------------------------------- run activation
+@contextmanager
+def run_ledger(workspace: str | Path, *, run: str = "", create: bool = True) -> Iterator[CostLedger | None]:
+ """Meter one run into ``<workspace>/telemetry/cost.jsonl``.
+
+ Binds the run name, points :func:`~codeverse.cost.ledger.record_call` at the
+ run's ledger and installs the agent seam; restores the previous default on
+ the way out so a second run in the same process is not mixed in.
+
+ ``create=False`` is for work done *after* a run finished (``3dcv judge``,
+ a post-hoc texture pass): it appends only when the run already keeps a
+ ledger, because a ledger holding nothing but the re-judge would be read as
+ the whole run's cost and hide everything the run really spent. Without one
+ the rows go to the per-process log instead.
+
+ **Nests and parallelises.** The previous ledger and run binding are restored
+ on the way out rather than cleared, so a bench cell that opens a ledger for
+ the cell and then a second one for the harness run inside it keeps both; and
+ both are context-local first (see :func:`codeverse.cost.context.bind_run`), so
+ ``bench.run_bench`` can run N prompts in N threads without their rows mixing."""
+ if not metering_enabled():
+ yield None
+ return
+ global _active_ledgers
+ ws = Path(workspace)
+ prev_run = run_binding().run
+ prev_path = default_ledger_path()
+ prev_metering = _per_call_var.get()
+ if not create and existing_ledger_path(ws) is None:
+ bind_run(run or ws.name)
+ try:
+ yield None
+ finally:
+ bind_run(prev_run)
+ return
+ ledger = open_run_ledger(ws)
+ set_default_ledger(ledger.path)
+ bind_run(run or ws.name)
+ install_agent_metering()
+ _per_call_var.set(True)
+ with _active_lock:
+ _active_ledgers += 1
+ try:
+ yield ledger
+ finally:
+ _per_call_var.set(prev_metering)
+ with _active_lock:
+ _active_ledgers -= 1
+ set_default_ledger(prev_path)
+ bind_run(prev_run)
+
+
+__all__ = ["MeteredAgent", "MeteredChatModel", "install_agent_metering", "metered_agent",
+ "metered_chat_model", "metering_enabled", "per_call_metering", "run_ledger"]
diff --git a/harness/codeverse/cost/ledger.py b/harness/codeverse/cost/ledger.py
index e5dbbf3a..a683302b 100644
--- a/harness/codeverse/cost/ledger.py
+++ b/harness/codeverse/cost/ledger.py
@@ -20,10 +20,12 @@ import os
import threading
import time
from collections.abc import Iterable, Sequence
+from contextvars import ContextVar
from pathlib import Path
from typing import Any
from codeverse.contracts.common import Usage
+from codeverse.cost.context import CallContext, attribute
from codeverse.cost.types import (
CallCost,
CostBucket,
@@ -31,6 +33,7 @@ from codeverse.cost.types import (
Stage,
Summary,
normalise_ids,
+ role_for_stage,
stage_for_label,
)
from codeverse.models.pricing import (
@@ -42,11 +45,20 @@ from codeverse.models.pricing import (
log = logging.getLogger(__name__)
-#: default ledger file inside a run workspace
+#: the ledger of a run lives in its telemetry bucket (docs/RUN_LAYOUT.md §telemetry)
+TELEMETRY_DIR = "telemetry"
+TELEMETRY_LEDGER = f"{TELEMETRY_DIR}/cost.jsonl"
+
+#: legacy/alias name at the run root. ``flywheel.telemetry.live_ledger_path`` looks
+#: here for "does this run have a live ledger", and runs recorded before the
+#: telemetry bucket existed have the real file here — so it stays readable, and
+#: :func:`open_run_ledger` leaves a symlink pointing at the telemetry copy.
LEDGER_NAME = "cost_ledger.jsonl"
#: env var that points ``record_call`` at a ledger when no path is passed
+#: (``off`` / ``0`` / ``none`` disables ledger writing for the process)
LEDGER_ENV = "CV3D_COST_LEDGER"
+_LEDGER_OFF = frozenset({"off", "0", "no", "none", "false"})
DIMENSIONS = ("run", "stage", "role", "backend", "model", "provider", "round", "outcome")
@@ -84,32 +96,130 @@ class CostLedger:
_default_lock = threading.Lock()
_default: CostLedger | None = None
+_fallback: CostLedger | None = None
+_fallback_read = False
+
+#: sentinel: "this execution context did not state a ledger"
+_UNSET: Any = object()
+#: context-local override of the default ledger. Mirrors
+#: ``codeverse.cost.context.bind_run``: a bench worker running its own prompt in
+#: its own thread must not have its rows land in a sibling run's file, while a
+#: plain worker thread that inherited no context still finds the process's ledger.
+_default_var: ContextVar[Any] = ContextVar("cv3d_cost_default_ledger", default=_UNSET)
def set_default_ledger(path: str | Path | None) -> CostLedger | None:
- """Point the module-level :func:`record_call` at ``path`` (``None`` disables it)."""
- global _default
+ """Point the module-level :func:`record_call` at ``path``.
+
+ ``None`` clears the override (back to ``$CV3D_COST_LEDGER`` / the per-process
+ log). Sets both the context-local override and the process-wide one, so a
+ nested or parallel run is attributed correctly and an uninstrumented worker
+ thread still writes somewhere sensible."""
+ global _default, _fallback_read
+ led = CostLedger(path) if path is not None else None
+ _default_var.set(led if led is not None else _UNSET)
+ with _default_lock:
+ _default = led
+ if led is None:
+ _fallback_read = False # re-read $CV3D_COST_LEDGER next time
+ return led
+
+
+def default_ledger_path() -> Path | None:
+ """The path :func:`set_default_ledger` last set here (``None`` = no override).
+ Used to save/restore around a nested :func:`~codeverse.cost.instrument.run_ledger`."""
+ led = _default_var.get()
+ if led is not _UNSET:
+ return led.path if led is not None else None
with _default_lock:
- _default = CostLedger(path) if path is not None else None
- return _default
+ return _default.path if _default is not None else None
def default_ledger() -> CostLedger | None:
"""The ledger :func:`record_call` writes to when no ``ledger=`` is given:
- whatever :func:`set_default_ledger` set, else ``$CV3D_COST_LEDGER``."""
- global _default
+ whatever :func:`set_default_ledger` set, else ``$CV3D_COST_LEDGER``, else the
+ per-process fallback log (:func:`process_ledger_path`) so a call made outside
+ any run — ``3dcv judge``, a bench script, a notebook — is still accounted for.
+ ``CV3D_COST_LEDGER=off`` turns writing off entirely."""
+ global _fallback, _fallback_read
+ led = _default_var.get()
+ if led is not _UNSET:
+ return led
with _default_lock:
if _default is not None:
return _default
+ if _fallback_read:
+ return _fallback
env = os.environ.get(LEDGER_ENV, "").strip()
- if env:
- _default = CostLedger(env)
- return _default
+ _fallback = None if env.lower() in _LEDGER_OFF else CostLedger(env or process_ledger_path())
+ _fallback_read = True
+ return _fallback
+
+
+def process_ledger_path() -> Path:
+ """The fallback log for calls made with no run context: one file per process
+ under ``<cache_dir>/cost/``. Never inside a run directory, so it can never be
+ mistaken for a run's own ledger."""
+ try:
+ from codeverse.config import get_settings
+
+ base = Path(get_settings().cache_dir)
+ except Exception: # pragma: no cover - settings must never break accounting
+ base = Path.home() / ".cache" / "codeverse"
+ stamp = time.strftime("%Y%m%d", time.localtime())
+ return base / "cost" / f"p{os.getpid()}-{stamp}.jsonl"
def ledger_path(workspace: str | Path) -> Path:
- """The conventional ledger path of a run workspace (``<ws>/cost_ledger.jsonl``)."""
- return Path(workspace) / LEDGER_NAME
+ """The live ledger of a run workspace: ``<ws>/telemetry/cost.jsonl``."""
+ return Path(workspace) / TELEMETRY_LEDGER
+
+
+def existing_ledger_path(workspace: str | Path) -> Path | None:
+ """The ledger file a run actually has — the telemetry one, else the legacy
+ root file — or ``None``."""
+ root = Path(workspace)
+ for name in (TELEMETRY_LEDGER, LEDGER_NAME):
+ p = root / name
+ if p.is_file():
+ return p
+ return None
+
+
+def open_run_ledger(workspace: str | Path) -> CostLedger:
+ """The run's live ledger, ready to append to.
+
+ Writes to ``telemetry/cost.jsonl`` and leaves ``<run>/cost_ledger.jsonl`` as a
+ relative symlink to it, so there is exactly one physical copy and the run
+ layout's ``live_ledger_path`` / ``telemetry/usage.jsonl`` alias keep working."""
+ root = Path(workspace)
+ path = root / TELEMETRY_LEDGER
+ try:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ alias = root / LEDGER_NAME
+ if not alias.exists() and not alias.is_symlink():
+ os.symlink(TELEMETRY_LEDGER, alias)
+ except OSError as e: # pragma: no cover - read-only dir / no symlinks
+ log.debug("cost: could not prepare the run ledger at %s: %s", path, e)
+ return CostLedger(path)
+
+
+def _as_stage(value: Stage | str | None) -> Stage | None:
+ if value is None or isinstance(value, Stage):
+ return value
+ try:
+ return Stage(value)
+ except ValueError:
+ return stage_for_label(str(value))
+
+
+def _as_role(value: Role | str | None) -> Role | None:
+ if value is None or isinstance(value, Role):
+ return value
+ try:
+ return Role(value)
+ except ValueError:
+ return Role.OTHER
def price_call(
@@ -162,7 +272,7 @@ def record_call(
usage: Usage | None = None,
*,
run: str = "",
- stage: Stage | str = Stage.OTHER,
+ stage: Stage | str | None = None,
role: Role | str | None = None,
round: int | None = None, # noqa: A002 - matches the record/round vocabulary
label: str = "",
@@ -186,21 +296,17 @@ def record_call(
record_call(res.usage, run=ws.slug, round=idx, stage="baseline",
role="generator", ledger=ws.root / "cost_ledger.jsonl")
+ Anything the caller leaves out (``run`` / ``round`` / ``stage`` / ``role``) is
+ resolved from the ambient run context and the call label — see
+ :mod:`codeverse.cost.context`.
+
Never raises: a failure to account is logged, not propagated."""
u = usage or Usage()
- try:
- st = Stage(stage) if not isinstance(stage, Stage) else stage
- except ValueError:
- st = stage_for_label(str(stage))
- if role is None:
- from codeverse.cost.types import role_for_stage
-
- rl = role_for_stage(st)
- else:
- try:
- rl = Role(role) if not isinstance(role, Role) else role
- except ValueError:
- rl = Role.OTHER
+ ctx = attribute(CallContext(run=run, round=round, stage=_as_stage(stage), role=_as_role(role),
+ label=label), label=label)
+ st = ctx.stage or Stage.OTHER
+ rl = ctx.role or role_for_stage(st)
+ run, round, label = ctx.run, ctx.round, ctx.label # noqa: A001 - see the signature
try:
cost, price_fields = price_call(u, backend=backend, model=model,
cache_write_tokens=cache_write_tokens, reprice=reprice)
@@ -237,11 +343,15 @@ def record_call(
def load_ledger(path: str | Path) -> list[CallCost]:
- """Read a ledger file (or a directory containing one). Bad lines are skipped
- with a debug log — a truncated last line never loses the rest of the file."""
+ """Read a ledger file (or a run directory containing one: ``telemetry/cost.jsonl``
+ first, then the legacy root ``cost_ledger.jsonl``). Bad lines are skipped with a
+ debug log — a truncated last line never loses the rest of the file."""
p = Path(path)
if p.is_dir():
- p = p / LEDGER_NAME
+ found = existing_ledger_path(p)
+ if found is None:
+ return []
+ p = found
if not p.is_file():
return []
rows: list[CallCost] = []
diff --git a/harness/codeverse/cost/profiles.py b/harness/codeverse/cost/profiles.py
new file mode 100644
index 00000000..2f75d9a5
--- /dev/null
+++ b/harness/codeverse/cost/profiles.py
@@ -0,0 +1,170 @@
+"""Cost/quality profiles — one name that sets the whole dial coherently.
+
+``3dcv make --profile economy|balanced|quality`` picks a :class:`Profile`; it
+resolves the model per role, the judge sample count, the number of refine
+rounds, the best-of-N width, the judge payload, the texture pass and the budget
+ceilings in one consistent move. Mixing knobs by hand is how a run ends up
+paying for a pro judge on a single-shot artifact, or running four refine rounds
+on a graphics shader that passed at round 0.
+
+``--profile X`` and ``CV3D_PROFILE=X`` (or ``profile:`` in ``config.yaml``)
+resolve to the **same** dial: ``codeverse.cli._common.resolve_dial`` is the one
+place that reads it, and ``tests/cost/test_profiles.py`` asserts every
+field of the resolved dial from both entry points. The only difference is
+precedence — the flag forces the dial over a value you stated yourself, the
+env var yields to it.
+
+Every number is taken from the measurements in ``docs/COST.md``:
+
+* **economy** — single-shot generation (graphics: 5/6 passed at $0.10/run; a
+ one-file generator plus compiler feedback needs no tool loop), the flash judge
+ at ``n=2`` (σ 0.083/√2 ≈ 0.059 — enough to rank two rounds, not enough for a
+ pass/fail that persists) and **2 refine rounds** (r01 buys a score point for
+ $4.93, r02 for $6.80, r03 for $33.49). It does **not** shrink the judge
+ payload and does **not** carry its own turn cap — both were measured and
+ neither paid (see the field comments below).
+* **balanced** — exactly today's defaults (api-agent generator, pro judge at
+ n=1, 4 rounds, no best-of-N, no texture): the arm every number in the audit
+ was measured on.
+* **quality** — api-agent + the pro judge at ``n=3`` (σ 0.030/√3 ≈ 0.017),
+ best-of-2 baselines and the texture pass behind its before/after gate.
+
+The expected $ per profile comes from the same 61 runs; see
+:func:`Profile.expectation` and ``docs/COST.md`` §8–9.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+#: profile names in increasing order of spend
+PROFILE_NAMES = ("economy", "balanced", "quality")
+
+DEFAULT_PROFILE = "balanced"
+
+
+@dataclass(frozen=True)
+class Profile:
+ """One coherent setting of the cost/quality dial."""
+
+ name: str
+ #: model per role ("" = leave the settings default alone)
+ generator: str = ""
+ planner: str = ""
+ judge: str = ""
+ captioner: str = ""
+ #: run shape
+ rounds: int = 4 # refine rounds after the baseline
+ candidates: int = 1 # best-of-N baselines
+ judge_samples: int = 1 # VLM judge samples per verdict
+ max_turns: int = 0 # agent turn cap (0 = the backend's own default)
+ texture: bool = False # run the derived texture pass
+ #: judge payload
+ judge_max_px: int = 1024 # measured: 768 bills the same and is noisier — see PROFILES
+ judge_montages: int = 3
+ judge_detail_crops: int = 2 # one crop ≈ 1,198 input tokens ($0.0024 on the pro judge);
+ # NOT reduced by any profile — the score effect is inside the
+ # judge's own noise in both directions (docs/COST.md §14)
+ #: budget ceilings a run of this shape should not need to exceed
+ max_usd: float = 5.0
+ max_minutes: float = 60.0
+ #: measured expectation (docs/COST.md) — reported by ``3dcv cost profiles``
+ expected_usd: float = 0.0
+ expected_score: str = ""
+ note: str = ""
+
+ def expectation(self) -> str:
+ return f"~${self.expected_usd:.2f}/run, {self.expected_score}"
+
+ def with_overrides(self, **kw: object) -> Profile:
+ """Copy with the fields the user stated explicitly (``None`` = not stated)."""
+ clean = {k: v for k, v in kw.items() if v is not None and hasattr(self, k)}
+ return replace(self, **clean) if clean else self # type: ignore[arg-type]
+
+
+#: the three dials. Generator ids are harness backend ids; judge/planner ids are
+#: ``<provider>:<model>``.
+PROFILES: dict[str, Profile] = {
+ "economy": Profile(
+ name="economy",
+ generator="single-shot:gemini:gemini-3.7-flash",
+ planner="gemini:gemini-3.7-flash",
+ judge="gemini:gemini-3.7-flash",
+ captioner="gemini:gemini-3.7-flash",
+ rounds=2, candidates=1, judge_samples=2, texture=False,
+ # max_turns=0 (2026-08-23): economy's generator is ``single-shot:``, which opens no
+ # agent session, so the 20-turn cap this profile used to carry could never fire. And
+ # the cap it was modelled on did not survive its own A/B — 3 runs per arm, a 28-turn
+ # cap cost $0.02 MORE and 0.205 of a score point (docs/COST.md §17) — so there is no
+ # default cap anywhere now. A caller who wants one still has
+ # ``Settings.limits.agent_max_turns`` / ``$CV3D_AGENT_MAX_TURNS``.
+ max_turns=0,
+ # 1024 px, not 768: measured on 8 recorded rounds, Gemini bills a montage the same at
+ # both sizes (156,834 vs 156,874 input tokens over 16 verdicts) while 768 raised the
+ # sampling σ 0.033 → 0.042.
+ # detail_crops=2, i.e. the same payload as every other profile. Dropping to 1 crop saves
+ # 1,198 input tokens ($0.0024/verdict on pro) and was adopted on a single draw that read
+ # 0.595 → 0.600; an independent re-draw of the same round read 0.600 → 0.552, a 0.048
+ # swing = 1.6x the pro judge's measured σ (0.030). Two draws that disagree by more than
+ # the instrument's noise do not license a payload cut worth a fifth of a cent, so the
+ # experiment is recorded as INCONCLUSIVE in docs/COST.md §14 and the budget is unchanged.
+ judge_max_px=1024, judge_montages=3, judge_detail_crops=2,
+ max_usd=1.50, max_minutes=30.0,
+ expected_usd=0.30,
+ expected_score="graphics 0.81 median (5/6 passed); objects clear the gates less often "
+ "— a one-shot flash generator scored 0.14 on compare_v1's hard cells",
+ note="single-shot generator + flash judge n=2 + 2 refine rounds; no best-of-N, no texture, "
+ "full judge payload",
+ ),
+ "balanced": Profile(
+ name="balanced",
+ generator="api-agent:gemini:gemini-3.7-flash",
+ planner="gemini:gemini-3.7-flash",
+ judge="gemini:gemini-3.1-pro-preview",
+ captioner="gemini:gemini-3.7-flash",
+ rounds=4, candidates=1, judge_samples=1, max_turns=0, texture=False,
+ judge_max_px=1024, judge_montages=3, judge_detail_crops=2,
+ max_usd=5.0, max_minutes=60.0,
+ expected_usd=1.47,
+ expected_score="0.835 mean on compare_v1, 36/61 runs passed ($2.50 per passing artifact)",
+ note="today's defaults — the arm every number in docs/COST.md was measured on",
+ ),
+ "quality": Profile(
+ name="quality",
+ generator="api-agent:gemini:gemini-3.7-flash",
+ planner="gemini:gemini-3.7-flash",
+ judge="gemini:gemini-3.1-pro-preview",
+ captioner="gemini:gemini-3.7-flash",
+ rounds=4, candidates=2, judge_samples=3, max_turns=0, texture=True,
+ judge_max_px=1024, judge_montages=3, judge_detail_crops=2,
+ max_usd=8.0, max_minutes=90.0,
+ expected_usd=3.20,
+ expected_score="best-of-2 lifted the stool baseline 0.563 → 0.612 and the texture pass "
+ "0.686 → 0.701; judge σ 0.017 at n=3",
+ note="api-agent + pro judge n=3 + 4 rounds + best-of-2 + texture pass",
+ ),
+}
+
+
+def get_profile(name: str | None) -> Profile:
+ """The named profile (case-insensitive). Unknown names raise ``ValueError``."""
+ key = (name or DEFAULT_PROFILE).strip().lower()
+ if key not in PROFILES:
+ raise ValueError(f"unknown profile {name!r}; known: {', '.join(PROFILE_NAMES)}")
+ return PROFILES[key]
+
+
+def profile_table() -> list[tuple[str, str, str, str, str, str, str]]:
+ """Rows for ``3dcv cost profiles``: name, generator, judge, shape, montage, $, note."""
+ rows = []
+ for name in PROFILE_NAMES:
+ p = PROFILES[name]
+ shape = f"{p.rounds}r × {p.candidates}cand, judge n={p.judge_samples}" + (
+ f", ≤{p.max_turns} turns" if p.max_turns else "")
+ rows.append((p.name, p.generator, p.judge, shape,
+ f"{p.judge_max_px}px/{p.judge_detail_crops}crop" + (" +texture" if p.texture else ""),
+ f"${p.expected_usd:.2f}", p.expected_score))
+ return rows
+
+
+__all__ = ["DEFAULT_PROFILE", "PROFILES", "PROFILE_NAMES", "Profile", "get_profile", "profile_table"]
diff --git a/harness/codeverse/cost/reconstruct.py b/harness/codeverse/cost/reconstruct.py
index 517886c6..cd4a1bcd 100644
--- a/harness/codeverse/cost/reconstruct.py
+++ b/harness/codeverse/cost/reconstruct.py
@@ -30,6 +30,7 @@ from __future__ import annotations
import json
import logging
+import re
from collections.abc import Iterator
from dataclasses import dataclass, field
from pathlib import Path
@@ -113,6 +114,7 @@ class RunLedger:
parallel: bool = False
resumed: bool = False
selected_candidate: str = "" # "c1" when the run ran best-of-N and c1 won
+ source: str = "reconstructed" # live (the run wrote its own ledger) | reconstructed
rows: list[CallCost] = field(default_factory=list)
notes: list[str] = field(default_factory=list)
@@ -165,6 +167,24 @@ def _gemini_cli_usages(stdout: dict[str, Any]) -> list[Usage]:
return out
+#: a retried / wrapped-up agent session records itself as ``<label>.a2`` / ``<label>.wrapup``
+_ATTEMPT_SUFFIX = re.compile(r"\.(a\d+|wrapup)$")
+
+
+def _base_label(label: str) -> str:
+ """The *job* label a session label was derived from.
+
+ A retry is recorded as ``<label>.a2``, a wrap-up as ``<label>.wrapup`` and a
+ best-of-N candidate as ``c<k>:<label>``, while the ``generate.done`` event that
+ summarises the whole job carries the plain job label. De-duplication has to
+ use this key: keying it on the session's own label left the retry's dollars in
+ a different bucket, the job's pool ran dry, and the event was counted a SECOND
+ time. That artifact — not lost money — was most of the "$4.80 invisible to
+ record.total_usage" in docs/COST.md §6; §6 now quotes the corrected figure."""
+ lab = label.split(":", 1)[-1] if ":" in label else label
+ return _ATTEMPT_SUFFIX.sub("", lab)
+
+
def _session_rows(d: Path, run: str, *, recheck: bool,
candidate: str = "") -> tuple[list[CallCost], str, int | None, float]:
"""Rows for one ``trajectories/<label>_rNN`` directory. ``candidate`` is the
@@ -203,6 +223,35 @@ def _session_rows(d: Path, run: str, *, recheck: bool,
return rows, label, rnd, duration
+def _live_rows(root: Path, *, run: str, recheck: bool) -> list[CallCost]:
+ """The run's own ledger rows, when it wrote one while it ran.
+
+ Rows carry their run name from the writer; stamp the directory's name on any
+ that do not (a battery copies runs around). ``recheck`` re-prices them from
+ today's table exactly like a reconstructed row."""
+ try:
+ from codeverse.cost.ledger import load_ledger
+
+ rows = load_ledger(root)
+ except Exception as e: # noqa: BLE001 - a corrupt ledger falls back to reconstruction
+ log.debug("cost: live ledger unreadable in %s: %s", root, e)
+ return []
+ out: list[CallCost] = []
+ for row in rows:
+ if not row.run:
+ row.run = run
+ if recheck:
+ usage = Usage(backend=row.backend, model=row.model_id or row.model,
+ input_tokens=row.input_tokens, cached_tokens=row.cached_tokens,
+ output_tokens=row.output_tokens, thoughts_tokens=row.thoughts_tokens,
+ tool_calls=row.tool_calls, cost_usd=row.recorded_usd)
+ cost, fields = price_call(usage, backend=row.backend, model=row.model_id or row.model,
+ cache_write_tokens=row.cache_write_tokens, reprice=True)
+ row = row.model_copy(update={"cost_usd": cost, **fields})
+ out.append(row)
+ return out
+
+
def _trajectory_dirs(root: Path) -> list[tuple[Path, str]]:
"""Every session directory of a run as ``(dir, candidate)``, including scene
asset sub-workspaces (``_assets/<name>/trajectories/*``) and best-of-N
@@ -220,6 +269,12 @@ def _trajectory_dirs(root: Path) -> list[tuple[Path, str]]:
def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
"""Build the ledger of one run directory.
+ A run that wrote its own ledger while it ran (``telemetry/cost.jsonl``, see
+ :mod:`codeverse.cost.instrument`) is read from that ledger — one priced row
+ per real call, nothing inferred. Runs recorded before the ledger existed are
+ rebuilt from ``trajectories/`` + ``record.json`` + ``events.jsonl`` exactly as
+ before, so all 61 reference runs keep auditing.
+
``recheck=True`` re-prices every row with the CURRENT price table (and rebuilds
gemini-cli rows from the raw CLI stats) — that is the audit view; the default
keeps the dollars the run was actually billed."""
@@ -246,15 +301,21 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
led.n_rounds = len(rounds)
led.passed = bool(record.get("status") == "passed")
- covered: dict[str, float] = {} # label -> $ already accounted for by its agent session(s);
- # a ``generate.done`` event within that budget is a duplicate
+ live_rows = _live_rows(root, run=run, recheck=recheck)
+ led.source = "live" if live_rows else "reconstructed"
+ led.rows += live_rows
+
+ covered: dict[str, float] = {} # JOB label (see _base_label) -> $ already accounted for by
+ # its session(s); a ``generate.done`` event within that
+ # budget is a duplicate, retries and wrap-ups included
model_s = 0.0
- for d, cand in _trajectory_dirs(root):
+ for d, cand in (() if live_rows else _trajectory_dirs(root)):
rows, label, rnd, duration = _session_rows(d, run, recheck=recheck, candidate=cand)
if rows:
led.rows += rows
- covered[label] = covered.get(label, 0.0) + sum(r.recorded_usd for r in rows)
+ key = _base_label(label)
+ covered[key] = covered.get(key, 0.0) + sum(r.recorded_usd for r in rows)
model_s += duration
judged_rounds: set[int] = set()
@@ -267,6 +328,8 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
ju = _usage(judgment.get("usage"))
if ju.input_tokens or ju.cost_usd:
judged_rounds.add(idx)
+ model_s += ju.latency_ms / 1000.0
+ if (ju.input_tokens or ju.cost_usd) and not live_rows:
n = int(judgment.get("n_samples") or 1)
degraded = bool((judgment.get("raw") or {}).get("degraded")) if isinstance(judgment.get("raw"), dict) else False
led.rows.append(_row(ju, run=run, stage=Stage.JUDGE, label=str(judgment.get("rubric") or "judge"),
@@ -274,7 +337,6 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
outcome="degraded" if degraded else "ok", reprice=recheck,
backend=str(judgment.get("judge_backend") or ju.backend) or ju.backend,
model=str(judgment.get("judge_backend") or ju.model) or ju.model))
- model_s += ju.latency_ms / 1000.0
# ------------------------------------------------------------------ events
first_t = last_t = 0.0
@@ -295,14 +357,16 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
if name == "judge.done":
model_s += float(ev.get("duration_s") or 0.0) if int(ev.get("round", -1)) not in judged_rounds else 0.0
cost = ev.get("cost_usd")
- if not isinstance(cost, (int, float)) or not cost or name in _AGGREGATE_EVENTS:
+ if live_rows or not isinstance(cost, (int, float)) or not cost or name in _AGGREGATE_EVENTS:
continue
rnd = ev.get("round")
rnd = int(rnd) if isinstance(rnd, (int, float)) else None
label = str(ev.get("label") or name.split(".")[0])
- if name == "generate.done" and covered.get(label, 0.0) >= float(cost) - 0.0005:
- covered[label] = covered.get(label, 0.0) - float(cost)
- continue # a session already contributed this spend
+ if name == "generate.done":
+ key = _base_label(label)
+ if covered.get(key, 0.0) >= float(cost) - 0.0005:
+ covered[key] = covered[key] - float(cost)
+ continue # a session already contributed this spend
if name == "judge.done" and rnd in judged_rounds:
continue
stage = Stage.TEXTURE if name.startswith("texture") else stage_for_label(label)
@@ -363,6 +427,12 @@ def reconstruct_cell(cell_dir: str | Path, *, recheck: bool = False) -> RunLedge
status=str(cell.get("status") or ""), passed=bool(cell.get("passed")),
final_score=cell.get("score"), n_rounds=1,
wall_s=float(cell.get("wall_s") or 0.0), model_s=float(cell.get("gen_seconds") or 0.0))
+ live = _live_rows(root, run=run, recheck=recheck)
+ if live: # the cell wrote its own ledger while it ran: one priced row per real call
+ led.source = "live"
+ led.rows += live
+ led.recorded_usd = float(cell.get("gen_cost_usd") or 0.0) + float(cell.get("judge_cost_usd") or 0.0)
+ return led
for attempt in sorted(root.glob("gen/attempt*")):
result = _read_json(attempt / "result.json")
usage = _usage(result.get("usage"))
@@ -415,5 +485,10 @@ def reconstruct(path: str | Path, *, recheck: bool = False) -> RunLedger:
if (p / "run" / "record.json").is_file():
led = reconstruct_run(p / "run", recheck=recheck)
led.run = f"{p.parent.name}/{p.name}"
+ # a compare cell also spends OUTSIDE the harness run it wraps: the fixed
+ # evaluator's judge, and a repair arm's extra generations. `run_cell` opens a
+ # ledger for the cell itself (nested around the run's own), and that spend is
+ # exactly the §6 gap the audit sees on the two cmp_easy_stool harness cells.
+ led.rows += _live_rows(p, run=led.run, recheck=recheck)
return led
return reconstruct_cell(p, recheck=recheck)
diff --git a/harness/codeverse/fanout.py b/harness/codeverse/fanout.py
index f111d18d..9d5a8fdb 100644
--- a/harness/codeverse/fanout.py
+++ b/harness/codeverse/fanout.py
@@ -9,6 +9,7 @@ retried). Exceptions that are ``BaseException`` but not ``Exception``
from __future__ import annotations
+import contextvars
import logging
import time
from collections.abc import Callable, Iterable, Sequence
@@ -41,6 +42,17 @@ class FanOutReport:
}
+def _in_caller_context(snapshot: contextvars.Context, fn: Callable[[int], None], i: int) -> None:
+ """Run ``fn(i)`` with the values the caller's context held at fan-out time.
+
+ ``Context.run`` cannot be re-entered from several threads, so each worker
+ replays the snapshot's variables into its own (already fresh) context.
+ """
+ for var, value in snapshot.items():
+ var.set(value)
+ fn(i)
+
+
def fan_out(
items: Sequence[T] | Iterable[T],
fn: Callable[[T], R],
@@ -57,6 +69,11 @@ def fan_out(
seconds)`` — keep it cheap and thread-safe (e.g. an EventLog.emit).
"""
items = list(items)
+ # Workers inherit the caller's context so ambient state set with contextvars
+ # (the cost ledger's run/stage/role attribution) follows a parallel judge
+ # sample, best-of-N candidate or bench cell instead of falling back to the
+ # process default.
+ ctx_snapshot = contextvars.copy_context()
results: list[R | Exception] = [None] * len(items) # type: ignore[list-item]
if not items:
return results
@@ -85,7 +102,8 @@ def fan_out(
_run(i)
else:
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=label) as pool:
- list(pool.map(_run, range(len(items))))
+ list(pool.map(lambda i: contextvars.copy_context().run(_in_caller_context, ctx_snapshot, _run, i),
+ range(len(items))))
n_failed = sum(1 for r in results if isinstance(r, Exception))
report = FanOutReport(label, len(items), len(items) - n_failed, n_failed, durations, time.time() - t_all)
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index 358c4d23..b2ec16a9 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -65,9 +65,9 @@ class VlmJudge:
*,
thinking: Literal["off", "low", "medium", "high"] = "low",
max_attempts: int = 3,
- max_montages: int = 3,
- detail_crops: int = 2,
- max_px: int = 1024,
+ max_montages: int | None = None,
+ detail_crops: int | None = None,
+ max_px: int | None = None,
chat_model: ChatModel | None = None,
cache_dir: Path | None = None,
label: str = "judge",
@@ -78,9 +78,11 @@ class VlmJudge:
self.temperature = temperature
self.thinking = thinking
self.max_attempts = max(1, int(max_attempts))
- self.max_montages = max(1, int(max_montages))
- self.detail_crops = max(0, int(detail_crops))
- self.max_px = max_px
+ # payload size: the profile's dial (Settings.judge) unless the caller states one
+ jd = get_settings().judge
+ self.max_montages = max(1, int(jd.montages if max_montages is None else max_montages))
+ self.detail_crops = max(0, int(jd.detail_crops if detail_crops is None else detail_crops))
+ self.max_px = int(jd.max_px if max_px is None else max_px)
self._model = chat_model
self.cache_dir = cache_dir
self.label = label
@@ -122,7 +124,10 @@ class VlmJudge:
)
reqs.append(ChatRequest(
messages=messages, system=system, response_schema=schema, temperature=self.temperature,
- thinking=self.thinking, label=f"{self.label}:{self.rubric.name}:s{k}",
+ thinking=self.thinking,
+ # the label is the ledger's attribution when the sample runs in a worker
+ # thread (fan_out does not carry context vars): keep stage + round in it
+ label=f"{self.label}:{self.rubric.name}:r{inp.round_index:02d}:s{k}",
))
if len(reqs) == 1: # serial path: no pool, no thread hop
outcomes = [self._sample(reqs[0], acceptance_ids, measured)]
diff --git a/harness/codeverse/models/gemini_image.py b/harness/codeverse/models/gemini_image.py
index 1af91063..1ae7e619 100644
--- a/harness/codeverse/models/gemini_image.py
+++ b/harness/codeverse/models/gemini_image.py
@@ -120,6 +120,19 @@ def _is_model_missing(err: ModelError) -> bool:
return err.status == 404 or (err.status == 400 and ("not found" in s or "not supported" in s))
+def _record(usage: Usage, n_images: int) -> None:
+ """One ledger row per image batch (the image model is not a ChatModel, so
+ ``models.registry`` cannot meter it). Never raises."""
+ try:
+ from codeverse.cost import record_call
+ from codeverse.cost.types import Role, Stage
+
+ record_call(usage, stage=Stage.TEXTURE, role=Role.IMAGE, label="image",
+ backend="gemini-image", model=usage.model, n_calls=max(1, n_images))
+ except Exception: # noqa: BLE001 - accounting must never break a texture pass
+ pass
+
+
class GeminiImageModel:
"""See module docstring. ``model`` is the primary; ``fallback`` the second try."""
@@ -193,6 +206,7 @@ class GeminiImageModel:
usage = usage + u
if len(images) > n:
images = images[:n]
+ _record(usage, len(images))
return images, usage
# ------------------------------------------------------------------ internals
diff --git a/harness/codeverse/models/registry.py b/harness/codeverse/models/registry.py
index 69dc710e..56bf1644 100644
--- a/harness/codeverse/models/registry.py
+++ b/harness/codeverse/models/registry.py
@@ -1,4 +1,9 @@
-"""Model id parsing + factory. Ids: ``<provider>:<model>``."""
+"""Model id parsing + factory. Ids: ``<provider>:<model>``.
+
+Every model handed out here is wrapped by ``codeverse.cost.instrument`` so each
+call lands in the cost ledger (the run's ``telemetry/cost.jsonl`` when a run is
+active, otherwise a per-process log). ``CV3D_COST_LEDGER=off`` /
+``Settings.cost_ledger=false`` returns the bare model."""
from __future__ import annotations
@@ -20,6 +25,13 @@ def parse_model_id(model_id: str) -> tuple[str, str]:
@lru_cache(maxsize=32)
def get_chat_model(model_id: str) -> ChatModel:
+ """The (metered) ChatModel for ``<provider>:<model>``."""
+ from codeverse.cost.instrument import metered_chat_model
+
+ return metered_chat_model(_build_chat_model(model_id))
+
+
+def _build_chat_model(model_id: str) -> ChatModel:
provider, model = parse_model_id(model_id)
if provider == "gemini":
from codeverse.models.gemini import GeminiModel
diff --git a/harness/codeverse/orchestrator/budget.py b/harness/codeverse/orchestrator/budget.py
index e9ed6170..90733ef5 100644
--- a/harness/codeverse/orchestrator/budget.py
+++ b/harness/codeverse/orchestrator/budget.py
@@ -4,6 +4,17 @@ Every model / agent call yields a ``Usage``; the track charges it here. When
either **hard** ceiling is crossed ``BudgetExceeded`` is raised so the round
loop can stop cleanly (status ``budget``) instead of silently overspending.
+**One door for every dollar.** ``charge`` and ``add`` are thin wrappers over
+:meth:`BudgetGuard.spend`, which is the only place money enters the run: it
+accumulates the ``Usage``, buckets it by stage and by round, appends one priced
+row to the run's cost ledger (``<run>/cost_ledger.jsonl``,
+``codeverse.cost.record_call``) and then — for ``charge`` — enforces the
+ceilings. The audit (docs/COST.md §6) found $4.80 of real spend that never
+reached ``record.total_usage``: rounds the budget cut after the work was done,
+retried ``.a2`` agent sessions and post-hoc texture passes. Anything that goes
+through ``spend`` is visible to the guard, to the record and to the ledger, so
+that class of blindness cannot come back.
+
On top of that a **soft** sub-budget (``soft_fraction`` of the hard ceilings)
lets a track see the wall coming: ``soft_ok()`` / ``soft_exceeded()`` say
"the baseline has used its share — degrade now (fewer assets, single-shot
@@ -15,11 +26,19 @@ build + render + judge) instead of dying with no score at all.
from __future__ import annotations
+import logging
import threading
import time
+from pathlib import Path
+from typing import Any
from codeverse.contracts.common import Budget, Usage
+log = logging.getLogger(__name__)
+
+#: stage label used when a caller does not say where the money went
+OTHER_STAGE = "other"
+
class BudgetExceeded(RuntimeError):
"""Raised when a run crosses its USD or wall-clock ceiling."""
@@ -38,7 +57,8 @@ class BudgetGuard:
``BudgetExceeded`` when ``max_usd`` or ``max_minutes`` is exceeded.
"""
- def __init__(self, budget: Budget, start_time: float | None = None, *, soft_fraction: float = 1.0):
+ def __init__(self, budget: Budget, start_time: float | None = None, *, soft_fraction: float = 1.0,
+ run: str = "", ledger: str | Path | Any | None = None):
self.budget = budget
self.start_time = time.time() if start_time is None else start_time
self.spent = Usage()
@@ -49,24 +69,87 @@ class BudgetGuard:
#: one-off extension of the HARD ceilings (finalise/salvage headroom)
self.grace_usd = 0.0
self.grace_minutes = 0.0
+ #: run slug + ledger sink for the per-call rows (None = do not write a ledger)
+ self.run = run
+ self.ledger = ledger
+ #: stage -> USD and round -> {stage -> USD}, so a round can report what it burned
+ self.by_stage: dict[str, float] = {}
+ self.by_round: dict[int, dict[str, float]] = {}
# ----------------------------------------------------------------- accounting
- def charge(self, usage: Usage | None) -> None:
- """Add ``usage`` to the running total, then enforce the ceilings."""
- self.add(usage)
- self.check()
+ def spend(
+ self,
+ usage: Usage | None,
+ *,
+ stage: str = OTHER_STAGE,
+ role: str | None = None,
+ label: str = "",
+ round_index: int | None = None,
+ outcome: str = "ok",
+ enforce: bool = True,
+ ) -> None:
+ """THE door every dollar goes through.
+
+ Accumulates ``usage``, buckets it by ``stage`` (and by round), appends a
+ priced row to the run's cost ledger, and — when ``enforce`` — raises
+ ``BudgetExceeded`` if a hard ceiling is now crossed. Accounting never
+ fails a run: a broken ledger is logged, not propagated."""
+ if usage is None:
+ return
+ with self._lock:
+ self.spent = self.spent + usage
+ self.calls += 1
+ key = stage or OTHER_STAGE
+ self.by_stage[key] = self.by_stage.get(key, 0.0) + float(usage.cost_usd)
+ if round_index is not None:
+ per = self.by_round.setdefault(int(round_index), {})
+ per[key] = per.get(key, 0.0) + float(usage.cost_usd)
+ self._ledger_row(usage, stage=key, role=role, label=label, round_index=round_index, outcome=outcome)
+ if enforce:
+ self.check()
- def add(self, usage: Usage | None) -> None:
- """Accumulate ``usage`` WITHOUT enforcing the ceilings.
+ def charge(self, usage: Usage | None, **kw: Any) -> None:
+ """Account for ``usage``, then enforce the ceilings (see :meth:`spend`)."""
+ kw.setdefault("enforce", True)
+ self.spend(usage, **kw)
+
+ def add(self, usage: Usage | None, **kw: Any) -> None:
+ """Account for ``usage`` WITHOUT enforcing the ceilings.
For work that is already done and persisted (a completed judge verdict,
a pairwise tie-break, the texture pass): the money is spent either way,
and raising here would throw away a finished, paid-for result. The
- round loop stops at its next ``ok()`` check instead."""
- if usage is not None:
- with self._lock:
- self.spent = self.spent + usage
- self.calls += 1
+ round loop stops at its next ``ok()`` check instead. The dollar is
+ still bucketed and still written to the ledger — "not enforced" never
+ means "not seen"."""
+ kw["enforce"] = False
+ self.spend(usage, **kw)
+
+ def _ledger_row(self, usage: Usage, *, stage: str, role: str | None, label: str,
+ round_index: int | None, outcome: str) -> None:
+ if self.ledger is None:
+ return
+ try:
+ from codeverse.cost import per_call_metering, record_call
+
+ # When the run is metered call by call (codeverse.cost.instrument wraps every
+ # ChatModel / CodingAgent), those rows already ARE this dollar with per-call
+ # tokens, cache hits and latency — one aggregate row on top would double count.
+ if per_call_metering():
+ return
+ record_call(usage, run=self.run, stage=stage, role=role, round=round_index, label=label,
+ backend=usage.backend, model=usage.model, outcome=outcome, ledger=self.ledger)
+ except Exception as e: # noqa: BLE001 — accounting must never fail a run
+ log.debug("cost ledger row failed (%s): %s", self.ledger, e)
+
+ def mark(self) -> Usage:
+ """Snapshot of the running total — diff it with :func:`usage_delta` to see
+ what a round burned even when the round itself raised half-way."""
+ return self.spent.model_copy(deep=True)
+
+ def round_costs(self, round_index: int) -> dict[str, float]:
+ """``{stage: USD}`` charged against one round index (empty when none)."""
+ return dict(self.by_round.get(int(round_index), {}))
def elapsed_minutes(self) -> float:
return (time.time() - self.start_time) / 60.0
@@ -162,3 +245,23 @@ class BudgetGuard:
"input_tokens": self.spent.input_tokens,
"output_tokens": self.spent.output_tokens,
}
+
+ def stage_summary(self) -> dict[str, float]:
+ """``{stage: USD}`` over the whole run (what the ``cost.round`` events add up to)."""
+ return {k: round(v, 6) for k, v in sorted(self.by_stage.items(), key=lambda kv: -kv[1])}
+
+
+def usage_delta(after: Usage, before: Usage) -> Usage:
+ """``after - before`` field by field (never negative). Used to report what a
+ round burned when it raised before it could fold its own usage together."""
+ return Usage(
+ backend=after.backend or before.backend,
+ model=after.model or before.model,
+ input_tokens=max(0, after.input_tokens - before.input_tokens),
+ output_tokens=max(0, after.output_tokens - before.output_tokens),
+ cached_tokens=max(0, after.cached_tokens - before.cached_tokens),
+ thoughts_tokens=max(0, after.thoughts_tokens - before.thoughts_tokens),
+ tool_calls=max(0, after.tool_calls - before.tool_calls),
+ cost_usd=max(0.0, after.cost_usd - before.cost_usd),
+ latency_ms=max(0, after.latency_ms - before.latency_ms),
+ )
diff --git a/harness/codeverse/orchestrator/rounds.py b/harness/codeverse/orchestrator/rounds.py
index ee457ac3..57bc9f85 100644
--- a/harness/codeverse/orchestrator/rounds.py
+++ b/harness/codeverse/orchestrator/rounds.py
@@ -1,11 +1,28 @@
"""Round loop policy: when to stop, which round is best, what to refine next.
All thresholds live in ``RoundPolicy`` (one calibrated object, no scattered
-ifs). ``StopPolicy.decide`` is a pure function of the round history;
+ifs). ``StopPolicy.evaluate`` is a pure function of the round history;
``BestSelector.pick`` is a pure function of the rounds; ``build_refine_
instructions`` compiles gate errors + failed acceptance + the judge's
improvement plan into typed ``RefineTask``s, and ``plan_parallel_groups``
decides which tasks may run concurrently from file ownership.
+
+**Two money stops** were added from the cost audit (docs/COST.md §5), both
+measured against the judge's own noise rather than a magic constant — the
+per-model σ table lives in ONE place, ``codeverse.cost.routing.JUDGE_NOISE``
+(pro 0.030, flash 0.083), and is read here through :func:`judge_sigma`:
+
+* ``regression`` — a refine round that scored more than 1 σ *below* the best is
+ not evidence that another round of the same shape will do better. The first
+ such round switches strategy (one whole-artifact rewrite instead of the same
+ per-part task set); a second one stops the run. 19 rounds in the audit scored
+ below the best and were thrown away ($14.23); this rule governs what the loop
+ is allowed to do *after* one of them (measured replay: 4 rounds re-shaped, and
+ a hard stop instead would have killed the recovery round that made
+ ``furn_hard_rolltop_desk`` pass).
+* ``diminishing_returns`` — from r03 on, a round only runs when the previous
+ round gained more than 1.5 σ and the run is still below target. r03 cost
+ $33.49 per score point in the audit, against $4.93 for r01.
"""
from __future__ import annotations
@@ -26,7 +43,42 @@ from codeverse.judges.metrics import best_index
log = logging.getLogger(__name__)
-StopReason = Literal["pass", "plateau", "budget", "continue", "max_rounds", "judge_unavailable"]
+StopReason = Literal["pass", "plateau", "budget", "continue", "max_rounds", "judge_unavailable",
+ "regression", "diminishing_returns"]
+
+#: what the next round should look like when the loop continues
+Strategy = Literal["same", "switch"]
+
+#: ``RoundRecord.kind`` of a round that already changed strategy after a regression
+REWRITE_KIND = "rewrite"
+
+#: σ for a judge that is not in the measured table: the default judge's
+#: (``gemini-3.1-pro-preview``, ``Settings.default_judge``), because that is what
+#: an unnamed judge almost always is. Calibrate a new judge
+#: (``python -m codeverse.judges.calibration``) and add it to JUDGE_NOISE rather
+#: than tuning the multipliers around it.
+DEFAULT_JUDGE_SIGMA = 0.030
+
+
+def judge_sigma(judge_model: str = "", *, default: float = DEFAULT_JUDGE_SIGMA) -> float:
+ """Measured score noise (std) of one judge model.
+
+ The numbers come from ``judges/calibration.py`` (n=3 on the e2e rounds) and
+ are tabulated ONCE in :data:`codeverse.cost.routing.JUDGE_NOISE`; this is
+ the only reader in the orchestrator. Accepts a bare model name or a full
+ backend id (``gemini:gemini-3.1-pro-preview``)."""
+ name = (judge_model or "").strip().split(":")[-1]
+ if not name:
+ return default
+ try:
+ from codeverse.cost.routing import JUDGE_NOISE
+ except Exception as e: # noqa: BLE001 — the cost package is optional at import time
+ log.debug("judge noise table unavailable: %s", e)
+ return default
+ hit = JUDGE_NOISE.get(name)
+ if hit is None: # version suffixes: gemini-3.7-flash-002 → gemini-3.7-flash
+ hit = next((v for k, v in JUDGE_NOISE.items() if name.startswith(k)), None)
+ return float(hit[0]) if hit else default
@dataclass(frozen=True)
@@ -45,6 +97,19 @@ class RoundPolicy:
pairwise_margin: float = 0.03 # |Δscore| below this = judge noise → pairwise tie-break
pairwise_min_confidence: float = 0.6 # new round replaces best only when pairwise is this sure
judge_samples: int = 1 # VLM judge samples per round (flash: std ≈ 0.001 between samples → 1 is enough)
+ # ---- money stops (docs/COST.md §5); thresholds are multiples of the judge's measured σ
+ judge_model: str = "" # judge backend id → σ via judge_sigma(); "" = the default judge's σ
+ regression_sigma: float = 1.0 # a round below (best − this × σ) regressed: change shape or stop
+ regression_allow_switch: bool = True # False = a regression stops the run outright
+ marginal_sigma: float = 1.5 # from marginal_from_round on, the last gain must beat this × σ
+ marginal_from_round: int = 3 # first refine round index the marginal-value test applies to
+ # ---- agent session shape. A 28-turn cap was TESTED AND REJECTED: a controlled
+ # A/B on one prompt (n=3 per arm, same generator/judge/budget) cost $1.381 mean at
+ # 0.479 mean score capped, against $1.360 at 0.684 uncapped — no saving and −0.205
+ # score (docs/COST.md §17). 0 = no policy cap: the session runs under the backend's
+ # own AgentJob.max_turns. Callers who want one set it explicitly (cost profiles do).
+ agent_max_turns: int = 0 # model turns one generation session may take (0 = uncapped)
+ agent_wrapup_turns: int = 6 # turns granted to land a final build + summary when a cap IS set
def with_candidates(self, n: int | None) -> RoundPolicy:
"""Copy with ``n_candidates`` set (``None`` → unchanged)."""
@@ -52,6 +117,43 @@ class RoundPolicy:
return self
return replace(self, n_candidates=max(1, int(n)))
+ def with_judge(self, judge_model: str | None) -> RoundPolicy:
+ """Copy that knows which judge scores the rounds (its σ sizes both money stops)."""
+ if not judge_model or judge_model == self.judge_model:
+ return self
+ return replace(self, judge_model=judge_model)
+
+ @property
+ def sigma(self) -> float:
+ """Measured judge noise for this run's judge (see :func:`judge_sigma`)."""
+ return judge_sigma(self.judge_model)
+
+ @property
+ def regression_delta(self) -> float:
+ """How far below the best a round must score to count as a regression."""
+ return self.regression_sigma * self.sigma
+
+ @property
+ def marginal_delta(self) -> float:
+ """Gain the previous round must have produced for r03+ to be worth starting."""
+ return self.marginal_sigma * self.sigma
+
+
+@dataclass(frozen=True)
+class StopDecision:
+ """``StopPolicy``'s answer: stop (and why), or continue (and in what shape)."""
+
+ reason: StopReason
+ strategy: Strategy = "same"
+ detail: str = ""
+
+ @property
+ def stop(self) -> bool:
+ return self.reason != "continue"
+
+ def event(self) -> dict[str, object]:
+ return {"reason": self.reason, "strategy": self.strategy, "detail": self.detail}
+
class StopPolicy:
"""Decide whether the loop goes on. ``history`` = rounds so far (baseline first)."""
@@ -60,19 +162,92 @@ class StopPolicy:
self.policy = policy
def decide(self, history: Sequence[RoundRecord], *, budget_ok: bool = True) -> StopReason:
+ """The stop reason only (``evaluate`` also says what shape the next round takes)."""
+ return self.evaluate(history, budget_ok=budget_ok).reason
+
+ def evaluate(self, history: Sequence[RoundRecord], *, budget_ok: bool = True) -> StopDecision:
if not budget_ok:
- return "budget"
+ return StopDecision("budget")
if not history:
- return "continue"
+ return StopDecision("continue")
last = history[-1]
if last.judgment is not None and last.judgment.passed and last.judgment.overall >= self.policy.target:
- return "pass"
+ return StopDecision("pass")
refine_rounds = len(history) - 1
if refine_rounds >= self.policy.max_rounds:
- return "max_rounds"
+ return StopDecision("max_rounds")
+ # Order is the economics. A regression whose strategy switch is already spent
+ # is the most specific answer there is, so it comes first. Otherwise the
+ # marginal-value test decides late rounds (at r03 no *shape* is worth $0.5),
+ # and only before that does a first regression buy one change of shape.
+ # Plateau is the fallback: the same money statement, less precisely.
+ exhausted = self._regression(history, switch=False)
+ if exhausted is not None:
+ return exhausted
+ marginal = self._diminishing(history)
+ if marginal is not None:
+ return marginal
+ regressed = self._regression(history)
+ if regressed is not None:
+ return regressed
if self._plateaued(history):
- return "plateau"
- return "continue"
+ return StopDecision("plateau")
+ return StopDecision("continue")
+
+ # ---------------------------------------------------------------- money stops
+ def _regression(self, history: Sequence[RoundRecord], *, switch: bool = True) -> StopDecision | None:
+ """The last round scored below the best by more than the judge's own noise.
+
+ Another round of the SAME shape is not justified by that evidence: the
+ first regression buys one change of strategy (a whole-artifact rewrite),
+ a second one — or one after a rewrite already failed to recover — stops
+ the run. ``switch=False`` asks only "is the switch already spent?", which
+ is the question that outranks every other stop."""
+ last = history[-1]
+ score, best_before = last.score, best_score(history[:-1])
+ if score is None or best_before is None:
+ return None
+ drop = best_before - score
+ if drop <= self.policy.regression_delta:
+ return None
+ detail = (f"r{last.index:02d} scored {score:.3f} vs best {best_before:.3f} "
+ f"({-drop:+.3f}, judge σ {self.policy.sigma:.3f})")
+ if not self.policy.regression_allow_switch:
+ return StopDecision("regression", detail=detail)
+ if last.kind == REWRITE_KIND or self._regressions(history) > 1:
+ return StopDecision("regression", detail=detail + " — a changed strategy did not recover it")
+ return StopDecision("continue", strategy="switch", detail=detail) if switch else None
+
+ def _diminishing(self, history: Sequence[RoundRecord]) -> StopDecision | None:
+ """From ``marginal_from_round`` on, only run when the last round actually
+ bought something (> ``marginal_sigma`` × σ) and the target is still open."""
+ nxt = len(history)
+ if nxt < self.policy.marginal_from_round:
+ return None
+ gain = last_gain(history)
+ best = best_score(history)
+ need = self.policy.marginal_delta
+ if gain is None:
+ return None
+ if gain <= need:
+ return StopDecision("diminishing_returns",
+ detail=f"r{nxt:02d} not started: last gain {gain:+.3f} ≤ {need:.3f} "
+ f"(1.5 × judge σ {self.policy.sigma:.3f})")
+ if best is not None and best >= self.policy.target:
+ return StopDecision("diminishing_returns",
+ detail=f"r{nxt:02d} not started: best {best:.3f} already at target "
+ f"{self.policy.target:.2f}")
+ return None
+
+ @staticmethod
+ def _regressions(history: Sequence[RoundRecord]) -> int:
+ """How many scored rounds ended below the best of everything before them."""
+ n = 0
+ for i in range(1, len(history)):
+ s, before = history[i].score, best_score(history[:i])
+ if s is not None and before is not None and s < before:
+ n += 1
+ return n
def _plateaued(self, history: Sequence[RoundRecord]) -> bool:
"""True when the last ``plateau_window`` scored rounds did not raise the best
@@ -87,25 +262,54 @@ class StopPolicy:
return (after - before) < self.policy.min_delta
+def best_score(history: Sequence[RoundRecord]) -> float | None:
+ """Best judged score in ``history`` (``None`` when nothing was scored)."""
+ scores = [r.score for r in history if r.score is not None]
+ return max(scores) if scores else None
+
+
+def last_gain(history: Sequence[RoundRecord]) -> float | None:
+ """How much the LAST round moved the best score (``None`` when unscored).
+
+ Negative when it regressed; 0.0 when it changed nothing measurable."""
+ if len(history) < 2 or history[-1].score is None:
+ return None
+ before = best_score(history[:-1])
+ if before is None:
+ return None
+ return float(history[-1].score) - before
+
+
class BestSelector:
- """Best round = highest score, then fewer gate errors, then earlier.
+ """Best round = highest score, then fewer gate errors, then later.
- ``codeverse.judges.metrics.best_index`` owns the ranking rule; rounds
- without any score fall back to the last successfully built one."""
+ ``codeverse.judges.metrics.best_index`` owns the ranking rule. A round that
+ did not build is never picked (it has no score, and delivering code that does
+ not run is never an improvement); when NO round has a score the fallback is
+ the built round with the fewest gate errors — later on a tie — so a round that
+ broke the gates cannot displace a clean earlier artifact just by being last."""
def pick(self, rounds: Sequence[RoundRecord]) -> int | None:
- scored = [(i, r) for i, r in enumerate(rounds) if r.score is not None]
+ scored = [(i, r) for i, r in enumerate(rounds) if r.score is not None and not _build_failed(r)]
if scored:
k = best_index([(float(r.score), gate_error_count(r)) for _, r in scored]) # type: ignore[arg-type]
return scored[int(k)][0]
built = [i for i, r in enumerate(rounds) if r.build is not None and r.build.ok]
- return built[-1] if built else None
+ if not built:
+ return None
+ return min(built, key=lambda i: (gate_error_count(rounds[i]), -i))
def gate_error_count(r: RoundRecord) -> int:
return sum(len(g.errors) for g in r.gates)
+def _build_failed(r: RoundRecord) -> bool:
+ """The round is KNOWN not to build (a record with no build at all is not a failure:
+ resumed/rejudged records may carry a score without one)."""
+ return r.build is not None and not r.build.ok
+
+
# ----------------------------------------------------------------------------- refine tasks
class RefineTask(BaseModel):
"""One targeted change request compiled for the next round."""
diff --git a/harness/codeverse/spatial/tools_texture.py b/harness/codeverse/spatial/tools_texture.py
index e3f76c6b..70f1e66d 100644
--- a/harness/codeverse/spatial/tools_texture.py
+++ b/harness/codeverse/spatial/tools_texture.py
@@ -46,6 +46,15 @@ def _spec_plan(ctx: ToolContext) -> tuple[Spec, StaticPlan]:
def texture_pass_tool(ctx: ToolContext, args: TexturePassArgs) -> Observation:
glb = glb_path(ctx)
spec, plan = _spec_plan(ctx)
+ texture_requested = lazy("codeverse.texturing.run", "texture_requested")
+ if not texture_requested(spec):
+ # ONE owner for "does this run texture?" (codeverse.texturing.run.texture_requested).
+ # The tool is registered for every object track, so without this an agent could —
+ # and did — buy a texture pass in a run whose spec says texture: false.
+ raise ToolUsageError(
+ "this run did not ask for texturing (spec.options.texture is false), so the "
+ "texture pass is off; finish the geometry instead",
+ "3dcv make ... --texture # or `3dcv texture pass <slug>` after the run")
texture_pass = lazy("codeverse.texturing.run", "texture_pass")
services = ctx.extra.get("texture_services") # the ONE injection point (TextureServices)
rep = texture_pass(ctx.workspace, spec, plan, model_id=args.model or spec.backends.planner,
diff --git a/harness/codeverse/texturing/run.py b/harness/codeverse/texturing/run.py
index cf968f6f..44856938 100644
--- a/harness/codeverse/texturing/run.py
+++ b/harness/codeverse/texturing/run.py
@@ -106,6 +106,24 @@ def _measure(glb: Path) -> Measurement | None:
return None
+#: the ONE owner of "does this run texture?". Three call sites used to answer it
+#: independently — ``tracks.lifecycle.finalise`` (spec.options.texture / a "texture" tag /
+#: ctx.extra), and the ``texture_pass`` spatial tool, which the coding agent could call in
+#: any run because the tool is registered for every object track. That is how a
+#: ``texture: false`` run still paid for a texture pass (docs/COST.md §15: the quality run's
+#: ledger was +9.5% over record.total_usage because the pass ran twice, once from inside a
+#: round-2 agent session). Both now ask this function.
+def texture_requested(spec: Spec) -> bool:
+ """True when the run asked for the derived texture pass.
+
+ ``Spec.options.texture`` is the switch; the legacy ``texture`` tag is still
+ honoured because recorded specs carry it. Nothing else may turn texturing
+ on — an agent calling the ``texture_pass`` tool in a run that did not ask for
+ it is refused, and ``3dcv texture pass <slug>`` is an explicit user
+ instruction that does not go through here at all."""
+ return bool(spec.options.texture or "texture" in (spec.tags or []))
+
+
def texture_pass(
ws: Workspace,
spec: Spec,
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index ff87c0d7..653adade 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -11,15 +11,36 @@
Neither path commits; the round commits once after all (possibly parallel)
tasks finished. Neither path builds; see ``tracks/repair.py``.
+
+**Turn budget — off by default, measured.** The audit's turn-≥20 share (39.3%
+of the agent bill, docs/COST.md §4) looked like $14 of waste, so wave 2 capped
+every session at 28 turns. A controlled A/B rejected it — same prompt, n=3 per
+arm: capped $1.381 / **0.479**, uncapped $1.360 / **0.684** (docs/COST.md §17).
+A session cut off mid-task leaves work the next round pays for again.
+
+So there is **no default cap** (``DEFAULT_AGENT_MAX_TURNS`` and
+``RoundPolicy.agent_max_turns`` are 0 → the backend's own ``AgentJob.max_turns``).
+A caller may still set one (``GenerationTask.max_turns`` > ``generate(max_turns=…)``
+> ``CV3D_AGENT_MAX_TURNS`` > ``Settings.limits.agent_max_turns``, which a cost
+profile sets), and a cap that IS set stays **graceful**: the session is not killed
+but asked, in a short wrap-up (``agent_wrapup_turns``), for one last build + summary.
+
+**Every dollar is charged.** Both strategies spend through
+``BudgetGuard.spend`` with their stage / label / round, so a retried session
+(``<label>.a2``) and a wrap-up session are visible to the guard, the record and
+the ledger even when the *next* attempt then raises.
"""
from __future__ import annotations
+import json
import logging
+import os
import subprocess
import threading
import time
from collections.abc import Callable
+from pathlib import Path
from typing import Any
from pydantic import BaseModel, Field
@@ -42,6 +63,53 @@ log = logging.getLogger(__name__)
SINGLE_SHOT_PREFIX = "single-shot:"
+#: default cap on model turns per agent session: 0 = none, the session runs under the
+#: backend's own ``AgentJob.max_turns``. A 28-turn default was measured and rejected
+#: (A/B: no saving, −0.205 score) — see the module docstring and docs/COST.md §17.
+DEFAULT_AGENT_MAX_TURNS = 0
+#: turns granted to the wrap-up session that lands a final build + summary when a cap IS set
+DEFAULT_WRAPUP_TURNS = 6
+
+WRAPUP_PROMPT = (
+ "You have reached the turn budget for this task, so this is your LAST session on it.\n"
+ "Do NOT start new work and do NOT begin any refactor.\n"
+ "1. Make sure every file you edited is written to disk and self-consistent.\n"
+ "2. Run the build tool ONCE. If it fails, make the smallest edit that makes it build "
+ "(or revert your last incomplete edit) and build once more.\n"
+ "3. Reply with a short summary: what you changed, what still does not match the task, "
+ "and the single next step you would take.\n\n"
+ "The task you were working on, for reference:\n"
+)
+
+
+def agent_max_turns(default: int = DEFAULT_AGENT_MAX_TURNS) -> int:
+ """The turn cap a machine/profile asks for: ``CV3D_AGENT_MAX_TURNS`` >
+ ``Settings.limits.agent_max_turns`` (what a cost profile sets) > ``default``.
+
+ ``0`` means *no cap*: the caller leaves ``AgentJob.max_turns`` at the backend's
+ own default. That is the measured default (docs/COST.md §17)."""
+ raw = os.environ.get("CV3D_AGENT_MAX_TURNS", "").strip()
+ if raw.isdigit() and int(raw) > 0:
+ return int(raw)
+ try:
+ from codeverse.config import get_settings
+
+ configured = int(getattr(get_settings().limits, "agent_max_turns", 0) or 0)
+ except Exception as e: # noqa: BLE001 — settings must never break a generation
+ log.debug("agent turn cap fell back to the default: %s", e)
+ configured = 0
+ return max(0, configured or int(default))
+
+
+def turn_capped(res: Any) -> bool:
+ """True when the agent stopped because it ran out of turns (not money/time).
+
+ api-agent says ``max_turns (N) reached``; claude-code reports the subtype
+ ``error_max_turns`` — both surface as ``exit_reason="budget"``."""
+ if getattr(res, "exit_reason", "") != "budget":
+ return False
+ return any("max_turns" in str(e) for e in (getattr(res, "errors", None) or []))
+
class GenerationTask(BaseModel):
"""One unit of code-writing work (baseline, refine group, asset, zone, repair)."""
@@ -57,6 +125,7 @@ class GenerationTask(BaseModel):
thinking: str = "medium"
max_output_tokens: int = 32000
timeout_s: int | None = None
+ max_turns: int = Field(default=0, description="model turns for an agent session (0 = the harness default)")
write_roots: list[str] = Field(default_factory=lambda: ["src", "public"])
@@ -69,6 +138,9 @@ class GenerationResult(BaseModel):
text: str = Field(default="", description="model/agent final text (truncated)")
transcript_path: str = ""
label: str = ""
+ turns: int = Field(default=0, description="turns the agent session(s) took, as the backend counts them")
+ sessions: int = Field(default=0, description="agent sessions run for this task (retry / wrap-up count)")
+ turn_capped: bool = Field(default=False, description="a session hit the turn budget and was wrapped up")
# ----------------------------------------------------------------------------- git helpers
@@ -138,12 +210,13 @@ def generate_files(
temperature=task.temperature,
thinking=task.thinking, # type: ignore[arg-type]
max_output_tokens=task.max_output_tokens,
- label=task.label,
+ # the round tag is the cost ledger's only way to attribute a single-shot call:
+ # it has no agent session to inherit an ambient round from (codeverse.cost.context)
+ label=f"{task.label}:r{task.round:02d}",
)
resp = model.generate(req)
usage = resp.usage
- if budget is not None:
- budget.charge(usage)
+ _charge(budget, usage, task=task, label=task.label, outcome=str(resp.finish_reason or "ok"))
if _is_truncated(resp):
# cut off by max_output_tokens: the envelope is unterminated — one retry with
# a doubled budget beats writing a half-file and burning repair attempts on it
@@ -152,8 +225,7 @@ def generate_files(
req = req.model_copy(update={"max_output_tokens": min(task.max_output_tokens * 2, 65536)})
resp = model.generate(req)
usage = usage + resp.usage
- if budget is not None:
- budget.charge(resp.usage)
+ _charge(budget, resp.usage, task=task, label=f"{task.label}.retry", outcome="truncated")
traj = ws.trajectory_dir(task.label.replace("/", "_"), task.round)
(traj / "prompt.md").write_text(f"# system\n{system}\n\n# user\n{task.prompt}\n")
(traj / "response.md").write_text(resp.text or "")
@@ -185,6 +257,33 @@ def generate_files(
text=(resp.text or "")[:2000], transcript_path=str(traj / "response.md"), label=task.label)
+#: task kinds whose money belongs to a differently-named stage of the cost vocabulary
+_STAGE_FOR_KIND = {"rebuild": "repair", "asset_fix": "assets", "asset": "assets",
+ "zone": "zones", "generate": "baseline", "compose": "assemble"}
+
+
+def task_stage(task: GenerationTask) -> str:
+ """The cost stage one generation task spends in (a ``codeverse.cost.types.Stage``
+ value when we can name one, else the label — which ``record_call`` maps through
+ ``stage_for_label`` (``asset_koi`` → assets, ``r00_baseline_repair1`` → repair))."""
+ kind = task.kind or ""
+ stage = _STAGE_FOR_KIND.get(kind, kind)
+ try:
+ from codeverse.cost.types import Stage
+
+ return str(Stage(stage))
+ except Exception: # noqa: BLE001 — unknown kind (or no cost package): let the label decide
+ return task.label or stage or "other"
+
+
+def _charge(budget: Any | None, usage: Usage, *, task: GenerationTask, label: str, outcome: str = "ok") -> None:
+ """Spend through the guard with the stage/label/round this task belongs to."""
+ if budget is None:
+ return
+ budget.charge(usage, stage=task_stage(task), role="generator", label=label,
+ round_index=task.round, outcome=outcome)
+
+
_TRUNCATED_FINISH = {"max_tokens", "max_output_tokens", "length"}
@@ -201,10 +300,23 @@ def run_agent_task(
budget: Any | None = None,
events: Any | None = None,
retry_silent_bail: bool = True,
+ max_turns: int = 0,
+ wrapup_turns: int = DEFAULT_WRAPUP_TURNS,
) -> GenerationResult:
- """CodingAgent path. ``ok`` = the agent changed files (envelope-vs-disk truth)."""
+ """CodingAgent path. ``ok`` = the agent changed files (envelope-vs-disk truth).
+
+ A turn cap is applied only when a caller asks for one (``task.max_turns`` >
+ ``max_turns`` > ``CV3D_AGENT_MAX_TURNS`` > ``Settings.limits.agent_max_turns``
+ > :data:`DEFAULT_AGENT_MAX_TURNS`, which is 0 = the backend's own
+ ``AgentJob.max_turns``; a 28-turn default was measured and rejected, see the
+ module docstring). Hitting whatever cap is in force is not a failure: one
+ short wrap-up session is asked for a final build and a summary, so the money
+ already spent leaves a buildable workspace instead of a session that was cut
+ off mid-edit. Every session — first attempt, silent-bail retry, wrap-up — is
+ charged to ``budget`` as it ends, so nothing is lost when a later attempt raises."""
before = ws.head()
timeout = task.timeout_s or (settings.limits.agent_timeout_s if settings is not None else 1800)
+ turns_cap = task.max_turns or max_turns or agent_max_turns() # 0 = leave AgentJob's own default
# typed job context honoured by every CodingAgent: round → trajectory dir + ToolContext,
# language/track → spatial tool filtering, files_hint → per-session attribution of
# files_changed when tasks run concurrently in ONE workspace (see agents/cli_common).
@@ -224,30 +336,120 @@ def run_agent_task(
model=getattr(agent, "model", ""), label=task.label, timeout_s=timeout,
spatial_tools=True, write_roots=task.write_roots,
round=task.round, kind=task.kind, language=language, track=track,
- files_hint=list(task.files_hint), extra=extra)
- res: AgentResult = agent.run(job)
- usage = res.usage
- if budget is not None:
- budget.charge(res.usage)
+ files_hint=list(task.files_hint), extra=extra,
+ **({"max_turns": turns_cap} if turns_cap > 0 else {}))
+ acc = _SessionAcc(task=task, budget=budget)
+ res = acc.run(agent, job)
changes = res.files_changed or _attributed_fallback(ws, task, before)
- if retry_silent_bail and not changes and res.exit_reason not in ("timeout", "budget"):
+
+ if turn_capped(res):
+ # the turns are gone, the money is not wasted: ask for a landing, not more work
+ if events is not None:
+ # NB: not ``cost_usd`` — an event carrying that key is priced as its own
+ # call by ``cost.reconstruct``, and this money is already in the session row.
+ events.emit("generate.turn_cap", label=task.label, round=task.round, max_turns=job.max_turns,
+ turns=acc.turns, session_usd=round(acc.usage.cost_usd, 4))
+ wrap = job.model_copy(update={"prompt": WRAPUP_PROMPT + task.prompt,
+ "max_turns": max(2, int(wrapup_turns))})
+ res = acc.run(agent, wrap, wrapup=True, optional=True) or res
+ changes = res.files_changed or changes or _attributed_fallback(ws, task, before)
+
+ if retry_silent_bail and not changes and not acc.wrapped and res.exit_reason not in ("timeout", "budget"):
if events is not None:
events.emit("generate.silent_bail", label=task.label, exit_reason=res.exit_reason)
job2 = job.model_copy(update={"prompt": (
"Your previous attempt ended WITHOUT writing any file. You must create/edit the files "
"described below and run the build tool before finishing.\n\n" + task.prompt)})
- res = agent.run(job2)
- usage = usage + res.usage
- if budget is not None:
- budget.charge(res.usage)
+ res = acc.run(agent, job2, optional=True) or res
changes = res.files_changed or _attributed_fallback(ws, task, before)
+
notes = f"exit={res.exit_reason}" + (f"; errors={res.errors[:3]}" if res.errors else "")
+ notes += ("; " + "; ".join(acc.errors)) if acc.errors else ""
+ if acc.wrapped:
+ notes += f"; turn cap {job.max_turns} reached → wrap-up session"
if events is not None:
events.emit("generate.done", label=task.label, strategy=getattr(agent, "kind", "agent"),
files=[c.path for c in changes], ok=bool(changes), exit_reason=res.exit_reason,
- cost_usd=round(usage.cost_usd, 4))
- return GenerationResult(ok=bool(changes), usage=usage, files_changed=changes, notes=notes,
- text=(res.text or "")[:2000], transcript_path=res.transcript_path, label=task.label)
+ turns=acc.turns, sessions=acc.sessions, turn_capped=acc.wrapped,
+ cost_usd=round(acc.usage.cost_usd, 4))
+ return GenerationResult(ok=bool(changes), usage=acc.usage, files_changed=changes, notes=notes,
+ text=(res.text or "")[:2000], transcript_path=res.transcript_path, label=task.label,
+ turns=acc.turns, sessions=acc.sessions, turn_capped=acc.wrapped)
+
+
+class _SessionAcc:
+ """Runs the agent sessions of ONE task and keeps their accounting.
+
+ Each session is charged the moment it returns (label ``<task>``/``<task>.a2``
+ for the retry, ``<task>.wrapup`` for the landing session), so a crash in a
+ later session can never erase what an earlier one already spent — the ``.a2``
+ hole of docs/COST.md §6."""
+
+ def __init__(self, *, task: GenerationTask, budget: Any | None):
+ self.task = task
+ self.budget = budget
+ self.usage = Usage()
+ self.turns = 0
+ self.sessions = 0
+ self.wrapped = False
+ self.errors: list[str] = []
+
+ def run(self, agent: Any, job: AgentJob, *, wrapup: bool = False, optional: bool = False) -> AgentResult | None:
+ """One session. ``optional`` sessions (wrap-up, silent-bail retry) never
+ take the task down: a crash there leaves the earlier, already charged
+ session as the result instead of discarding money that was really spent."""
+ self.sessions += 1
+ self.wrapped = self.wrapped or wrapup
+ label = self.task.label + (".wrapup" if wrapup else (f".a{self.sessions}" if self.sessions > 1 else ""))
+ try:
+ res: AgentResult = agent.run(job)
+ except Exception as e: # noqa: BLE001 — a budget stop still propagates (see below)
+ self.sessions -= 1
+ if not optional or _is_budget_stop(e):
+ raise
+ log.warning("optional agent session %s failed: %s: %s", label, type(e).__name__, e)
+ self.errors.append(f"{label}: {type(e).__name__}: {e}")
+ return None
+ self.usage = self.usage + res.usage
+ self.turns += session_turns(res)
+ self.charge(res.usage, label=label, outcome=res.exit_reason or "ok")
+ return res
+
+ def charge(self, usage: Usage, *, label: str, outcome: str) -> None:
+ if self.budget is None or not (usage.cost_usd or usage.input_tokens or usage.output_tokens):
+ return
+ self.budget.charge(usage, stage=task_stage(self.task), role="generator",
+ label=label, round_index=self.task.round, outcome=outcome)
+
+
+def _is_budget_stop(e: BaseException) -> bool:
+ from codeverse.orchestrator.budget import BudgetExceeded
+
+ return isinstance(e, BudgetExceeded)
+
+
+def session_turns(res: Any) -> int:
+ """The session's turn count as the BACKEND reports it — the ``turns`` field
+ ``agents/cli_common.finish_session`` writes into ``result.json`` (0 when the
+ backend does not report one).
+
+ Backends count slightly differently (the api-agent reports transcript
+ messages, roughly two per model turn), so this is a size signal for the
+ ``cost.round`` event, not the number the turn cap is compared against —
+ that one is enforced inside the session by ``job.max_turns``."""
+ path = getattr(res, "transcript_path", "") or ""
+ if not path:
+ return 0
+ p = Path(path)
+ result = (p if p.is_dir() else p.parent) / "result.json"
+ try:
+ if result.is_file():
+ value = json.loads(result.read_text(errors="replace")).get("turns")
+ if isinstance(value, (int, float)):
+ return int(value)
+ except (OSError, ValueError) as e: # noqa: PERF203 - best effort telemetry
+ log.debug("session turns unreadable (%s): %s", result, e)
+ return 0
def _attributed_fallback(ws: Workspace, task: GenerationTask, before: str) -> list[FileChange]:
@@ -276,6 +478,8 @@ def generate(
events: Any | None = None,
model_factory: Callable[[str], Any] | None = None,
agent_factory: Callable[[str], Any] | None = None,
+ max_turns: int = 0,
+ wrapup_turns: int = DEFAULT_WRAPUP_TURNS,
) -> GenerationResult:
"""Dispatch to the single-shot or the coding-agent strategy by ``agent_id``.
@@ -291,7 +495,8 @@ def generate(
if agent is None:
factory = agent_factory or _default_agent_factory
agent = factory(agent_id)
- return run_agent_task(ws, agent=agent, task=task, settings=settings, budget=budget, events=events)
+ return run_agent_task(ws, agent=agent, task=task, settings=settings, budget=budget, events=events,
+ max_turns=max_turns, wrapup_turns=wrapup_turns)
def _default_model_factory(model_id: str) -> Any:
diff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.py
index 3b2a66ce..e9afd078 100644
--- a/harness/codeverse/tracks/lifecycle.py
+++ b/harness/codeverse/tracks/lifecycle.py
@@ -27,11 +27,15 @@ from codeverse.contracts.spec import Spec
from codeverse.events import EventLog
from codeverse.orchestrator.budget import BudgetExceeded, BudgetGuard
from codeverse.orchestrator.rounds import (
+ REWRITE_KIND,
BestSelector,
+ RefineTask,
RoundPolicy,
StopPolicy,
StopReason,
+ Strategy,
TaskGroup,
+ best_score,
build_refine_instructions,
plan_refine_groups,
)
@@ -67,9 +71,31 @@ log = logging.getLogger(__name__)
REFERENCE_RUBRIC = "reference_v1"
+#: fallback name of the run's live cost ledger when the cost package cannot place
+#: one itself (``flywheel/telemetry.py`` publishes it as ``telemetry/usage.jsonl``)
+LEDGER_NAME = "cost_ledger.jsonl"
+
+
+def run_ledger_path(ws: Workspace) -> Any:
+ """Where this run's priced per-call rows go.
+
+ ``codeverse.cost`` owns the run layout (``telemetry/cost.jsonl`` + the
+ ``cost_ledger.jsonl`` alias); fall back to the alias name if it cannot."""
+ try:
+ from codeverse.cost.ledger import open_run_ledger
+
+ return open_run_ledger(ws.root)
+ except Exception as e: # noqa: BLE001 — accounting must never fail a run
+ log.debug("cost ledger placement fell back to %s: %s", LEDGER_NAME, e)
+ return ws.root / LEDGER_NAME
+
+
_STATUS: dict[str, RunStatus] = {
"pass": RunStatus.PASSED, "plateau": RunStatus.PLATEAU, "max_rounds": RunStatus.PLATEAU,
"budget": RunStatus.BUDGET,
+ # the two money stops (orchestrator/rounds.py): the code is intact and the best
+ # round is delivered — the run stopped because another round was not worth buying.
+ "regression": RunStatus.PLATEAU, "diminishing_returns": RunStatus.PLATEAU,
# judge outage (degraded/crashed verdicts even after a retry): the code is intact
# and the best built round is delivered — a stop, not a failure.
"judge_unavailable": RunStatus.PLATEAU,
@@ -191,8 +217,14 @@ class BaseTrack:
raise NotImplementedError
# ------------------------------------------------------------------ refine scaffold
- def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord]) -> tuple[list[GenerationTask], list[str]]:
- """index → rebuild guard → instructions → groups → per-track ``_refine_task``."""
+ def refine_tasks(self, ctx: RunContext, last: RoundRecord, history: Sequence[RoundRecord],
+ *, strategy: Strategy = "same") -> tuple[list[GenerationTask], list[str]]:
+ """index → rebuild guard → instructions → groups → per-track ``_refine_task``.
+
+ ``strategy="switch"`` is what the loop asks for after a regression: the same
+ evidence, but ONE whole-artifact task instead of the per-part set that just
+ made the score worse (docs/COST.md §5 — repeating the shape is what the 19
+ regression rounds did)."""
index = len(history)
if last.build is None or not last.build.ok:
return [self._rebuild_task(ctx, last, index)], ["rebuild: previous round did not build"]
@@ -202,11 +234,15 @@ class BaseTrack:
extra=self.extra_refine_tasks(ctx, last))
if not tasks:
return [], []
- groups, parallel = plan_refine_groups(tasks, allow_fanout=self.allow_refine_fanout,
- parallel_min_tasks=ctx.policy.parallel_min_tasks)
+ if strategy == "switch":
+ tasks = [rewrite_task(last, best_score(history))] + tasks
+ groups, parallel = [TaskGroup(tasks=tasks, files=[])], False
+ else:
+ groups, parallel = plan_refine_groups(tasks, allow_fanout=self.allow_refine_fanout,
+ parallel_min_tasks=ctx.policy.parallel_min_tasks)
gen_tasks = [self._refine_task(ctx, g, last, index, parallel=parallel) for g in groups]
ctx.events.emit("refine.planned", round=index, n_tasks=len(tasks), n_groups=len(groups), parallel=parallel,
- targets=[g.targets for g in groups])
+ strategy=strategy, targets=[g.targets for g in groups])
return gen_tasks, [t.line() for t in tasks]
def _rebuild_task(self, ctx: RunContext, last: RoundRecord, index: int) -> GenerationTask:
@@ -268,12 +304,14 @@ class BaseTrack:
def build_context(self, spec: Spec, ws: Workspace, events: EventLog, state: RunState) -> RunContext:
settings = self._settings or get_settings()
runtime = self._runtime or self.services.runtime(spec.language)
- budget = BudgetGuard(spec.budget, soft_fraction=self.soft_budget_fraction)
+ budget = BudgetGuard(spec.budget, soft_fraction=self.soft_budget_fraction,
+ run=ws.root.name, ledger=run_ledger_path(ws))
spent = state.extra.get("spent_usage")
if spent:
budget.spent = Usage.model_validate(spent)
n_cand = self._resolve_candidates(spec, state, settings)
- policy = (self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds)).with_candidates(n_cand)
+ policy = ((self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds))
+ .with_candidates(n_cand).with_judge(spec.backends.judge))
# reference images: static objects are scored with reference_v1 (adds the measured silhouette
# criterion); other tracks keep their rubric but the judge still sees the references.
rubric = REFERENCE_RUBRIC if spec.references and self.track is Track.STATIC_OBJECT else self.rubric
@@ -381,10 +419,14 @@ class BaseTrack:
selector = BestSelector()
rejudged: set[int] = set()
while True:
- decision = stop_policy.decide(rounds, budget_ok=ctx.budget.ok()) if rounds else "continue"
- if decision != "continue":
- ctx.events.emit("stop", reason=decision, rounds=len(rounds), best=ctx.state.best_round)
- return decision
+ decision = stop_policy.evaluate(rounds, budget_ok=ctx.budget.ok())
+ if decision.stop:
+ ctx.events.emit("stop", rounds=len(rounds), best=ctx.state.best_round, **decision.event())
+ return decision.reason
+ if decision.strategy == "switch":
+ # the last round regressed by more than the judge's noise: one round of a
+ # DIFFERENT shape (whole-artifact rewrite), never another of the same.
+ ctx.events.emit("strategy.switch", round=len(rounds), reason="regression", detail=decision.detail)
index = len(rounds)
previous = rounds[-1].judgment if rounds else None
if index == 0:
@@ -404,8 +446,8 @@ class BaseTrack:
ctx.events.emit("best.updated", round=best, score=rounds[best].score)
self._save_spent(ctx)
continue # decide() re-runs with the recovered score
- tasks, instructions = self.refine_tasks(ctx, last, rounds)
- kind = "refine"
+ tasks, instructions = self.refine_tasks(ctx, last, rounds, strategy=decision.strategy)
+ kind = REWRITE_KIND if decision.strategy == "switch" else "refine"
if not tasks:
if last.judgment is None and last.build is not None and last.build.ok:
# no tasks only because the judge never scored the round: stopping
@@ -422,7 +464,7 @@ class BaseTrack:
else:
rec = run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,
previous=previous, files_hint=self.round_files_hint(ctx),
- extra_notes=self.round_extra_notes(ctx))
+ extra_notes=self.round_extra_notes(ctx), previous_best=best_score(rounds))
except RoundFailed as e:
if index == 0:
raise
@@ -490,7 +532,7 @@ class BaseTrack:
ctx.events.emit("finalise.rebuild", round=best, ok=build.ok)
except Exception as e: # noqa: BLE001 — the best round already built once; report, don't fail
ctx.events.emit("finalise.rebuild_failed", error=f"{type(e).__name__}: {e}")
- if rounds and (ctx.spec.options.texture or "texture" in ctx.spec.tags or ctx.extra.get("texture")):
+ if rounds and self._texture_wanted(ctx):
self._texture_pass(ctx)
ctx.state.status, ctx.state.stop_reason, ctx.state.error = status, stop_reason, error
ctx.state.save(ctx.ws)
@@ -512,9 +554,35 @@ class BaseTrack:
log.warning("changed_files failed in finalise: %s", e)
return False
+ @staticmethod
+ def _texture_wanted(ctx: RunContext) -> bool:
+ """``codeverse.texturing.run.texture_requested`` is the single owner of this
+ decision (spec.options.texture / the legacy tag) — the same predicate the
+ ``texture_pass`` spatial tool asks, so a ``texture: false`` run cannot be
+ textured from inside an agent session either."""
+ try:
+ from codeverse.texturing.run import texture_requested
+
+ return texture_requested(ctx.spec)
+ except Exception as e: # noqa: BLE001 — texturing is optional; never block finalise
+ log.debug("texture_requested unavailable: %s", e)
+ return bool(ctx.spec.options.texture)
+
def _texture_pass(self, ctx: RunContext) -> None:
- """Optional post-hoc texture pass (spec tag 'texture'). Additive: any failure
- is logged + emitted, never fatal; usage is added to the budget totals."""
+ """Optional post-hoc texture pass (``Spec.options.texture``). Additive: any
+ failure is logged + emitted, never fatal.
+
+ Its usage goes through the SAME guard as everything else (stage ``texture``,
+ role ``image``) so it lands in ``record.total_usage`` and in the ledger — four
+ post-hoc passes on ``furn_hard_rolltop_desk`` ($0.656) were invisible to both
+ (docs/COST.md §6). It is charged without enforcement: the run is finalising,
+ the tiles are already paid for, and raising here would throw away the pack."""
+ if self._already_textured(ctx):
+ # An agent can call the `texture_pass` spatial tool during a round; the
+ # finalise hook must not buy the same pack a second time (a texture pass
+ # is $0.10-0.30 and the input GLB is unchanged).
+ ctx.events.emit("texture.skipped", reason="already_textured_this_artifact")
+ return
try:
from codeverse.texturing.run import texture_pass
@@ -525,21 +593,58 @@ class BaseTrack:
log.warning("texture pass failed: %s", e)
ctx.events.emit("texture.failed", error=f"{type(e).__name__}: {e}")
return
- ctx.budget.add(trep.usage)
+ ctx.budget.add(trep.usage, stage="texture", role="image", label="texture_pass")
ctx.extra["texturing"] = trep.summary()
+ if not ctx.budget.ok():
+ # visible, but NOT `budget.exceeded`: that event means "the loop stopped",
+ # and a texture pass runs after the loop has already finished.
+ ctx.events.emit("budget.overrun", stage="texture",
+ reason=f"the texture pass took the run past its ceiling "
+ f"(${ctx.budget.spent.cost_usd:.3f} of ${ctx.budget.hard_usd:.2f})",
+ spent_usd=round(ctx.budget.spent.cost_usd, 4))
+
+ @staticmethod
+ def _already_textured(ctx: RunContext) -> bool:
+ """True when a texture pass already ran against the current ``object.glb``.
+
+ The report (``artifacts/texturing.json``) records the GLB it consumed; the
+ hook compares content hashes so a texture pass from an EARLIER round (whose
+ GLB has since been rebuilt) does not suppress the final one."""
+ import hashlib
+ import json as _json
+
+ report = ctx.ws.artifacts / "texturing.json"
+ glb = ctx.ws.artifacts / "object.glb"
+ if not report.is_file() or not glb.is_file():
+ return False
+ try:
+ data = _json.loads(report.read_text())
+ prev = Path(str(data.get("glb_in") or ""))
+ if not prev.is_file():
+ return False
+ digest = lambda p: hashlib.sha256(p.read_bytes()).hexdigest() # noqa: E731
+ return digest(prev) == digest(glb)
+ except Exception: # noqa: BLE001 — a malformed report must not block texturing
+ return False
def _record(self, ctx: RunContext, rounds: list[RoundRecord], status: RunStatus, *, error: str, stop_reason: str) -> RunRecord:
best = ctx.state.best_round
baseline = rounds[0].score if rounds else None
final = rounds[best].score if best is not None and best < len(rounds) else None
total = sum_usage(rounds)
- # planner + other non-round calls are in the budget guard but not in rounds
+ # planner, aborted rounds, retried sessions, the texture pass: all in the guard,
+ # none of them in ``rounds`` — the guard is the honest total (docs/COST.md §6).
if ctx.budget.spent.cost_usd > total.cost_usd:
total = ctx.budget.spent
extra: dict[str, Any] = {"stop_reason": stop_reason, "rubric": ctx.rubric, "budget": ctx.budget.summary(),
- "n_candidates": ctx.policy.n_candidates, "candidates": ctx.state.extra.get("candidates")}
+ "n_candidates": ctx.policy.n_candidates, "candidates": ctx.state.extra.get("candidates"),
+ "cost_by_stage": ctx.budget.stage_summary()}
if ctx.extra.get("texturing"):
extra["texturing"] = ctx.extra["texturing"]
+ if ctx.extra.get("aborted_rounds"):
+ # rounds the budget (or a crash) cut after the money was spent: they are not
+ # in ``rounds``, so name them here and keep their dollars in total_usage.
+ extra["aborted_rounds"] = ctx.extra["aborted_rounds"]
return RunRecord(
spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, best_round=best,
baseline_score=baseline, final_score=final, total_usage=total,
@@ -551,5 +656,17 @@ class BaseTrack:
)
+def rewrite_task(last: RoundRecord, best: float | None) -> RefineTask:
+ """The instruction that turns a repeat into a change of approach."""
+ was = f"{last.score:.3f}" if last.score is not None else "no score"
+ peak = f"{best:.3f}" if best is not None else "the earlier best"
+ return RefineTask(
+ target="overall", kind="rewrite", priority=0, source="judge",
+ instruction=(f"The previous round made the artifact WORSE ({was} against {peak}) — its edits are not the "
+ "way forward. Do NOT repeat them part by part: re-read the plan and the failures below, then "
+ "rewrite the whole artifact coherently, keeping only what the evidence shows was already right."),
+ )
+
+
def generator_label(agent_id: str) -> str:
return "single-shot" if is_single_shot(agent_id) else agent_id.split(":", 1)[0]
diff --git a/harness/codeverse/tracks/repair.py b/harness/codeverse/tracks/repair.py
index 31f0299f..11000120 100644
--- a/harness/codeverse/tracks/repair.py
+++ b/harness/codeverse/tracks/repair.py
@@ -35,6 +35,7 @@ class RepairOutcome(BaseModel):
attempts: list[GenerationResult] = Field(default_factory=list)
usage: Usage = Field(default_factory=Usage)
repaired: bool = False
+ max_attempts: int = Field(default=0, description="repair budget this outcome ran under")
@property
def ok(self) -> bool:
@@ -149,7 +150,7 @@ def build_with_repair(ctx: RunContext, *, round_index: int, label: str, files_hi
build, lint = build_once(ctx)
ctx.events.emit("build.done", round=round_index, ok=build.ok, lint_errors=len(lint.errors),
error=build.error_message[:200], duration_ms=build.duration_ms)
- outcome = RepairOutcome(build=build, lint=lint)
+ outcome = RepairOutcome(build=build, lint=lint, max_attempts=max_attempts)
prev_sig = ""
repeats = 0
attempt = 0
@@ -162,7 +163,8 @@ def build_with_repair(ctx: RunContext, *, round_index: int, label: str, files_hi
label=f"{label}_repair{attempt}", files_hint=files_hint)
ctx.events.emit("repair.attempt", round=round_index, attempt=attempt, repeats=repeats, signature=sig[:200])
res = generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model,
- settings=ctx.settings, budget=ctx.budget, events=ctx.events)
+ settings=ctx.settings, budget=ctx.budget, events=ctx.events,
+ max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
outcome.attempts.append(res)
outcome.usage = outcome.usage + res.usage
if not res.ok:
diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.py
index f2da07cb..f4a08b60 100644
--- a/harness/codeverse/tracks/steps.py
+++ b/harness/codeverse/tracks/steps.py
@@ -4,6 +4,18 @@
gates / measurement / rendering. Every step writes its artifact under the
workspace (gates/rNN, renders/rNN, judge/rNN.json, rounds/rNN.json) so a run
can be resumed and the flywheel can replay it.
+
+Two cost rules live here (docs/COST.md §5, §6):
+
+* **never pay for a verdict you will not use** — :func:`skip_judge_reason` drops
+ the judge only where the verdict is provably never bought at all, and
+ :func:`rejudge_round` re-buys one only after the judge was TRIED and failed, so
+ a skip is a saving and not a deferral. Two wave-2 branches were removed for
+ failing exactly that test (docs/COST.md §17).
+* **every round reports what it burned** — :func:`emit_round_cost` writes one
+ ``cost.round`` event ({stage → $}, judge $, agent turns, wasted flag) at the
+ end of the round *and* when the round raises half-way, so the ledger never has
+ to reconstruct a round the budget cut.
"""
from __future__ import annotations
@@ -20,6 +32,7 @@ from codeverse.contracts.common import Usage
from codeverse.contracts.judgment import Judgment
from codeverse.contracts.plan import AcceptanceItem, Plan
from codeverse.contracts.run import RoundRecord
+from codeverse.orchestrator.budget import usage_delta
from codeverse.orchestrator.fanout import fan_out
from codeverse.tracks.common import RunContext
from codeverse.tracks.generation import GenerationResult, GenerationTask, generate
@@ -71,7 +84,8 @@ def run_generation_tasks(ctx: RunContext, tasks: Sequence[GenerationTask]) -> li
def _one(task: GenerationTask) -> GenerationResult:
return generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model,
- settings=ctx.settings, budget=ctx.budget, events=ctx.events)
+ settings=ctx.settings, budget=ctx.budget, events=ctx.events,
+ max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
# fan_out also for ONE task: a crashing generator (503 storm, parse error) becomes a failed
# result → RoundFailed (refine rounds treat that as a plateau) instead of killing the run.
@@ -105,34 +119,75 @@ def run_round(
files_hint: Sequence[str] = (),
extra_usage: Usage | None = None,
extra_notes: Sequence[str] = (),
+ previous_best: float | None = None,
) -> RoundRecord:
"""Execute one round and persist its record. Budget is charged as it goes.
``tasks`` may be empty when the code is already in place (scene stages,
best-of-N winner copied in): the round is then build → gates → render →
judge only. ``extra_usage`` / ``extra_notes`` fold pre-round work
- (candidate generation) into the record."""
+ (candidate generation) into the record; ``previous_best`` is the best score
+ before this round, used only to flag the round as wasted in ``cost.round``."""
+ mark = ctx.budget.mark()
+ try:
+ return _run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,
+ previous=previous, files_hint=files_hint, extra_usage=extra_usage,
+ extra_notes=extra_notes, previous_best=previous_best)
+ except BaseException as e:
+ # the round died half-way (budget stop, 503 storm, RoundFailed). Whatever it
+ # burned is already in the guard: report it so the round is not invisible.
+ burned = usage_delta(ctx.budget.spent, mark)
+ record_aborted_round(ctx, index=index, kind=kind, usage=burned, error=f"{type(e).__name__}: {e}")
+ raise
+
+
+def _run_round(
+ ctx: RunContext,
+ *,
+ index: int,
+ kind: str,
+ tasks: Sequence[GenerationTask],
+ pipeline: RoundPipeline,
+ instructions: Sequence[str] = (),
+ previous: Judgment | None = None,
+ files_hint: Sequence[str] = (),
+ extra_usage: Usage | None = None,
+ extra_notes: Sequence[str] = (),
+ previous_best: float | None = None,
+) -> RoundRecord:
t0 = time.time()
ctx.events.emit("round.start", round=index, kind=kind, n_tasks=len(tasks))
rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, instructions=list(instructions),
started_at=datetime.now(UTC))
usage = extra_usage or Usage()
notes: list[str] = list(extra_notes)
+ cost: dict[str, float] = {}
+ if extra_usage is not None and extra_usage.cost_usd:
+ cost["candidates"] = round(extra_usage.cost_usd, 6)
gens = run_generation_tasks(ctx, tasks)
+ turns = 0
for g in gens:
usage = usage + g.usage
+ turns += g.turns
if not g.ok:
notes.append(f"{g.label}: {g.notes}")
+ if gens:
+ cost["generate"] = round(sum(g.usage.cost_usd for g in gens), 6)
ctx.ws.commit(f"r{index:02d} {kind}: generated")
outcome: RepairOutcome = build_with_repair(ctx, round_index=index, label=f"r{index:02d}_{kind}",
files_hint=list(files_hint))
usage = usage + outcome.usage
+ if outcome.usage.cost_usd:
+ cost["repair"] = round(outcome.usage.cost_usd, 6)
rec.build = outcome.build
gates: list[GateReport] = [outcome.lint]
if outcome.attempts:
- notes.append(f"repair attempts: {len(outcome.attempts)} ({'fixed' if outcome.ok else 'still failing'})")
+ notes.append(
+ f"repair attempts: {len(outcome.attempts)}/{outcome.max_attempts} "
+ f"({'fixed' if outcome.ok else 'still failing'})"
+ )
if outcome.build.ok:
rec.measurement = pipeline.measure(ctx, outcome.build)
@@ -144,18 +199,20 @@ def run_round(
n_err = sum(len(g.errors) for g in gates)
ctx.events.emit("gates.done", round=index, n_gates=len(gates), n_errors=n_err,
tri_count=rec.measurement.tri_count if rec.measurement else None)
- if ctx.judge is not None and rec.renders is not None and rec.renders.views and (n_err == 0 or ctx.policy.judge_on_gate_errors):
+ skip = skip_judge_reason(ctx, gates=gates, renders=rec.renders)
+ if skip:
+ notes.append(f"judge skipped ({skip})")
+ ctx.events.emit("judge.skipped", round=index, reason=skip)
+ else:
rec.judgment = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous)
if rec.judgment is not None:
usage = usage + rec.judgment.usage
+ cost["judge"] = round(rec.judgment.usage.cost_usd, 6)
# add, never charge: the verdict exists and is paid for — raising here
# would drop a fully judged round before it is committed/recorded
# (the loop stops at its next budget_ok check instead, AFTER best promotion).
- ctx.budget.add(rec.judgment.usage)
- elif ctx.judge is None:
- notes.append("no judge configured")
- else:
- notes.append("judge skipped (gate errors)")
+ ctx.budget.add(rec.judgment.usage, stage="judge", role="judge", round_index=index,
+ label=rec.judgment.rubric or "judge")
else:
notes.append(f"build failed: {outcome.build.error_type}: {outcome.build.error_message[:200]}")
rec.gates = gates
@@ -168,9 +225,117 @@ def run_round(
ctx.ws.write_json(round_record_path(ctx, index), rec)
ctx.events.emit("round.done", round=index, kind=kind, commit=rec.commit[:10], score=rec.score,
build_ok=outcome.build.ok, duration_s=rec.duration_s, cost_usd=round(usage.cost_usd, 4))
+ emit_round_cost(ctx, index=index, kind=kind, cost=cost, usage=usage, turns=turns, score=rec.score,
+ previous_best=previous_best, build_ok=outcome.build.ok, judged=rec.judgment is not None)
return rec
+# ----------------------------------------------------------------------------- cost of a round
+def skip_judge_reason(ctx: RunContext, *, gates: Sequence[GateReport], renders: RenderSet | None) -> str:
+ """Why this round must NOT be judged (``""`` = judge it).
+
+ Only states in which the verdict is never bought at all — a skip that the
+ loop buys back a moment later (``rejudge_round``) is not a saving, it is a
+ ``judge.retry`` plus a lost score:
+
+ * ``no judge configured`` / ``no renders`` — there is nothing to buy.
+ * ``budget already exceeded`` — the loop's next ``budget_ok`` check ends the
+ run, so this verdict could not promote anything. Measured: 2 verdicts /
+ $0.09 in the audit were bought past the run's wall clock (docs/COST.md §5).
+ * ``gate errors`` — only when the caller set ``judge_on_gate_errors=False``;
+ ``rejudge_round`` honours that too, so the verdict is not re-bought.
+
+ A broken build never gets here (``run_round`` judges only when the build is
+ ok) and a round in which nothing changed never gets here either
+ (``run_generation_tasks`` raises :class:`RoundFailed` first)."""
+ if ctx.judge is None:
+ return "no judge configured"
+ if renders is None or not renders.views:
+ return "no renders"
+ if not ctx.budget.ok():
+ return "budget already exceeded"
+ if judge_blocked_by_gates(ctx, gates):
+ return "gate errors"
+ return ""
+
+
+def judge_blocked_by_gates(ctx: RunContext, gates: Sequence[GateReport]) -> bool:
+ """The caller asked not to judge rounds with gate errors, and this round has some."""
+ return bool(not ctx.policy.judge_on_gate_errors and any(g.errors for g in gates))
+
+
+def emit_round_cost(
+ ctx: RunContext,
+ *,
+ index: int,
+ kind: str,
+ cost: dict[str, float],
+ usage: Usage,
+ turns: int,
+ score: float | None,
+ previous_best: float | None,
+ build_ok: bool,
+ judged: bool,
+ aborted: str = "",
+) -> None:
+ """One ``cost.round`` event per round: ``{stage → $}``, judge $, agent turns and
+ whether the money bought anything. The ledger/audit reads this instead of
+ reconstructing the round from trajectories and events."""
+ wasted, why = _waste_flag(score=score, previous_best=previous_best, build_ok=build_ok,
+ judged=judged, aborted=aborted)
+ ctx.events.emit(
+ "cost.round", round=index, kind=kind,
+ stages={k: round(v, 6) for k, v in cost.items()},
+ judge_usd=round(cost.get("judge", 0.0), 6),
+ total_usd=round(usage.cost_usd, 6),
+ agent_turns=turns,
+ input_tokens=usage.input_tokens, output_tokens=usage.output_tokens, cached_tokens=usage.cached_tokens,
+ score=score, previous_best=previous_best, wasted=wasted, waste_reason=why, aborted=aborted,
+ run_usd=round(ctx.budget.spent.cost_usd, 6),
+ )
+
+
+def _waste_flag(*, score: float | None, previous_best: float | None, build_ok: bool,
+ judged: bool, aborted: str) -> tuple[bool, str]:
+ """Did this round buy anything? (docs/COST.md §5 vocabulary)"""
+ if aborted:
+ return True, "aborted"
+ if not build_ok:
+ return True, "build_failed"
+ if not judged or score is None:
+ return True, "unjudged"
+ if previous_best is None:
+ return False, ""
+ delta = score - previous_best
+ if delta < -0.005:
+ return True, "regression"
+ if abs(delta) < 0.005:
+ return True, "zero_delta"
+ return False, ""
+
+
+def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage, error: str) -> None:
+ """A round that raised half-way still reports what it burned.
+
+ Its record is written next to the round records as ``aborted_rNN.json`` (never
+ ``rNN.json``: the round did not happen, and ``load_round_records`` must not
+ resume from it), a ``cost.round`` event is emitted, and the usage is remembered
+ in ``ctx.extra`` so the run record can carry it. Accounting a stop must never
+ raise on top of the stop that is already happening."""
+ try:
+ rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, usage=usage,
+ notes=f"aborted: {error}")
+ path = ctx.ws.root / "rounds" / f"aborted_r{index:02d}.json"
+ path.parent.mkdir(parents=True, exist_ok=True)
+ ctx.ws.write_json(path, rec)
+ aborted = ctx.extra.setdefault("aborted_rounds", [])
+ aborted.append({"index": index, "kind": kind, "cost_usd": round(usage.cost_usd, 6), "error": error[:300]})
+ emit_round_cost(ctx, index=index, kind=kind, cost={"aborted": round(usage.cost_usd, 6)}, usage=usage,
+ turns=0, score=None, previous_best=None, build_ok=False, judged=False, aborted=error[:300])
+ except Exception as e: # noqa: BLE001 — never mask the exception that stopped the round
+ log.warning("could not record aborted round r%02d: %s", index, e)
+
+
def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildResult, gates: list[GateReport],
rec: RoundRecord, previous: Judgment | None) -> Judgment | None:
from codeverse.judges.base import JudgeInput
@@ -209,7 +374,8 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildRes
if _is_degraded(judgment):
# a glitch, never a score: keep the raw verdict on disk, pay for it, but do not
# let 0.0 poison plateau/best/refine (judges/scoring.degraded_judgment contract)
- ctx.budget.add(judgment.usage)
+ ctx.budget.add(judgment.usage, stage="judge", role="judge", round_index=index,
+ label=judgment.rubric or "judge", outcome="degraded")
ctx.events.emit("judge.degraded", round=index, error=judgment.summary[:300],
cost_usd=round(judgment.usage.cost_usd, 4))
rec.notes = (rec.notes + "; " if rec.notes else "") + f"judge degraded: {judgment.summary[:200]}"
@@ -229,12 +395,19 @@ def _is_degraded(judgment: Judgment) -> bool:
def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, previous: Judgment | None = None) -> bool:
- """Re-judge an already built+rendered round whose judgment failed or was degraded.
+ """Re-judge an already built+rendered round whose judgment FAILED or was degraded.
No regeneration, no rebuild: the same commit is judged again. On success the
round record is updated in place and re-persisted. Returns True when the
- round now has a usable judgment."""
- if rec.build is None or not rec.build.ok or rec.renders is None or not rec.renders.views:
+ round now has a usable judgment.
+
+ A verdict :func:`skip_judge_reason` deliberately did not buy is NOT bought
+ here — otherwise the skip only moves the same dollar one loop iteration later
+ (the verifier's reproduction: three rounds "skipped", two of them re-judged on
+ the next iteration and the last one left without a score)."""
+ if ctx.judge is None or rec.build is None or not rec.build.ok or rec.renders is None or not rec.renders.views:
+ return False
+ if judge_blocked_by_gates(ctx, rec.gates):
return False
ctx.events.emit("judge.retry", round=rec.index)
judgment = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous)
@@ -242,7 +415,8 @@ def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, pr
return False
rec.judgment = judgment
rec.usage = rec.usage + judgment.usage
- ctx.budget.add(judgment.usage)
+ ctx.budget.add(judgment.usage, stage="judge", role="judge", round_index=rec.index,
+ label=judgment.rubric or "judge", outcome="rejudge")
ctx.ws.write_json(round_record_path(ctx, rec.index), rec)
return True
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 1738f9b7..07bda81f 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -93,6 +93,13 @@ codeverse/
contract.py (authoring-frame hints), sections.py, silhouette.py, probes.py,
joints*.py + joints_collide.py (deterministic penetration), registry.py, tools*.py
(tools_texture.py, tools_graphics.py), mcp_server.py (MCP name: 3dcv)
+ cost/ types.py (CallCost/Stage/Role) ledger.py (append-only telemetry/cost.jsonl + price provenance)
+ context.py (per-call > ambient attribution) instrument.py (MeteredChatModel /
+ MeteredAgent — one row per ChatModel.generate; one session row only for a backend
+ that does NOT meter itself; run_ledger nests + is context-local so bench --parallel works)
+ profiles.py (economy|balanced|quality; cli._common.resolve_dial is THE resolver)
+ caching.py (Block/order_blocks/session_cache — measurement only, docs/COST.md §13)
+ guard.py routing.py reconstruct.py (old runs) audit.py report.py
judges/ rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,
prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.py
(compare_many), reference.py, calibration.py, metrics.py
@@ -127,7 +134,8 @@ runs/<slug>/
public/ (scene) compiled assets public/assets/<snake>.glb; (textured scenes) public/textures/*.png + manifest.json
_assets/<snake>/ (scene) sub-workspaces for blender_glb assets (gitignored)
_cand/c<k>/ (--candidates N) throw-away best-of-N sub-workspaces (gitignored, kept for the flywheel)
- stages/<name>.json rounds/rNN.json rounds/candidates.json
+ stages/<name>.json rounds/rNN.json rounds/candidates.json rounds/aborted_rNN.json (a round the
+ budget/a crash cut: what it burned, never resumed from) cost_ledger.jsonl (live ledger)
artifacts/ object.glb object.stl|step robot.urdf meshes/ articulation.json build.json census.json
measurement.json … ; graphics: frames/fNN_tT.png frames_sheet.png preview.gif metrics.json
texturing: object_textured.glb textures/{<id>.png, texture_plan.json, texturing.json, gate/}
@@ -230,13 +238,23 @@ round 0 "baseline": generate → build_with_repair → measure → gates → ren
selection build_ok → quick score → fewer gate errors with pairwise tie-break; winner copied back, normal r00 follows)
repeat while StopPolicy says continue (≤ max_rounds refine rounds, plateau_window=2 / min_delta=0.02,
target = rubric threshold, budget ok):
+ money stops, sized by the judge's MEASURED noise σ (cost/routing.JUDGE_NOISE: pro 0.030, flash 0.083):
+ regression — last round scored < best − 1σ ⇒ never another round of the same shape: the first one
+ switches strategy (ONE whole-artifact rewrite, kind "rewrite"), a second stops the run
+ diminishing_returns — from r03 on, only start when the last gain > 1.5σ AND best < target
refine tasks = gate ERRORS (fix hints, authoring-frame numbers) ∪ failed must-acceptance ∪ judge improvement plan
(≤ 6 tasks, ≤ 6 compacted instruction lines each; reference runs add an IoU task when silhouette IoU < 0.6)
fan out when ≥ 2 file-disjoint groups AND every task maps to files (threejs/blender parts, scene zones/assets/env)
- generate → build+repair (error-focused, escalates on identical signatures) → gates → … → judge
+ generate (NO turn cap by default — 28 was A/B'd and rejected, +$0.02/−0.21 score, docs/COST.md §17;
+ a cap a caller sets (CV3D_AGENT_MAX_TURNS / task; no profile sets one) still buys a wrap-up session
+ that lands a final build + summary instead of being killed) → build+repair (error-focused,
+ escalates on identical signatures) → gates → … → judge (SKIPPED only where the verdict is never
+ bought at all: no judge/renders, budget already exceeded, or judge_on_gate_errors=False)
BestSelector: highest score, tie → fewer gate errors; |Δ| < pairwise_margin (0.03) → position-swapped
PairwiseJudge decides (replace only at confidence ≥ 0.6; note persisted in rNN.json)
-stop reasons: pass | plateau | budget | max_rounds | no_change | no_refine_tasks
+ every round emits cost.round {stage → $, judge $, agent turns, wasted flag}; a round that raises mid-way
+ still reports what it burned (rounds/aborted_rNN.json + record.extra["aborted_rounds"])
+stop reasons: pass | plateau | budget | max_rounds | no_change | no_refine_tasks | regression | diminishing_returns
finalise: restore best commit, rebuild so artifacts match delivered code, finalize_record → record.json
```
Track-specific gates: static `connectivity` + `contract` (+ `reference_silhouette`),
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index cc8deff8..25a50306 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -21,21 +21,30 @@ old runs (`codeverse/cost/reconstruct.py`), so it works on every run recorded so
far — no re-instrumentation needed. `bench/out/compare_v1_full` (a partial,
superseded compare battery) is excluded.
+> **Re-run 2026-08-23 (wave 3).** Every figure in Part I below was recomputed
+> after a de-duplication bug in the reconstruction was fixed (§6): a retried agent
+> session is recorded as `<label>.a2` while the `generate.done` event that
+> summarises the job carries the plain label, so the event was counted a *second*
+> time on 14 runs. The reconstructed total drops **$89.88 → $86.30 (−4.0 %)** and
+> lands $1.22 above the runs' own `record.total_usage` ($85.08) instead of $4.80.
+> The waste analysis (§5) and the round economics are unchanged — they are built
+> from round records, not from those events.
+
---
## 1. Headline
| | |
|---|---|
-| total spend | **$89.88** over 61 runs (26.4 h of run time, 7,558 model calls) |
-| per run | $1.47 (median $0.95) |
-| **per passing artifact** | **$2.50** — 36 of 61 runs passed |
+| total spend | **$86.30** over 61 runs (26.4 h of run time, 7,544 model calls) |
+| per run | $1.41 (median $0.95) |
+| **per passing artifact** | **$2.40** — 36 of 61 runs passed |
| input tokens | 278.6M, of which **80% are cache reads** |
| output tokens | 3.94M (+0.66M thoughts) |
-| generation : judging : planning | **90.3% : 7.2% : 0.8%** |
-| identified waste | **$27.35 (30%)** — see §5 |
-| recorded vs measured | `record.total_usage` says $85.08; the ledger finds **$89.88** (§6) |
-| re-priced with the corrected table | **$92.67** (+3.1%, §7) |
+| generation : judging : planning | **89.8% : 7.5% : 0.8%** |
+| identified waste | **$27.35 (32%)** — see §5 |
+| recorded vs measured | `record.total_usage` says $85.08; the ledger finds **$86.30** (§6) |
+| re-priced with the corrected table | **$89.10** (+3.2%, §7) |
**The money is generation, not judging.** Nine dollars in ten are the coding
agent writing and rewriting code; the judge is a rounding error next to it
@@ -49,15 +58,15 @@ cheaper judge" is optimising 7% of the bill — and, as §8 shows, would lose mo
| key | USD | share | calls | input | cached | output | $/1k tok | model time |
|---|---|---|---|---|---|---|---|---|
| plan | $0.7234 | 0.8% | 56 | (not recorded) | – | – | – | 0.0 h |
-| assets (scene) | $5.61 | 6.2% | 587 | 13.24M | 61% | 236.8k | $0.00041 | 1.0 h |
+| assets (scene) | $5.61 | 6.5% | 587 | 13.24M | 61% | 236.8k | $0.00041 | 1.0 h |
| env (scene) | $0.7762 | 0.9% | 92 | 1.96M | 62% | 19.1k | $0.00039 | 0.1 h |
-| zones (scene) | $4.75 | 5.3% | 452 | 15.55M | 76% | 140.8k | $0.00030 | 0.7 h |
-| baseline | $25.76 | 28.7% | 1,815 | 86.76M | 85% | 1.39M | $0.00029 | 4.0 h |
+| zones (scene) | $4.49 | 5.2% | 451 | 15.55M | 76% | 140.8k | $0.00028 | 0.7 h |
+| baseline | $25.06 | 29.0% | 1,814 | 86.76M | 85% | 1.39M | $0.00028 | 4.0 h |
| repair | $1.31 | 1.5% | 152 | 5.08M | 79% | 49.9k | $0.00026 | 0.2 h |
-| **refine** | **$42.94** | **47.8%** | 4,243 | 154.50M | 80% | 1.82M | $0.00027 | 8.4 h |
-| judge | $6.48 | 7.2% | 126 | 1.33M | 8% | 212.1k | $0.00383 | 1.2 h |
-| texture | $1.10 | 1.2% | 16 | (event only) | – | – | – | 0.0 h |
-| other / unattributed | $0.43 | 0.5% | 19 | 235.2k | 10% | 67.3k | $0.00142 | 0.0 h |
+| **refine** | **$40.28** | **46.7%** | 4,231 | 154.50M | 80% | 1.82M | $0.00026 | 8.4 h |
+| judge | $6.48 | 7.5% | 126 | 1.33M | 8% | 212.1k | $0.00383 | 1.2 h |
+| texture | $1.10 | 1.3% | 16 | (event only) | – | – | – | 0.0 h |
+| other / unattributed | $0.4776 | 0.6% | 19 | 235.2k | 10% | 67.3k | $0.00158 | 0.0 h |
`skeleton`, `assemble`, `gates` and `render` cost **$0.00**: they are
deterministic harness work. Blender build 0.13 s, gates ~2 s, an 8-view GPU
@@ -67,29 +76,30 @@ render ~1.5 s — the "cheap first" law is holding.
| role | USD | share | calls | $/1k tokens |
|---|---|---|---|---|
-| generator | $81.14 | 90.3% | 7,341 | $0.00029 |
-| judge | $6.48 | 7.2% | 126 | $0.00383 |
+| generator | $77.52 | 89.8% | 7,327 | $0.00028 |
+| judge | $6.48 | 7.5% | 126 | $0.00383 |
| planner | $0.72 | 0.8% | 56 | – |
| image (texturing) | $0.61 | 0.7% | 5 | – |
+| other (event-only) | $0.97 | 1.1% | 30 | – |
| track | runs | USD | $/run | passed | **$/passing artifact** | median rounds |
|---|---|---|---|---|---|---|
-| static_object | 37 | $55.95 | $1.51 | 23 | **$2.43** | 2 |
-| articulated_object | 13 | $19.19 | $1.48 | 8 | **$2.40** | 2 |
-| scene | 5 | $14.14 | $2.83 | **0** | **∞** | 2 |
+| static_object | 37 | $54.45 | $1.47 | 23 | **$2.37** | 2 |
+| articulated_object | 13 | $17.34 | $1.33 | 8 | **$2.17** | 2 |
+| scene | 5 | $13.92 | $2.78 | **0** | **∞** | 2 |
| graphics | 6 | $0.59 | $0.10 | 5 | **$0.12** | 1 |
-The graphics track is **20x cheaper per passing artifact** than the object
+The graphics track is **18x cheaper per passing artifact** than the object
tracks (one single-shot generation, compiler feedback, no tool loop). The scene
track has not produced a single passing artifact in this data set at $2.83/run.
| model | USD | share | calls | cached | note |
|---|---|---|---|---|---|
-| gemini-3.7-flash | $74.08 | 82.4% | 7,400 | 80% | generator + planner + cheap judge |
-| gemini-3.1-pro-preview | $6.23 | 6.9% | 109 | 7% | default judge |
-| (event-only rows) | $5.40 | 6.0% | 44 | – | plan / texture / pairwise: cost recorded, tokens not |
-| gpt-5.6-sol (codex) | $2.26 | 2.5% | 3 | 92% | one telescope run |
-| claude-fable-5 (claude-code) | $1.90 | 2.1% | 2 | 0% | recorded as `claude-haiku-4-5` — see §7 |
+| gemini-3.7-flash | $74.13 | 85.9% | 7,400 | 80% | generator + planner + cheap judge |
+| gemini-3.1-pro-preview | $6.23 | 7.2% | 109 | 7% | default judge |
+| gpt-5.6-sol (codex) | $2.26 | 2.6% | 3 | 92% | one telescope run |
+| claude-fable-5 (claude-code) | $1.90 | 2.2% | 2 | 0% | recorded as `claude-haiku-4-5` — see §7 |
+| (event-only rows) | $1.78 | 2.1% | 30 | – | plan / texture / pairwise: cost recorded, tokens not |
## 4. Tokens, caching and the clock
@@ -97,8 +107,8 @@ track has not produced a single passing artifact in this data set at $2.83/run.
loop's input tokens correspond to only 13.3M tokens of *final* context — the
transcript is re-sent **19.5x** on average (median 22 model turns per session,
p90 52). Implicit caching absorbs most of that: cache reads cost $16.98, and
-the same traffic with a 0% hit rate would have cost **$242.70 instead of $89.88**.
-Caching is already saving 63% of the bill.
+the same traffic with a 0% hit rate would have cost **$239.12 instead of $86.30**.
+Caching is already saving 64% of the bill.
The uncached remainder is still ~$40 of that traffic, and the bill grows
quadratically with session length (the 262 agent sessions that kept a per-turn
@@ -173,12 +183,12 @@ model — **88% of the clock is waiting for a model**, 12% is the harness
| waste | n | USD | share |
|---|---|---|---|
-| regression — a round scored **below** the best and was discarded | 19 | $14.23 | 15.8% |
-| post-budget — a round completed after the budget was already gone | 9 | $8.76 | 9.7% |
+| regression — a round scored **below** the best and was discarded | 19 | $14.23 | 16.5% |
+| post-budget — a round completed after the budget was already gone | 9 | $8.76 | 10.1% |
| zero-delta round — score moved < 0.005 | 4 | $1.68 | 1.9% |
-| unpromoted judge — a verdict on a round that never became best | 23 | $1.37 | 1.5% |
+| unpromoted judge — a verdict on a round that never became best | 23 | $1.37 | 1.6% |
| repair loop that never converged (run did not pass) | 4 | $1.31 | 1.5% |
-| **total** | 59 | **$27.35** | **30.4%** |
+| **total** | 59 | **$27.35** | **31.7%** |
Concrete examples:
@@ -200,19 +210,61 @@ Marginal economics of a refine round:
The first refine round is the good buy. r02 is a coin flip; r03 costs 7x r01
per point of score.
-## 6. $4.80 of spend is invisible to `record.total_usage`
+## 6. $1.22 of spend is invisible to `record.total_usage` (was reported as $4.80)
+
+**Corrected 2026-08-23.** The first pass of this section said *"14 of 61 runs,
+$4.80 (5.6% of all spend)"*. Most of that was a bug in the **reconstruction**,
+not money the record lost. `cost/reconstruct.py` de-duplicates a
+`generate.done` event against the agent sessions that already accounted for the
+same work, and it keyed that on the session's own label — but a retried session
+records itself as `<label>.a2` (and a wrap-up as `<label>.wrapup`) while the
+event carries the plain job label. The retry's dollars therefore sat in a
+different bucket, the job's pool ran dry, and the event was added on top of the
+sessions. `_base_label()` now strips the attempt suffix before the lookup, and a
+test pins the three worst runs.
-On 14 of 61 runs the reconstructed ledger is **larger** than the run's own
-`total_usage` — $4.80 (5.6% of all spend) that the record, and therefore the
-`BudgetGuard`, never saw:
+Re-running the audit with the fixed code
+(`python bench/cost_report.py`, or the script in the wave-3 scratchpad):
-* rounds the budget cut *after* the work was done (the round is not appended to
- `record.rounds`, and its cost is not folded into the total);
-* retried agent sessions (`<label>.a2_rNN`) whose usage lands in the trajectory
- but not in the round;
-* post-hoc texture passes (`furn_hard_rolltop_desk` ran four, $0.656).
+| | before the fix | after |
+|---|---|---|
+| reconstructed total | $89.88 | **$86.30** |
+| Σ `record.total_usage` | $85.08 | $85.08 |
+| runs whose ledger exceeds their record by > $0.005 | 14 | **4** |
+| **off-record total** | $4.80 | **$1.22** |
-Worst case: `art_hard_door_handle`, $1.38 off-record on a $4.45 run.
+What is left is real, and it is one thing plus small change:
+
+| run | record | ledger | off-record | why |
+|---|---|---|---|---|
+| `furn_hard_rolltop_desk` | $1.2165 | $1.8727 | **$0.6562** | four post-hoc texture passes, run after the record was written |
+| `cmp_easy_stool/harness_api-agent…` | $0.3434 | $0.6118 | $0.2685 | compare cell: the fixed evaluator's judge is outside the harness run's total |
+| `cmp_easy_stool/harness_gemini-cli…` | $0.8810 | $1.0437 | $0.1627 | idem |
+| `e2e_bench_threejs` | $0.6840 | $0.8186 | $0.1347 | a gemini-cli session re-priced from its own `stdout.json` stats (§7 #1) |
+
+The other 19 runs with a positive gap are all under half a cent. The three worst
+offenders of the old table reconcile to the **cent** once the double count is
+gone — `art_hard_door_handle` $3.0690 vs $3.0690, `tool_med_hand_drill` $4.0210 vs
+$4.0210, `art_easy_laptop` $1.1223 vs $1.1223 (reconstructed rows excluding the
+synthetic `residual` filler) — which is the independent recount in the wave-2
+hand-off (`waste-and-accounting/MEASUREMENTS.md` §1, "$0.00 genuinely
+off-record") reproduced by the shipped code.
+`tests/cost/test_reconstruct.py` pins all three plus a synthetic retry.
+
+**A second recount reported 25 runs / $4.65; it reproduces exactly when the RE-PRICED ledger is compared with the recorded record.total_usage (the original $4.80 / 14 runs used the old price table and a higher per-run threshold). Both numbers are correct under their own method; the re-priced 25-run figure is the current one.** That figure
+is not reproducible against this tree: with the current code the same 61 runs give
+28 runs with any positive gap / $4.797 *before* the de-duplication fix and 23 /
+$1.222 after, on either the recorded or the re-priced price table. The numbers
+above are what `audit_runs()` prints today, and the script that prints them is in
+the scratchpad; anyone who can reproduce 25/$4.37 should say which run set it used.
+
+**Both holes are now closed at the source, so this section is about history:**
+the live ledger (§12) writes one row per call at the time of the call, the
+`BudgetGuard` charges the planner, aborted rounds, retried sessions and the
+texture pass through the same door, and a post-hoc pass joins the run's ledger
+(`3dcv texture pass` opens it with `create=False`). A run recorded from now on
+cannot have an off-record dollar; the reconstruction path exists for the 61 runs
+recorded before it.
## 7. Price hygiene (checked 2026-08-23)
@@ -256,8 +308,8 @@ see the hand-off notes):
`price_source="provider-reported"` and are never re-priced from the token
table.
-Re-pricing everything with the corrected table moves the total from $89.88 to
-**$92.67 (+3.1%)**: +$3.43 gemini-cli, −$0.50 codex, −$0.13 elsewhere.
+Re-pricing everything with the corrected table moves the total from $86.30 to
+**$89.10 (+3.2%)**: +$3.43 gemini-cli, −$0.50 codex, −$0.13 elsewhere.
**Not modelled** (documented in `pricing.py`): cache *storage* fees for explicit
Gemini caching, Anthropic 1-hour cache writes, batch/flex discounts, data
@@ -291,14 +343,14 @@ judge and would cost $24.2 as "cheap" flash at n=8.
## 9. Ranked optimisation opportunities
-Savings are estimated **on this data set** (61 runs, $89.88) unless stated.
+Savings are estimated **on this data set** (61 runs, $86.30) unless stated.
| # | change | est. saving | confidence | where |
|---|---|---|---|---|
| 1 | **Stop refining after r01 unless the last delta ≥ 0.05.** r02+r03 cost $12.16 and bought +1.44 score points across 8 of 20 rounds. | **$9–12 (10–13%)** | high (measured) | `RoundPolicy` (min_delta 0.02 → 0.05 from r02, or plateau_window 1 after r01) |
| 2 | **Check the budget *before* starting a round, against the estimated round cost**, not only between steps. | **$8.76 (9.7%)** | high (measured waste) | `orchestrator/budget.py` + `tracks/steps.py`; use `codeverse.cost.estimate_call` / median round cost |
-| 3 | **Cap agent turns at ~25 and compact old tool results.** 39.3% of the agent bill is turn ≥20; 19 of the 33 sessions that ran ≥50 turns were cut off by their own budget. | **$8–15 (9–17%)**, needs a quality check | medium | `AgentJob.max_turns`, `api_agent` transcript compaction |
-| 4 | **Fold the off-record spend into the budget** (retried sessions, cut rounds, texture passes). | $0 saved, **$4.80 of blindness removed** | high | §6; the ledger (`codeverse.cost.record_call`) makes it automatic |
+| 3 | ~~**Cap agent turns at ~25 and compact old tool results.** 39.3% of the agent bill is turn ≥20; 19 of the 33 sessions that ran ≥50 turns were cut off by their own budget.~~ **Tested, rejected: +$0.02 and −0.21 score** (A/B, n=3 per arm, §17) — the cap is off by default. | est. $8–15, **measured $0** | high (A/B) | `RoundPolicy.agent_max_turns=0`; still settable per caller |
+| 4 | **Fold the off-record spend into the budget** (cut rounds, post-hoc texture passes). | $0 saved, **$1.22 of blindness removed** | high | §6; the ledger (`codeverse.cost.record_call`) makes it automatic |
| 5 | **Scene track: 73% of scene spend is assets+zones, 0/5 passed.** Trim the per-zone context (each zone session re-sends the whole scene contract) and judge assets before zones start. | ~$1/run of $2.83 | medium | `tracks/scene*.py` |
| 6 | **Drop `oneshot:claude-code` from default compare arms** ($1.04/artifact at 0.673 — the worst score per dollar measured). | bench-only | high | `bench/compare_backends.py` arms |
| 7 | **Gate the texture pass on the materials criterion** (< 0.7) — 5 passes ran, 2 shipped, $1.10 spent. | ~$0.5 | medium | `texturing/run.py` |
@@ -350,7 +402,7 @@ Two helpers exist for the callers:
## 11. Caveats
* `plan`, `texture` and `pairwise` costs come from events that record a dollar
- but no tokens, so those rows have no token composition ($6.13 of $89.88).
+ but no tokens, so those rows have no token composition ($2.50 of $86.30).
Wiring `record_call` at those call sites fixes it.
* Wall clock per run is the sum of the stage/round clocks (the event span
includes hours a bench run spent queued behind other runs, and
@@ -361,3 +413,380 @@ Two helpers exist for the callers:
directional, not significant.
* Prices were read from the providers' public pages on 2026-08-23 and are the
standard, non-batch, global-endpoint tiers.
+
+---
+
+# Part II — the controls (wave 2, 2026-08-23)
+
+Part I above is the audit: where $89.88 went. Part II is what was built on top of
+it and, for each item, **what it measured** — including the two things that turned
+out not to pay.
+
+## 12. The live ledger
+
+`codeverse.cost.instrument` meters the two places money is actually spent:
+
+* **`MeteredChatModel`** wraps everything `models.get_chat_model` hands out, so one
+ `CallCost` row is appended per `ChatModel.generate` — planner, judges,
+ captioner, texturing, single-shot generation **and every turn of the in-process
+ api-agent**.
+* **`MeteredAgent`** wraps everything `agents.get_coding_agent` hands out: it sets
+ the ambient round/stage for the session (so the rows above land in the right
+ bucket), applies whatever turn cap the settings name (only ever *lowering*
+ `job.max_turns`; no profile names one — §17)
+ and, for a backend whose calls we cannot see (gemini-cli / claude-code / codex /
+ agy), records one session row from `AgentResult.usage`.
+
+ **Which backends those are is a property of the backend, not a guess.** The
+ first version asked "did anybody write a ledger row while this session ran?"
+ (a thread-local counter). That is wrong in both directions, and both were
+ reproduced: a gemini-cli session during which *any* in-process tool billed a
+ model — a texture pass, a captioner — looked metered, so its session row was
+ dropped and **$1.23 of the reproduction vanished**; and an in-process session
+ whose turns ran in a worker thread looked unmetered and was counted **twice**.
+ `IN_PROCESS_AGENT_KINDS = {"api-agent"}` (or a backend's own
+ `meters_own_calls` attribute) decides it now, and
+ `tests/cost/test_instrument.py` pins both directions.
+
+**Attribution.** A call is filed under what *it* says it is, not under what
+surrounds it: `cost.context.attribute()` puts an explicit stage/role first, then a
+label that names a job of its own (`judge:…`, `planner`, `texture_gate`,
+`caption…`, `pairwise:…`), then the ambient agent session. A spatial tool that
+bills a model inside a refine session used to land on `stage=refine /
+role=generator`; it now lands on its own stage. A *generation* label still yields
+to the session, which knows more (best-of-N runs `job.kind="candidate"` while its
+turns are labelled `api-agent:baseline:tN`).
+
+**Who opens a ledger.** `3dcv make` / `3dcv resume` (`cli.main._run_track`),
+`3dcv texture pass` (with `create=False`), **and the bench drivers** —
+`bench/run_bench.py` opens one per prompt and `bench/compare_backends.py` one per
+cell (plus a nested one for a harness arm's own run, and one for the pairwise
+arena). The batteries produce most of the runs in this repo, so until wave 3
+most priced rows were going to the per-process fallback log. `run_ledger` nests
+(it restores the outer ledger and run binding instead of clearing them) and binds
+context-locally with a process-wide fallback, so `--parallel N` keeps N ledgers
+apart while a fan-out worker thread that inherited no context still finds its run.
+
+The row carries run, round, stage, role, label, backend/provider/model, the four
+token counts, the three **unit prices actually used** plus their provenance
+(`price_source` / `price_approximate` / `price_checked`), $, latency,
+`cache_hit`, outcome and `n_calls`. Writing never raises and never blocks: a
+pricing failure logs and keeps the recorded dollar.
+
+**Where it lands.** `<run>/telemetry/cost.jsonl` (the run-layout telemetry
+bucket), with `<run>/cost_ledger.jsonl` left as a relative symlink so
+`flywheel.telemetry.live_ledger_path` and the `telemetry/usage.jsonl` alias keep
+working — one physical copy. A call made with no run context (a `3dcv judge`
+outside a run, a bench script, a notebook) goes to a per-process log under
+`<cache_dir>/cost/`; `CV3D_COST_LEDGER=off` disables writing entirely.
+
+**No double counting.** `BudgetGuard.spend` also knows how to write rows; while
+`cost.instrument.per_call_metering()` is true it skips its own, because the
+per-call rows *are* that dollar with the tokens, cache hits and latency attached.
+
+**Reading it.** `cost.reconstruct.reconstruct_run` prefers a live ledger and
+falls back to rebuilding from trajectories / verdicts / events, so
+`3dcv cost`, `bench/cost_report.py` and the run layout's `telemetry/cost.json`
+all pick the live rows up automatically and the 61 recorded runs keep auditing
+(`RunLedger.source` says `live` or `reconstructed`).
+
+### Verified on two fresh live runs
+
+| run | track / generator | rows | ledger | `record.total_usage` | difference |
+|---|---|---|---|---|---|
+| `verify_graphics` | graphics / api-agent flash | 12 | **$0.195275** | $0.195275 | **+0.000 %** |
+| `prof_chair_balanced` | static_object / api-agent flash | 55 | **$0.510770** | $0.510770 | **+0.000 %** |
+
+Both are well inside the 1 % target. The chair run also shows the audit's §6
+blindness closed: its retried `baseline.a2` session (6 calls, $0.046) is a
+first-class set of rows, and a model error mid-session is recorded as an
+`outcome="error"` row at $0 instead of vanishing.
+
+```
+3dcv cost <slug> # one run: stage/role/model + a reconciliation block
+3dcv cost --runs-dir <root> # a battery, aggregated
+3dcv cost cache <slug> # per session: cold first call, cached share, saved $ / cold $
+```
+
+## 13. Cache-friendly prompt ordering — measured, and REVERTED
+
+Wave 2 reordered the object-track generation prompts stable-prefix-first: the
+standing contract, the frame doc, the cookbook excerpt, the standing rules, the
+tool cards and the output format were hoisted into one shared include
+(`prompts/tracks/_object_head.j2`) that `generate_static.j2`,
+`generate_articulated.j2` and `refine_object.j2` each began with, byte-identical.
+
+**That include is gone (2026-08-23).** The head was not *shared* with anything
+already being sent — it was **duplicated into every prompt**, so the shared prefix
+it created was paid for four times over. Measured with `cost.prefix_report` on
+the real templates (one baseline + three refine prompts of one run, same context):
+
+| | shared prefix | per-prompt tokens | mean per call |
+|---|---|---|---|
+| task-first (shipped, restored) | 46 tokens | 5,072 / 1,325 / 1,325 / 1,325 | **2,261** |
+| stable-head-first (reverted) | **4,970 tokens** | 5,110 / 5,211 / 5,211 / 5,211 | **5,186** |
+
+**+2,925 tokens on the average call, +3,886 on every refine call** — and the task
+prompt is message 0 of an agent session, re-sent on every turn, so the run-level
+figure is that times the session's turn count. Against it: the 4,970-token prefix
+is far below the **~12k floor** §4 measured for Gemini's implicit cache, so on a
+real run the first turn still reports `cached_tokens = 0`. The wave-2 note read
+the two arms as "≈ $0.00" run-level and kept the ordering as a free precondition;
+an independent re-measure showed the delta is not zero, it is a **net increase**,
+and the only arm that reached the cache floor did so by padding the head to 24k —
+i.e. by buying the cache block with more tokens than it returns.
+
+**Is there anywhere the prefix is genuinely long enough?** Measured, no:
+
+| candidate prefix, sent on every turn without duplication | tokens |
+|---|---|
+| api-agent system prompt (`AGENTS.md`, contract + cookbook note) | 4,879 |
+| its tool declarations (16 tools, JSON schemas) | 1,691 |
+| **total, already first in every request** | **6,570** |
+
+6.6k is still under the floor, and it is already at position 0 — there is no
+reordering left to do there. `codeverse/cost/caching.py` keeps `Block` /
+`order_blocks` / `prefix_report` / `prefix_signature` as **measurement helpers**
+with these numbers in its docstring, and `3dcv cost cache <slug>` still reports
+what a run's sessions actually cached. The rule they encode: measure the prefix
+against the floor *before* reordering a prompt family, and never pad to reach it.
+
+## 14. The judge payload — 768 px saves bytes, not dollars
+
+Re-judged 8 recorded rounds (5 static / articulated, 1 threejs, 1 compare cell, 1
+shader) at 1024 px and 768 px, `gemini-3.1-pro-preview`, n=2 per size, detail
+crops held constant.
+
+**Payload per verdict** (1024 px): 3.0k–4.6k text tokens, 3–5 images, **311–714 KB**
+of PNG. At 768 px the same montages are **229–541 KB (−26 % bytes)**.
+
+| | 1024 px | 768 px |
+|---|---|---|
+| mean overall (8 rounds) | **0.6609** | **0.6536** |
+| sd across rounds | 0.0976 | 0.1079 |
+| mean within-round σ (n=2) | **0.0326** | **0.0421** |
+| **input tokens, 16 verdicts** | **156,834** | **156,874** |
+| total $ | $0.8779 | $0.8617 |
+
+Per-round Δ(768−1024): mean **−0.0074**, sd 0.0576, worst case −0.135
+(`e2e_cabinet_urdf`, where the 768 arm's own within-round σ was 0.155 — noise, not
+signal).
+
+The mean difference is inside the judge's noise, so by the pre-registered rule the
+cheaper size would be adoptable — **except that it is not cheaper**: Gemini bills a
+montage at the same tile count at both sizes (**+0.03 % tokens, not −20 %**), and
+768 px measurably *raises* sampling σ (0.0326 → 0.0421, +29 %). **1024 px stays
+the default for every profile.**
+
+### The crop-count experiment — INCONCLUSIVE, not adopted
+
+The other payload lever is the **image count**. The same round
+(`e2e_chair_blender` r01, `gemini-3.1-pro-preview`, n=1) was re-judged at 2 / 1 / 0
+detail crops **twice, independently**:
+
+| detail crops | input tokens | $/verdict (pro) | draw A overall | draw B overall |
+|---|---|---|---|---|
+| 2 (the default) | 9,454 | $0.0480 | 0.595 | 0.600 |
+| 1 | 8,255 | $0.0449 | **0.600** | **0.552** |
+| 0 | 7,058 | $0.0403 | 0.600 | 0.600 |
+
+One crop is **1,198 input tokens ≈ $0.0024/verdict**. Draw A read the 1-crop arm
+as unchanged and wave 2 adopted it for the economy profile; draw B read the same
+arm **0.048 lower** — **1.6× the pro judge's measured σ of 0.030** §8, and larger
+than the effect being claimed. Two draws that disagree by more than the
+instrument's noise measure nothing.
+
+**Not adopted.** `judge_detail_crops = 2` in **every** profile, and
+`Settings.judge.detail_crops` keeps its default of 2 — the previous image budget,
+restored. Buying a fifth of a cent per verdict is not worth a payload change we
+cannot show is harmless, and at n=1 per arm the experiment cannot show it: it
+needs ~n=8 per arm to resolve 0.048 against σ=0.030, which costs more than the
+change saves on the 112 verdicts in this data set. A caller that wants a smaller
+payload can still say so (`VlmJudge(detail_crops=…)` / `CV3D_JUDGE__DETAIL_CROPS`);
+no profile says it for them.
+
+Both halves of §14 confirm §8's conclusion: on judging, the lever is *how many
+verdicts*, not how big each one is.
+
+## 15. Profiles — one name for the whole dial
+
+`Settings.profile` + `3dcv make --profile economy|balanced|quality` set model per
+role, judge samples, refine rounds, best-of-N width, the texture pass and the
+budget ceilings together (`codeverse/cost/profiles.py`).
+
+**One resolver, both entry points.** `codeverse.cli._common.resolve_dial` is the
+only place the dial is read, and it returns a `ResolvedDial` with every field.
+`--profile X` *forces* the dial over a value the user stated in `config.yaml` /
+`CV3D_*`; `CV3D_PROFILE=X` (or `profile:` in the config file) sets the same dial
+as a *default*, so a value you stated yourself survives it. With nothing else
+stated the two paths resolve **identically** — which they did not before: the CLI
+read `candidates` and `texture` off the *flag* rather than off the resolved
+profile, so `CV3D_PROFILE=quality` silently ran best-of-1 with no texture pass.
+`tests/cost/test_profiles.py` asserts every field of the dial from both entry
+points, per profile. An explicit CLI flag still beats both, and
+`Spec.options.profile` now records the resolved name **whichever way it was
+given**, so `3dcv resume` reproduces it.
+
+| | economy | balanced | quality |
+|---|---|---|---|
+| generator | `single-shot:gemini:gemini-3.7-flash` | `api-agent:gemini:gemini-3.7-flash` | `api-agent:gemini:gemini-3.7-flash` |
+| judge | flash, n=2 | **pro, n=1** | **pro, n=3** |
+| refine rounds | 2 | 4 | 4 |
+| best-of-N | 1 | 1 | 2 |
+| agent turn cap | – (see below) | – | – |
+| judge payload | 1024 px, 2 crops | 1024 px, 2 crops | 1024 px, 2 crops |
+| texture pass | off | off | on |
+| ceilings | $1.50 / 30 min | $5 / 60 min | $8 / 90 min |
+| expected from the audit | ~$0.30/run | ~$1.47/run | ~$3.20/run |
+
+Why these numbers: r01 buys a score point for $4.93 and r02 for $6.80 while r03
+costs $33.49 (§5), so economy stops at 2; the pro judge is *cheaper* than matching
+its precision with flash (§8), so balanced and quality both use it and only
+economy trades precision for price; best-of-2 and the texture pass are the two
+levers measured to raise a score, so they belong to quality.
+
+**Two dials wave 2 set were removed as unmeasurable (2026-08-23).**
+
+* **economy's 20-turn agent cap.** Economy's generator is `single-shot:`, which
+ opens no agent session — the cap could never fire, and no recorded economy run
+ shows it firing. The cap it was modelled on also lost its own A/B in this wave
+ (§17: a 28-turn cap cost $0.02 more and 0.205 of a score point over 3 runs per
+ arm), so there is no default cap anywhere now: `Profile.max_turns` is 0 in all
+ three profiles and `tracks.generation.DEFAULT_AGENT_MAX_TURNS` is 0.
+ `Settings.limits.agent_max_turns` / `CV3D_AGENT_MAX_TURNS` remain the knob for a
+ run or a machine that wants one by name.
+* **economy's 1 detail crop.** §14: two independent draws of the crop experiment
+ disagreed by 1.6× the judge's σ over a $0.0024/verdict saving. Every profile
+ sends the full payload.
+
+**`texture: false` now actually prevents the texture pass.** Three call sites used
+to answer "does this run texture?" independently — `Spec.options.texture`, a
+`texture` spec tag, `ctx.extra["texture"]` — and, worse, the `texture_pass` spatial
+tool is registered for every object track, so the coding agent could buy a pass in
+any run. That is why the quality run below shows a ledger 9.5 % above its record:
+the pass ran twice, once from inside a round-2 agent session.
+`codeverse.texturing.run.texture_requested(spec)` is now the single owner;
+`tracks.lifecycle.finalise` and the tool both ask it, and the tool refuses with a
+usage error (costing $0) in a run whose spec says no. `3dcv texture pass <slug>`
+is an explicit user instruction and is unaffected.
+
+### One live run per profile, same prompt
+
+`"a mid-century wooden dining chair with four tapered legs and a curved slatted
+backrest"`, static_object / blender, 2026-08-23 (during a flash 503 storm, so the
+minutes include provider backoff):
+
+| profile | baseline → final | rounds | cost | minutes | stop | judge |
+|---|---|---|---|---|---|---|
+| economy | 0.600 → **0.700** | 3 | **$0.3030** | 14.0 | max_rounds | flash n=2 |
+| balanced | 0.700 → **0.856** ✅ passed | 2 | **$0.5108** | 5.3 | pass | pro n=1 |
+| quality | 0.600 → **0.600** | 3 | **$1.8079** (ledger $1.9802) | 14.3 | diminishing_returns | pro n=3 |
+
+Every dial landed: economy really ran single-shot + flash at n=2 and stopped at 2
+refine rounds, quality really ran best-of-2 + pro at n=3 and shipped a texture
+pass (Δ materials +0.55, Δ overall +0.148 on the textured GLB). Economy came in
+at $0.303 against the $0.30 predicted from the audit; balanced at $0.51 against
+$1.47 (this prompt passed at round 1, the audit's median run does not).
+
+**The honest result is that quality lost this one**: it cost 3.5× balanced and
+finished 0.256 lower. Two mechanisms, both visible in the numbers — best-of-2
+picked a 0.600 baseline where balanced's single baseline happened to be 0.700, and
+a pro judge at n=3 has less noise to be lucky with (its three verdicts were
+0.600 / 0.600 / 0.600). **n=1 run per profile is an existence check, not an
+experiment**: it proves the dial is wired end to end, and it says nothing
+significant about which profile scores better. The audit's per-arm numbers (§8)
+remain the quality evidence.
+
+One number worth carrying forward: at `n=3` the judge is **31.5 % of that run's
+bill** ($0.62 of $1.98, 11 calls) against 6.0 % at `n=1` on the balanced run —
+§8's "keep the pro judge" is about *which* judge, not about how many samples of
+it, and `n=3` is a calibration setting, not a production one.
+
+Its ledger is also **$0.172 larger than `record.total_usage` (+9.5 %)**: the
+texture pass ran twice — once as a tool call inside the round-2 agent session,
+once post-loop — and the post-loop pass is outside the run total. The ledger made
+that visible per call instead of invisible (§6); the *second* pass should never
+have happened, and cannot now: `texture_requested()` is the single owner of the
+decision and the tool refuses in a run that did not ask (above). These three
+runs also predate the caveats above — they were run with the (now reverted)
+prompt reordering and with economy's 1-crop payload, so the economy row's judge
+payload and all three runs' prompt sizes differ slightly from what ships today.
+
+## 16. Price maintenance
+
+`3dcv cost prices` prints the table with provenance and now **flags** every row it
+cannot stand behind: `stale>90d` (checked date older than the window),
+`approximate` (the `Price.approximate` flag) and the non-`verified` statuses
+(`inferred` / `unverified`). `--stale` shows only flagged rows, `--days N` moves
+the window, `--unverified` keeps the old status filter. Today: 52 rows, 6 flagged
+(`claude-haiku-4`, `o1-mini`, `gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.2-codex`,
+`gemini-2.5-flash-image`), none stale.
+
+Two tests hold the line: `test_every_recorded_model_id_resolves_to_a_price` (no
+silent $0 — a model that billed us in any recorded run must have a row) and
+`test_every_price_row_was_checked_within_the_maintenance_window` (re-read the
+providers' pages and bump `CHECKED`; never raise the threshold). Every ledger row
+carries the provenance of the price it used, so a dollar can always be traced to a
+row and a date.
+
+## 17. Two round-level controls that did not survive their own measurement
+
+Wave 2 landed four round-level controls. Two of them were re-measured, failed,
+and are gone; this section is the evidence, so nobody re-adds them from §5/§9.
+
+### The 28-turn agent cap — A/B'd, rejected, removed
+
+§4 counted 39.3 % of the agent bill at turn ≥ 20 and 19 of 33 long sessions cut
+off by their own budget, and estimated $14.29 net for a graceful 28-turn cap
+(`waste-and-accounting/turn_cap_report.txt`). That counterfactual assumed the
+turns past 28 buy nothing. A controlled A/B says they do:
+
+| arm | run 1 | run 2 | run 3 | **mean $** | **mean score** |
+|---|---|---|---|---|---|
+| `agent_max_turns=28` | $1.430 / 0.354 | $1.668 / 0.600 | $1.047 / 0.482 | **$1.381** | **0.479** |
+| uncapped (backend default) | $1.001 / 0.852 | $1.603 / 0.600 | $1.477 / 0.600 | **$1.360** | **0.684** |
+
+Same prompt (an articulated desk lamp), same generator
+(`api-agent:gemini:gemini-3.7-flash`), same judge (`gemini-3.1-pro-preview`),
+same budget and box; 3 runs per arm; scripts and run dirs in
+`verifier/ab_uncapped.py` + `verifier/live/vstatic_hard_lamp + vlamp_cap60*`. The cap **cost $0.02 more
+and 0.205 of a score point**: a session stopped at turn 28 leaves work the next
+round pays for again, and the wrap-up session it buys is not free either.
+
+So there is no default cap: `RoundPolicy.agent_max_turns = 0` and
+`tracks.generation.DEFAULT_AGENT_MAX_TURNS = 0` leave `AgentJob.max_turns` at the
+backend's own default. The plumbing stays for callers who choose one —
+`GenerationTask.max_turns` > `generate(max_turns=…)` > `$CV3D_AGENT_MAX_TURNS` >
+`Settings.limits.agent_max_turns` — and a cap that IS set still lands gracefully
+(wrap-up session, `generate.turn_cap` event). **Nothing sets one by default any
+more**: the profiles independently dropped their own caps in the same wave
+(`cost/profiles.py`, all three at `max_turns=0`), so a cap now only exists when a
+run, a bench arm or `$CV3D_AGENT_MAX_TURNS` asks for it by name.
+
+### `skip_judge_reason` — two of four branches removed
+
+A skip only saves money if the verdict is never bought. Reproductions in
+`verifier/test_rejudge_defer.py`:
+
+| branch | verdict | why |
+|---|---|---|
+| `no file change` | **removed** | unreachable: a generation result is `ok` only when a file changed, so `run_generation_tasks` raises `RoundFailed` (→ the loop's `no_change` plateau) before any judge question is asked |
+| `build not repaired within the repair budget` | **removed** | a broken build never reaches the judge at all (`run_round` judges only when `build.ok`), so it only ever fired on a round that BUILT but still lint-failed — and the loop's `rejudge_round` then bought the same verdict one iteration later. Measured on the reproduction: 3 rounds "skipped", 2 verdicts re-bought as `judge.retry`, and the last round left **without a score — it was the best of the three (0.7)** and could not be promoted |
+| `budget already exceeded` | **kept** | the loop's next `budget_ok` check ends the run, so the verdict is never bought later. Guard, not a measured saving: 0 of the 108 recorded rounds bought a verdict after the budget ended, so this branch has saved $0 so far — it is kept because the state is knowable in advance and the run is over either way (§5) |
+| `no judge` / `no renders` / `gate errors` | **kept** | pre-existing; the first two are guards (nothing to buy), the third only fires when a caller sets `judge_on_gate_errors=False` |
+
+Two changes were needed to make the kept branches real:
+
+1. `rejudge_round` re-buys a verdict only after the judge was *tried* and failed
+ or came back degraded — never one that policy deliberately skipped. Otherwise
+ every skip is a `judge.retry` with the same price tag.
+2. `BestSelector` never promotes a round that did not build, and when no round has
+ a score at all it falls back to the built round with the **fewest gate errors**
+ (later on a tie) instead of simply the last one — an unjudged round that broke
+ the gates can no longer displace the clean artifact that came before it.
+
+The two controls that DID survive are unchanged and re-verified after this
+correction: the regression "switch then stop" rule and the r03+ marginal stop
+still cut 2 runs / **$1.14** with **0 best rounds lost** over the 107 recorded
+rounds (`waste-and-accounting/replay_stops.py`), and every round still emits its
+`cost.round` event.
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index a17ca2ea..667a46c2 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -77,6 +77,56 @@ Rules that callers must know:
(`-nano`, `-mini`, `-codex`, `-lite`) never inherit the parent's price — they are
unknown (0.0 + warning) unless a row exists.
+## cost/ (ledger · profiles · price provenance)
+```python
+from codeverse.cost import record_call, load_ledger, summarise, open_run_ledger
+record_call(usage, *, run="", round=None, stage=None, role=None, label="", backend="", model="",
+ outcome="ok", latency_ms=None, n_calls=1, source="live", ledger=None, reprice=False) -> CallCost
+ # Δ everything left out is resolved from the ambient context + the call label (cost.context);
+ # unknown model -> $0 and price_source="unknown" (flagged, never silently dropped); never raises.
+from codeverse.cost.instrument import (run_ledger, metered_chat_model, metered_agent,
+ per_call_metering, meters_own_calls, IN_PROCESS_AGENT_KINDS)
+with run_ledger(ws.root, run=slug): # binds the run, points record_call at <run>/telemetry/cost.jsonl
+ ... # (+ a <run>/cost_ledger.jsonl symlink for the run-layout alias)
+ # NESTS (restores the outer ledger + run binding on exit) and is context-local first, so
+ # bench.run_bench / compare_backends can hold one ledger per prompt across N threads
+ # models.get_chat_model returns a MeteredChatModel → ONE row per ChatModel.generate
+ # agents.get_coding_agent returns a MeteredAgent → ambient round/stage for the session, the
+ # profile's turn cap (only ever LOWERS job.max_turns), and — iff NOT meters_own_calls(agent),
+ # i.e. the backend is not in IN_PROCESS_AGENT_KINDS = {"api-agent"} — one session row.
+ # The rule is the BACKEND, never "did a row get written while it ran": a CLI session with one
+ # in-process tool call used to be dropped entirely, an in-process session whose turns ran in
+ # another thread used to be counted twice. A backend may declare `meters_own_calls`.
+per_call_metering() -> bool # aggregate writers (BudgetGuard) skip their row while True
+from codeverse.cost.context import CallContext, call_context, bind_run, context_from_label, SELF_DESCRIBING
+ # precedence: explicit > a label naming a job of its OWN (SELF_DESCRIBING = plan/judge/pairwise/
+ # texture/caption) > ambient (call_context) > the rest of the label > nothing. A judge or a
+ # texture pass that bills a model INSIDE an agent session is filed under its own stage, not the
+ # session's; a generation label yields to the session (best-of-N: kind="candidate" beats
+ # label="baseline"); Stage.OTHER means "the label said nothing" and displaces nothing.
+ # labels understood: api-agent:<job label>:t<turn> · judge:<rubric>:r<NN>:s<k> · planner · pairwise:… · texture… · caption…
+from codeverse.cost.caching import Block, order_blocks, prefix_report, session_cache, session_key
+session_cache(rows) -> [SessionCache] # per session: cold first call, cached share, saved_usd, cold_usd
+from codeverse.cost.profiles import get_profile, PROFILES # economy | balanced | quality
+get_settings().apply_profile(name, *, force=False) -> Profile
+ # sets default_{generator,planner,judge,captioner}, default_candidates, Settings.judge
+ # (max_px/montages/detail_crops/samples) and limits.agent_max_turns; a value the user stated in
+ # config.yaml / CV3D_* survives unless force (3dcv make --profile forces).
+from codeverse.cli._common import resolve_dial, ResolvedDial # THE resolver, one per `3dcv make`
+resolve_dial(settings, profile_flag=None, *, rounds=None, candidates=None, max_usd=None,
+ max_minutes=None, texture=False) -> ResolvedDial
+ # profile/generator/planner/judge/captioner, judge_samples/judge_max_px/judge_montages/
+ # judge_detail_crops, agent_max_turns, rounds, candidates, texture, max_usd, max_minutes.
+ # `--profile X` and `CV3D_PROFILE=X` resolve to the SAME dial (they used to disagree on
+ # candidates + texture); an explicit flag beats both. Spec.options.profile always records the
+ # resolved name so `3dcv resume` re-applies it.
+from codeverse.models.pricing import price_provenance # (provider, model) -> PriceRow(price, match, status, checked)
+```
+`3dcv cost <slug|path>…` · `3dcv cost --runs-dir <root>` · `3dcv cost cache <slug>` ·
+`3dcv cost prices [--stale] [--days N] [--unverified]` · `3dcv cost profiles` · `3dcv cost estimate`.
+A run with a live ledger is read from it (`RunLedger.source == "live"`); older runs are
+reconstructed, so all 61 recorded runs keep auditing.
+
## agents/
```python
from codeverse.agents import get_coding_agent # (agent_id) -> CodingAgent .run(job) .available() .id .kind .model
@@ -223,10 +273,26 @@ from codeverse.tracks import get_track
rec = get_track(spec.track, **options).run(spec, ws, resume=False) -> RunRecord # Δ kwargs forwarded to the constructor:
# services=, judge=, agent=, model=, runtime=, policy=RoundPolicy, settings=, planner_model=, n_candidates=
# (CLI --candidates > run_state.extra > settings.default_candidates); StaticObject | Articulated | Scene | Graphics
-from codeverse.orchestrator.rounds import RoundPolicy, StopPolicy, BestSelector, build_refine_instructions, compact_instructions
+from codeverse.orchestrator.rounds import RoundPolicy, StopPolicy, StopDecision, BestSelector, judge_sigma, \
+ best_score, last_gain, REWRITE_KIND, build_refine_instructions, compact_instructions
RoundPolicy(max_rounds=4, plateau_window=2, min_delta=0.02, target=0.8, judge_on_gate_errors=True, max_refine_tasks=6,
max_instructions_per_task=6, parallel_min_tasks=2, n_candidates=1, pairwise_margin=0.03,
- pairwise_min_confidence=0.6, judge_samples=1).with_candidates(n)
+ pairwise_min_confidence=0.6, judge_samples=1,
+ judge_model="", regression_sigma=1.0, regression_allow_switch=True, # money stops (docs/COST.md §5)
+ marginal_sigma=1.5, marginal_from_round=3, # r03+ must beat 1.5σ
+ agent_max_turns=0, agent_wrapup_turns=6 # 0 = UNCAPPED: a 28-turn cap was A/B'd
+ # and rejected (+$0.02, −0.21 score, docs/COST.md §17); wrapup applies to a cap a caller sets
+ ).with_candidates(n).with_judge(spec.backends.judge)
+policy.sigma / .regression_delta / .marginal_delta # judge_sigma() reads cost.routing.JUDGE_NOISE — THE σ table
+StopPolicy(policy).evaluate(history, budget_ok=True) -> StopDecision(reason, strategy="same"|"switch", detail)
+ # .decide(...) -> StopReason is unchanged; strategy "switch" = ONE whole-artifact rewrite round (kind REWRITE_KIND)
+from codeverse.orchestrator.budget import BudgetGuard, usage_delta
+BudgetGuard(budget, *, soft_fraction=1.0, run="", ledger=None)
+ .spend(usage, *, stage="other", role=None, label="", round_index=None, outcome="ok", enforce=True)
+ # THE door every dollar goes through: accumulate → bucket by stage/round → one priced ledger row
+ # (skipped when cost.instrument.per_call_metering() already writes them) → enforce the ceilings.
+ # charge(...) = spend(enforce=True); add(...) = spend(enforce=False) — "not enforced" never means "not seen".
+ .by_stage / .by_round / .round_costs(i) / .stage_summary() / .mark() # what a round burned, live
from codeverse.orchestrator.candidates import CandidateRecord, rank_candidates, decide_best # pure decision logic
from codeverse.tracks.candidates import run_best_of_n, choose_best_round # N parallel baselines in <ws>/_cand/c<k>
# (quick 4-view judge, crashed candidate retried once; selection by build_ok → quick score → fewer gate errors, pairwise
@@ -236,8 +302,22 @@ from codeverse.tracks.reference import silhouette_gate, reference_refine_tasks
from codeverse.tracks.prompting import base_prompt_context, reference_images, file_for_target_factory # Δ split out of
from codeverse.tracks.common import RunContext, Services # common.py (lazy re-exports keep old imports working)
from codeverse.tracks.generation import generate, run_agent_task, parse_multifile, is_single_shot
-from codeverse.tracks.repair import build_with_repair
-from codeverse.tracks.steps import run_round
+generate(ws, *, agent_id, task, ..., budget=BudgetGuard, max_turns=0, wrapup_turns=6) -> GenerationResult
+ # GenerationResult adds turns / sessions / turn_capped. A turn cap is applied ONLY if a caller
+ # asks: task.max_turns > max_turns > $CV3D_AGENT_MAX_TURNS > settings.limits.agent_max_turns >
+ # DEFAULT_AGENT_MAX_TURNS (0 = leave AgentJob.max_turns at the backend's own default — a 28-turn
+ # default was measured and rejected, docs/COST.md §17). A session that hits a cap that IS set is
+ # asked for a final build + summary (WRAPUP_PROMPT) instead of being killed.
+ # EVERY session (attempt 1, <label>.a2 retry, <label>.wrapup) is charged as it ends.
+from codeverse.tracks.repair import build_with_repair # RepairOutcome(.ok/.repaired/.max_attempts, attempts, usage)
+from codeverse.tracks.steps import run_round, skip_judge_reason, emit_round_cost, record_aborted_round
+skip_judge_reason(ctx, *, gates, renders) -> str # "" = judge it. ONLY states where the verdict is
+ # never bought at all: no judge / no renders / budget already exceeded / gate errors with
+ # policy.judge_on_gate_errors=False. rejudge_round honours the last one too, so a policy skip is
+ # never re-bought (docs/COST.md §17 — "no file change" and "build not repaired" were removed)
+run_round(ctx, *, index, kind, tasks, pipeline, ..., previous_best=None) -> RoundRecord
+ # emits cost.round {stages{}, judge_usd, total_usd, agent_turns, wasted, waste_reason}; on ANY exception it
+ # records what the round burned (rounds/aborted_rNN.json, ctx.extra["aborted_rounds"]) and re-raises
from codeverse.tracks.planner import plan, plan_model_for, ensure_acceptance # graphics uses tracks/graphics_steps.plan_graphics
```
`run_round` = generate → commit → `build_with_repair` → measure → gates → render →
@@ -250,7 +330,11 @@ generation prompts.
## texturing/ (derived asset pack; code stays truth — object.glb is never touched)
```python
-from codeverse.texturing.run import texture_pass, load_report
+from codeverse.texturing.run import texture_pass, texture_requested, load_report
+texture_requested(spec) -> bool # THE owner of "does this run texture?" (Spec.options.texture, or
+ # the legacy "texture" tag). Asked by tracks.lifecycle.finalise AND by the texture_pass spatial
+ # tool, which refuses in a run that did not ask — the tool is registered for every object track,
+ # so `texture: false` used to be bypassable from inside an agent session.
texture_pass(ws, spec, plan, *, model_id, image_model=None, judge=True, judge_model_id=None, rubric=None,
glb_in=None, views=OBJECT_VIEWS_QUICK, size=1024, ..., events=None, update_record=True) -> TextureReport
# 1 vision call material plan (cacheable) → tileable textures (mirror cross-fade, seam_score ≤ 0.08) → world-metre UV
@@ -277,7 +361,8 @@ from codeverse.flywheel.gallery import write_gallery, gallery_items, render_gall
# thumbs); bench/report.py reuses it
from codeverse.flywheel.index import build_index, query, summary # sqlite + parquet: adds quality_tier, gate_errors, cost_usd,
# rounds, status, code_fingerprint, prompt_hash, duplicate_of, has_captions
-3dcodeverse make|resume|status|render|judge|tools|mcp|texture {pass,scene-pack,show}
+3dcodeverse make [--profile economy|balanced|quality]|resume|status|show|render|judge|tools|mcp
+ |texture {pass,scene-pack,show}|cost {<slug>,show,cache,prices,profiles,estimate}
|flywheel {export,pairs,caption,index,dedupe,gallery}|bench {run,report}|doctor # alias: 3dcv
```
@@ -290,7 +375,15 @@ Event names: `run.start`, `stage.start/done`, `plan.done`, `workspace.materializ
`candidate.start/done/failed/retry/selected`, `pairwise.done`,
`texture.start/plan/generated/applied/gate/done`, `budget.exceeded`,
`finalise.rebuild`, `stop`, `run.done` / `run.failed`.
+Cost events: **`cost.round`** (per round: `stages{stage → $}`, `judge_usd`, `total_usd`,
+`agent_turns`, `score`, `previous_best`, `wasted`, `waste_reason` ∈ aborted | build_failed |
+unjudged | regression | zero_delta, `run_usd`) — emitted for aborted rounds too;
+`judge.skipped` (reason), `generate.turn_cap` (label, max_turns, turns, cost_usd),
+`strategy.switch` (regression → whole-artifact rewrite), `budget.overrun` (a post-loop
+texture pass crossed the ceiling; the loop is already finished, so not `budget.exceeded`).
`RunRecord` (record.json): spec, plan, workspace, status, rounds[RoundRecord],
best_round, baseline_score, final_score, total_usage, environment,
prompt_hashes{contract, cookbook, generate, refine}, error, extra{stop_reason,
-rubric, budget, rounds_summary, n_candidates?, candidates?, texturing?, captions?}.
+rubric, budget, cost_by_stage, aborted_rounds?, rounds_summary, n_candidates?,
+candidates?, texturing?, captions?}. `total_usage` is the BudgetGuard total whenever it
+exceeds the sum of the rounds (aborted rounds, retried sessions, texture pass).
diff --git a/harness/tests/compare_bench/test_compare_matrix.py b/harness/tests/compare_bench/test_compare_matrix.py
index 5d5640a6..7c1f7437 100644
--- a/harness/tests/compare_bench/test_compare_matrix.py
+++ b/harness/tests/compare_bench/test_compare_matrix.py
@@ -211,3 +211,55 @@ def test_redo_status_reuses_recorded_answers(tmp_path: Path):
r3 = next(r for r in rows3 if r.arm == "oneshot:claude-code")
assert r3.status == "no_code" and "empty" in r3.error
assert not (out / "cells" / "cmp_easy_stool" / "oneshot_claude-code" / "gen" / "attempt0" / "result.json").exists()
+
+
+def test_every_cell_and_its_harness_run_open_a_ledger(tmp_path: Path):
+ """compare_backends is a run-producing path: its cells must be priced into the
+ tree, not into the per-process fallback log (docs/COST.md §12). The harness arm
+ gets a nested ledger for its own run; the cell's own ledger holds the spend that
+ sits OUTSIDE that run (the fixed evaluator's judge) — the §6 gap."""
+ from codeverse.contracts.chat import ChatMessage, ChatRequest, ChatResponse
+ from codeverse.contracts.common import Usage
+ from codeverse.cost.instrument import MeteredChatModel
+ from codeverse.cost.ledger import load_ledger
+
+ class FakeChat:
+ provider, model, id = "gemini", "gemini-3.7-flash", "gemini:gemini-3.7-flash"
+
+ def supports_vision(self) -> bool:
+ return True
+
+ def generate(self, request: ChatRequest) -> ChatResponse:
+ return ChatResponse(text="ok", usage=Usage(backend="gemini", model="gemini-3.7-flash",
+ input_tokens=1000, output_tokens=10))
+
+ def bill(label: str) -> None:
+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label=label))
+
+ ev = FakeEvaluator()
+ inner_eval = ev.evaluate
+
+ def evaluate(ws, spec): # the fixed evaluator buys a verdict
+ bill("judge:static_object_v1:r00:s0")
+ return inner_eval(ws, spec)
+
+ ev.evaluate = evaluate # type: ignore[method-assign]
+ inner_track = fake_run_track(0.9)
+
+ def run_track(spec, ws, resume): # the harness run spends inside its own ledger
+ bill("api-agent:baseline:t0")
+ return inner_track(spec, ws, resume)
+
+ deps = _deps(ev, {"claude-code": FakeBackend(["```python\n" + GOOD.format(score=0.6) + "```"])}, run_track)
+ out = tmp_path / "cmp"
+ opts = CompareOptions(judge="gemini:fixed", limit=1, parallel=2, pairwise=False)
+ rows = run_matrix(BATTERY, out, parse_arms("harness:api-agent:gemini:gemini-3.7-flash,oneshot:claude-code"),
+ opts, deps)
+ assert len(rows) == 2
+ cells = {r.arm: Path(r.workspace) for r in rows}
+ harness = cells["harness:api-agent:gemini:gemini-3.7-flash"]
+ assert [r.label for r in load_ledger(harness / "run")] == ["api-agent:baseline:t0"]
+ assert [r.label for r in load_ledger(harness)] == ["judge:static_object_v1:r00:s0"]
+ oneshot = cells["oneshot:claude-code"]
+ assert not (oneshot / "run").exists()
+ assert [r.label for r in load_ledger(oneshot)] == ["judge:static_object_v1:r00:s0"]
diff --git a/harness/tests/cost/test_cli.py b/harness/tests/cost/test_cli.py
index c617772f..6ccf0a38 100644
--- a/harness/tests/cost/test_cli.py
+++ b/harness/tests/cost/test_cli.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
from pathlib import Path
from typer.testing import CliRunner
@@ -48,3 +49,70 @@ def test_bench_cost_report_entry_point(fake_run: Path, tmp_path: Path):
assert main([str(fake_run), "--out", str(out), "--quiet"]) == 0
assert "# Cost report" in out.read_text()
assert main([str(tmp_path / "nothing"), "--quiet"]) == 2
+
+
+# --------------------------------------------------------------------- the live ledger
+def _live_ledger(run: Path, rows: list[dict]) -> None:
+ p = run / "telemetry" / "cost.jsonl"
+ p.parent.mkdir(parents=True, exist_ok=True)
+ p.write_text("".join(json.dumps(r) + "\n" for r in rows))
+
+
+def test_cost_takes_a_bare_run_path_or_slug(fake_run: Path):
+ r = runner.invoke(app, ["cost", str(fake_run)])
+ assert r.exit_code == 0, r.output
+ assert "reconciliation" in r.output and "reconstructed" in r.output
+ r = runner.invoke(app, ["cost", fake_run.name, "--runs-dir", str(fake_run.parent)])
+ assert r.exit_code == 0 and "reconciliation" in r.output
+
+
+def test_cost_runs_dir_aggregates_a_battery(fake_run: Path, tmp_path: Path):
+ r = runner.invoke(app, ["cost", "--runs-dir", str(fake_run.parent)])
+ assert r.exit_code == 0 and "runs: 1" in r.output
+
+
+def test_a_live_ledger_is_read_instead_of_being_reconstructed(fake_run: Path):
+ """The run wrote its own rows while it ran: they ARE the ledger."""
+ _live_ledger(fake_run, [
+ {"run": fake_run.name, "round": 0, "stage": "baseline", "role": "generator",
+ "backend": "gemini", "provider": "gemini", "model": "gemini-3.7-flash",
+ "input_tokens": 30_000, "cached_tokens": 10_000, "output_tokens": 300,
+ "cost_usd": 0.0201, "price_source": "exact"},
+ {"run": fake_run.name, "round": 0, "stage": "judge", "role": "judge",
+ "backend": "gemini", "provider": "gemini", "model": "gemini-3.1-pro-preview",
+ "input_tokens": 12_000, "output_tokens": 1_000, "cost_usd": 0.0999, "price_source": "exact"},
+ ])
+ r = runner.invoke(app, ["cost", str(fake_run)])
+ assert r.exit_code == 0, r.output
+ assert "1 live / 0 reconstructed" in r.output
+ assert "$0.0999" in r.output # the ledger's own judge dollar, not the reconstruction's $0.036
+ # what the ledger does not explain is reported, never hidden
+ assert "unattributed" in r.output or "difference" in r.output
+
+
+def test_cost_cache_reports_the_cold_head_of_every_session(fake_run: Path):
+ _live_ledger(fake_run, [
+ {"run": "x", "round": 0, "stage": "baseline", "label": "api-agent:baseline:t0",
+ "provider": "gemini", "model": "gemini-3.7-flash", "input_tokens": 12_000,
+ "cached_tokens": 0, "cost_usd": 0.009, "price_input": 0.75, "price_cached": 0.075},
+ {"run": "x", "round": 0, "stage": "baseline", "label": "api-agent:baseline:t1",
+ "provider": "gemini", "model": "gemini-3.7-flash", "input_tokens": 20_000,
+ "cached_tokens": 12_000, "cost_usd": 0.007, "price_input": 0.75, "price_cached": 0.075},
+ ])
+ r = runner.invoke(app, ["cost", "cache", str(fake_run)])
+ assert r.exit_code == 0, r.output
+ assert "r0:baseline" in r.output and "12,000" in r.output
+ assert "cold heads cost" in r.output
+
+
+def test_cost_prices_flags_stale_and_approximate_rows():
+ r = runner.invoke(app, ["cost", "prices", "--stale"])
+ assert r.exit_code == 0
+ assert "flags" in r.stdout and ("approximate" in r.stdout or "none older than" in r.stdout)
+ r = runner.invoke(app, ["cost", "prices", "--days", "-1"])
+ assert r.exit_code == 0 and "stale>-1d" in r.stdout # nothing is younger than -1 days
+
+
+def test_cost_show_still_works_the_long_way(fake_run: Path):
+ r = runner.invoke(app, ["cost", "show", str(fake_run), "--no-per-run"])
+ assert r.exit_code == 0 and "per stage" in r.output.lower() or "stage:" in r.output
diff --git a/harness/tests/cost/test_context.py b/harness/tests/cost/test_context.py
new file mode 100644
index 00000000..e80f7812
--- /dev/null
+++ b/harness/tests/cost/test_context.py
@@ -0,0 +1,75 @@
+"""Call attribution: explicit > a label naming its own job > ambient > the label."""
+
+from __future__ import annotations
+
+from codeverse.cost.context import (
+ CallContext,
+ attribute,
+ bind_run,
+ call_context,
+ context_from_label,
+ current,
+)
+from codeverse.cost.types import Role, Stage
+
+
+def test_label_tells_stage_role_and_round():
+ c = context_from_label("api-agent:refine_drip_tray:t7")
+ assert c.stage is Stage.REFINE and c.role is Role.GENERATOR
+ c = context_from_label("judge:static_object_v1:r02:s1")
+ assert c.stage is Stage.JUDGE and c.role is Role.JUDGE and c.round == 2
+ assert context_from_label("planner-retry").stage is Stage.PLAN
+ assert context_from_label("pairwise:0v1:fwd").stage is Stage.PAIRWISE
+ assert context_from_label("texture_plan").stage is Stage.TEXTURE
+ assert context_from_label("captioner").role is Role.CAPTIONER
+ assert context_from_label("").stage is None
+
+
+def test_repair_label_wins_over_the_label_it_repairs():
+ assert context_from_label("api-agent:baseline_repair1:t0").stage is Stage.REPAIR
+
+
+def test_a_generation_label_yields_to_the_session_and_explicit_beats_both():
+ # best-of-N: the session knows it is a candidate, its turns only say "baseline"
+ with call_context(round=3, stage=Stage.CANDIDATE, label="cand_c1"):
+ got = attribute(label="api-agent:baseline:t0")
+ assert got.stage is Stage.CANDIDATE and got.round == 3
+ assert got.label == "api-agent:baseline:t0" # the most specific label is kept
+ forced = attribute(CallContext(stage=Stage.REPAIR, round=9), label="api-agent:baseline:t0")
+ assert forced.stage is Stage.REPAIR and forced.round == 9
+
+
+def test_a_call_that_names_its_own_job_beats_the_session_it_runs_inside():
+ """The verifier's finding: a spatial tool that bills a model inside a refine
+ session was recorded as stage=refine / role=generator."""
+ with call_context(round=2, stage=Stage.REFINE, role=Role.GENERATOR, label="refine_seat"):
+ tex = attribute(label="texture_plan")
+ assert tex.stage is Stage.TEXTURE and tex.role is Role.OTHER and tex.round == 2
+ gate = attribute(label="texture_gate")
+ assert gate.stage is Stage.TEXTURE and gate.role is Role.JUDGE
+ verdict = attribute(label="judge:static_object_v1:r02:s0")
+ assert verdict.stage is Stage.JUDGE and verdict.role is Role.JUDGE and verdict.round == 2
+ cap = attribute(label="caption:best")
+ assert cap.stage is Stage.CAPTION and cap.role is Role.CAPTIONER
+ # a label that names nothing keeps the session's attribution
+ anon = attribute(label="summarise")
+ assert anon.stage is Stage.REFINE and anon.role is Role.GENERATOR
+
+
+def test_nested_contexts_merge_and_unwind():
+ try:
+ bind_run("run-a")
+ assert current().run == "run-a"
+ with call_context(stage=Stage.BASELINE, round=0):
+ with call_context(label="inner"):
+ inner = current()
+ assert inner.stage is Stage.BASELINE and inner.round == 0 and inner.label == "inner"
+ assert current().label == ""
+ assert current().stage is None and current().run == "run-a"
+ finally:
+ bind_run("")
+
+
+def test_role_is_derived_from_stage_when_unstated():
+ assert attribute(CallContext(stage=Stage.ZONES)).role is Role.GENERATOR
+ assert attribute(CallContext(stage=Stage.PAIRWISE)).role is Role.JUDGE
diff --git a/harness/tests/cost/test_instrument.py b/harness/tests/cost/test_instrument.py
new file mode 100644
index 00000000..9de389e5
--- /dev/null
+++ b/harness/tests/cost/test_instrument.py
@@ -0,0 +1,293 @@
+"""Metering: every ChatModel.generate and every CodingAgent.run becomes a ledger row."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from codeverse.contracts.agent import AgentJob, AgentResult
+from codeverse.contracts.chat import ChatMessage, ChatRequest, ChatResponse
+from codeverse.contracts.common import Usage
+from codeverse.cost.instrument import (
+ MeteredAgent,
+ MeteredChatModel,
+ per_call_metering,
+ run_ledger,
+)
+from codeverse.cost.ledger import load_ledger
+from codeverse.cost.types import Role, Stage
+
+
+def _usage(**kw: object) -> Usage:
+ base = {"backend": "gemini", "model": "gemini-3.7-flash", "input_tokens": 12_000,
+ "cached_tokens": 8_000, "output_tokens": 300, "latency_ms": 1234}
+ base.update(kw)
+ return Usage(**base) # type: ignore[arg-type]
+
+
+class FakeChat:
+ provider = "gemini"
+ model = "gemini-3.7-flash"
+
+ def __init__(self, error: Exception | None = None):
+ self.error = error
+ self.requests: list[ChatRequest] = []
+
+ @property
+ def id(self) -> str:
+ return "gemini:gemini-3.7-flash"
+
+ def supports_vision(self) -> bool:
+ return True
+
+ def generate(self, request: ChatRequest) -> ChatResponse:
+ self.requests.append(request)
+ if self.error is not None:
+ raise self.error
+ return ChatResponse(text="ok", usage=_usage())
+
+
+class FakeAgent:
+ kind = "api-agent"
+ model = "gemini:gemini-3.7-flash"
+
+ def __init__(self, chat, turns: int = 3):
+ self.chat, self.turns, self.seen_turns = chat, turns, 0
+
+ @property
+ def id(self) -> str:
+ return f"{self.kind}:{self.model}"
+
+ def available(self) -> tuple[bool, str]:
+ return True, "ok"
+
+ def run(self, job: AgentJob) -> AgentResult:
+ self.seen_turns = job.max_turns
+ total = Usage()
+ for t in range(self.turns):
+ r = self.chat.generate(ChatRequest(messages=[ChatMessage.user("x")],
+ label=f"api-agent:{job.label}:t{t}"))
+ total = total + r.usage
+ return AgentResult(ok=True, exit_reason="completed", usage=total, tool_calls=self.turns)
+
+
+class CliAgent(FakeAgent):
+ kind = "gemini-cli"
+
+ def run(self, job: AgentJob) -> AgentResult:
+ self.seen_turns = job.max_turns
+ return AgentResult(ok=True, exit_reason="completed", tool_calls=4,
+ usage=_usage(backend="gemini-cli", input_tokens=50_000, cached_tokens=40_000))
+
+
+def test_a_metered_call_becomes_one_priced_row(tmp_path: Path):
+ chat = FakeChat()
+ with run_ledger(tmp_path, run="r1"):
+ resp = MeteredChatModel(chat).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r01:s0"))
+ assert resp.text == "ok"
+ (row,) = load_ledger(tmp_path)
+ assert row.run == "r1" and row.stage is Stage.JUDGE and row.role is Role.JUDGE and row.round == 1
+ assert row.input_tokens == 12_000 and row.cached_tokens == 8_000 and row.cache_hit is True
+ assert row.price_source == "exact" and row.price_input == 0.75 and row.price_checked
+ assert row.latency_ms == 1234 and row.cost_usd > 0
+
+
+def test_a_failed_call_is_recorded_and_re_raised(tmp_path: Path):
+ with run_ledger(tmp_path, run="r1"), pytest.raises(TimeoutError):
+ MeteredChatModel(FakeChat(TimeoutError("boom"))).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="planner"))
+ (row,) = load_ledger(tmp_path)
+ assert row.outcome == "timeout" and row.stage is Stage.PLAN and row.cost_usd == 0.0
+
+
+def test_api_agent_turns_are_metered_once_each_with_the_job_round(tmp_path: Path):
+ chat = MeteredChatModel(FakeChat())
+ with run_ledger(tmp_path, run="r1"):
+ MeteredAgent(FakeAgent(chat)).run(AgentJob(workspace=str(tmp_path), prompt="p", label="refine",
+ round=2, kind="refine"))
+ rows = load_ledger(tmp_path)
+ assert len(rows) == 3 # one per turn, no extra session row
+ assert {r.round for r in rows} == {2} and {r.stage for r in rows} == {Stage.REFINE}
+ assert {r.role for r in rows} == {Role.GENERATOR}
+
+
+def test_a_cli_agent_we_cannot_see_inside_gets_one_session_row(tmp_path: Path):
+ with run_ledger(tmp_path, run="r1"):
+ MeteredAgent(CliAgent(FakeChat())).run(
+ AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0, kind="baseline"))
+ (row,) = load_ledger(tmp_path)
+ assert row.source == "session" and row.backend == "gemini-cli" and row.n_calls == 4
+ assert row.stage is Stage.BASELINE and row.round == 0 and row.input_tokens == 50_000
+
+
+def test_the_turn_cap_only_ever_lowers(tmp_path: Path):
+ agent = FakeAgent(MeteredChatModel(FakeChat()))
+ job = AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", max_turns=60)
+ MeteredAgent(agent, max_turns=20).run(job)
+ assert agent.seen_turns == 20
+ MeteredAgent(agent, max_turns=90).run(job)
+ assert agent.seen_turns == 60 # a bigger cap never raises the caller's own
+
+
+def test_run_ledger_writes_telemetry_and_leaves_a_root_alias(tmp_path: Path):
+ assert per_call_metering() is False
+ with run_ledger(tmp_path, run="r1"):
+ assert per_call_metering() is True
+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label="planner"))
+ assert per_call_metering() is False
+ assert (tmp_path / "telemetry" / "cost.jsonl").is_file()
+ alias = tmp_path / "cost_ledger.jsonl"
+ assert alias.is_symlink() and alias.is_file() # resolves to the telemetry copy
+ assert len(load_ledger(tmp_path)) == 1 == len(load_ledger(alias))
+
+
+def test_accounting_never_breaks_a_call(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
+ def boom(*a: object, **k: object) -> None:
+ raise RuntimeError("ledger on fire")
+
+ monkeypatch.setattr("codeverse.cost.instrument.record_call", boom)
+ with run_ledger(tmp_path, run="r1"):
+ resp = MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")]))
+ assert resp.text == "ok"
+
+
+def test_the_proxy_forwards_everything_else():
+ chat = FakeChat()
+ chat.pool = "the-key-pool" # type: ignore[attr-defined]
+ m = MeteredChatModel(chat)
+ assert m.id == "gemini:gemini-3.7-flash" and m.provider == "gemini" and m.supports_vision()
+ assert m.pool == "the-key-pool"
+ agent = MeteredAgent(FakeAgent(chat))
+ assert agent.kind == "api-agent" and agent.available() == (True, "ok")
+
+
+def test_get_chat_model_hands_out_a_metered_model(monkeypatch: pytest.MonkeyPatch):
+ from codeverse.models import registry
+
+ monkeypatch.setattr(registry, "_build_chat_model", lambda mid: FakeChat())
+ registry.get_chat_model.cache_clear()
+ try:
+ assert isinstance(registry.get_chat_model("gemini:gemini-3.7-flash"), MeteredChatModel)
+ finally:
+ registry.get_chat_model.cache_clear()
+
+
+def test_post_hoc_work_never_creates_a_partial_ledger(tmp_path: Path):
+ """A re-judge of an old run must not leave a ledger holding only that verdict:
+ a reader would take it for the whole run's cost."""
+ with run_ledger(tmp_path, run="old", create=False) as led:
+ assert led is None
+ MeteredChatModel(FakeChat()).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0"))
+ assert not (tmp_path / "telemetry" / "cost.jsonl").exists()
+ assert not (tmp_path / "cost_ledger.jsonl").exists()
+ # but a run that already keeps one is appended to
+ with run_ledger(tmp_path, run="old"):
+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label="planner"))
+ with run_ledger(tmp_path, run="old", create=False) as led:
+ assert led is not None
+ MeteredChatModel(FakeChat()).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0"))
+ assert len(load_ledger(tmp_path)) == 2
+
+
+# ---------------------------------------------------- who meters the inside of a session
+class SideEffectCliAgent(CliAgent):
+ """A subscription CLI during whose session something else bills a model."""
+
+ def __init__(self, chat, side_effect):
+ super().__init__(chat)
+ self.side_effect = side_effect
+
+ def run(self, job: AgentJob) -> AgentResult:
+ self.side_effect()
+ return super().run(job)
+
+
+def test_a_cli_session_is_recorded_even_when_a_tool_bills_a_model_inside_it(tmp_path: Path):
+ """The old rule was "did anybody write a row while the session ran?", which a
+ single in-process call (a texture pass, a summariser) flipped — and the whole
+ CLI session, the only record of that money, was dropped. Reproduction:
+ scratchpad costfix/verifier/adversarial.py."""
+ def inner_model_call() -> None:
+ MeteredChatModel(FakeChat()).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="texture_plan"))
+
+ with run_ledger(tmp_path, run="r1"):
+ MeteredAgent(SideEffectCliAgent(FakeChat(), inner_model_call)).run(
+ AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0, kind="baseline"))
+ rows = load_ledger(tmp_path)
+ assert len(rows) == 2, [r.label for r in rows]
+ session = next(r for r in rows if r.source == "session")
+ assert session.backend == "gemini-cli" and session.input_tokens == 50_000
+ tool = next(r for r in rows if r.source != "session")
+ assert tool.stage is Stage.TEXTURE # and it is NOT filed under the session's stage
+
+
+def test_an_in_process_session_is_never_counted_twice_even_from_another_thread(tmp_path: Path):
+ """The mirror failure: the api-agent's turns are already rows, so its
+ AgentResult.usage must never be added on top — including when the turns ran in
+ a worker thread the row counter could not see."""
+ import threading
+
+ class ThreadedAgent(FakeAgent):
+ def run(self, job: AgentJob) -> AgentResult:
+ out: list[AgentResult] = []
+ t = threading.Thread(target=lambda: out.append(FakeAgent.run(self, job)))
+ t.start()
+ t.join()
+ return out[0]
+
+ with run_ledger(tmp_path, run="r1"):
+ MeteredAgent(ThreadedAgent(MeteredChatModel(FakeChat()))).run(
+ AgentJob(workspace=str(tmp_path), prompt="p", label="baseline", round=0, kind="baseline"))
+ rows = load_ledger(tmp_path)
+ assert len(rows) == 3 and not any(r.source == "session" for r in rows)
+ assert sum(r.input_tokens for r in rows) == 36_000 # 3 turns, not 3 turns + a session
+
+
+def test_a_backend_may_declare_that_it_meters_itself(tmp_path: Path):
+ from codeverse.cost.instrument import meters_own_calls
+
+ cli = CliAgent(FakeChat())
+ assert meters_own_calls(cli) is False and meters_own_calls(FakeAgent(FakeChat())) is True
+ cli.meters_own_calls = True # type: ignore[attr-defined]
+ assert meters_own_calls(cli) is True
+
+
+# ------------------------------------------------------------------- nesting / parallelism
+def test_run_ledgers_nest_and_restore_the_outer_one(tmp_path: Path):
+ """A bench cell opens a ledger for the cell and another for the harness run
+ inside it; the cell's must come back when the inner one closes."""
+ outer, inner = tmp_path / "cell", tmp_path / "cell" / "run"
+ with run_ledger(outer, run="cell"):
+ MeteredChatModel(FakeChat()).generate(ChatRequest(messages=[ChatMessage.user("x")], label="planner"))
+ with run_ledger(inner, run="cell:run"):
+ MeteredChatModel(FakeChat()).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="api-agent:baseline:t0"))
+ MeteredChatModel(FakeChat()).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="judge:static_object_v1:r00:s0"))
+ assert [r.stage for r in load_ledger(inner)] == [Stage.BASELINE]
+ assert [r.stage for r in load_ledger(outer)] == [Stage.PLAN, Stage.JUDGE]
+ assert {r.run for r in load_ledger(outer)} == {"cell"}
+ assert per_call_metering() is False
+
+
+def test_parallel_runs_in_their_own_threads_keep_their_own_ledgers(tmp_path: Path):
+ """`bench.run_bench` runs N prompts in N threads; their rows must not mix."""
+ from concurrent.futures import ThreadPoolExecutor
+
+ def one(name: str) -> None:
+ with run_ledger(tmp_path / name, run=name):
+ for _ in range(3):
+ MeteredChatModel(FakeChat()).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="api-agent:baseline:t0"))
+
+ names = [f"p{i}" for i in range(4)]
+ with ThreadPoolExecutor(max_workers=4) as pool:
+ list(pool.map(one, names))
+ for name in names:
+ rows = load_ledger(tmp_path / name)
+ assert len(rows) == 3 and {r.run for r in rows} == {name}
diff --git a/harness/tests/cost/test_live.py b/harness/tests/cost/test_live.py
index 117f95f1..e242d6d7 100644
--- a/harness/tests/cost/test_live.py
+++ b/harness/tests/cost/test_live.py
@@ -74,3 +74,33 @@ def test_estimate_is_within_15_percent_of_the_bill():
usage = _send(get_chat_model("gemini:gemini-3.7-flash"),
text + "\n\nReply with ONE JSON object {\"ok\": true}.")
assert abs(est.input_tokens - usage.input_tokens) / usage.input_tokens < 0.15
+
+
+# --------------------------------------------------------------- the metered run loop
+def test_a_metered_run_reconciles_with_its_own_record(tmp_path: Path):
+ """LIVE end-to-end: a real (cheap) run's ledger must equal record.total_usage.
+
+ This is the invariant docs/COST.md §12 reports at 0.000% on two fresh runs;
+ the threshold here is the 1% the wave asked for."""
+ from typer.testing import CliRunner
+
+ from codeverse.cli.main import app
+ from codeverse.cost.ledger import load_ledger
+ from codeverse.flywheel.record import load_record
+ from codeverse.workspace import Workspace
+
+ runs = tmp_path / "runs"
+ r = CliRunner().invoke(app, [
+ "make", "a smooth grey ceramic bowl", "--track", "static_object", "--language", "blender",
+ "--generator", "single-shot:gemini:gemini-3.7-flash", "--judge", "gemini:gemini-3.7-flash",
+ "--rounds", "0", "--max-usd", "1", "--max-minutes", "15",
+ "--runs-dir", str(runs), "--slug", "cost_live_bowl"])
+ assert r.exit_code in (0, 1), r.output # a failed judge/build is still a metered run
+ ws = Workspace(runs / "cost_live_bowl")
+ rows = load_ledger(ws.root)
+ assert rows and (ws.root / "telemetry" / "cost.jsonl").is_file()
+ assert all(row.run == "cost_live_bowl" for row in rows)
+ assert {row.stage.value for row in rows} >= {"plan", "baseline"}
+ recorded = load_record(ws).total_usage.cost_usd
+ ledger = sum(row.cost_usd for row in rows)
+ assert recorded > 0 and abs(ledger - recorded) / recorded < 0.01, f"{ledger} vs {recorded}"
diff --git a/harness/tests/cost/test_price_hygiene.py b/harness/tests/cost/test_price_hygiene.py
index 9892dac2..a7b0d4b4 100644
--- a/harness/tests/cost/test_price_hygiene.py
+++ b/harness/tests/cost/test_price_hygiene.py
@@ -119,3 +119,40 @@ def test_every_recorded_model_id_resolves_to_a_price():
unknown = sorted(f"{p}:{m}" for p, m in _recorded_model_ids()
if p and price_provenance(p, m).price is None)
assert not unknown, f"models with no price row (they cost $0 in every report): {unknown}"
+
+
+# --------------------------------------------------------------- staleness maintenance
+def test_every_price_row_was_checked_within_the_maintenance_window():
+ """A price nobody re-checked for 90 days is not evidence. When this fails,
+ re-read the providers' pricing pages, update PRICES/PROVENANCE and bump
+ ``CHECKED`` — do not raise the threshold."""
+ from datetime import date, datetime
+
+ from codeverse.cli.cost_cmd import STALE_AFTER_DAYS
+ from codeverse.models.pricing import PRICES, price_provenance
+
+ stale = []
+ for prov, model in sorted(PRICES):
+ row = price_provenance(prov, model)
+ try:
+ age = (date.today() - datetime.strptime(row.checked, "%Y-%m-%d").date()).days
+ except (TypeError, ValueError):
+ stale.append(f"{prov}:{model} (no checked date)")
+ continue
+ if age > STALE_AFTER_DAYS:
+ stale.append(f"{prov}:{model} ({age} days)")
+ assert not stale, f"price rows older than {STALE_AFTER_DAYS} days: {stale}"
+
+
+def test_a_ledger_row_carries_the_provenance_of_the_price_it_used(tmp_path):
+ """The audit trail: which row produced this dollar, and can we stand behind it."""
+ from codeverse.contracts.common import Usage as U
+ from codeverse.cost import record_call
+
+ row = record_call(U(backend="gemini", model="gemini-3.1-pro-preview", input_tokens=1_000,
+ output_tokens=100), run="r", stage="judge", ledger=tmp_path / "l.jsonl")
+ assert row.price_source == "exact" and row.price_approximate is False
+ assert row.price_checked and row.price_input == 2.0 and row.price_output == 12.0
+ unknown = record_call(U(backend="gemini", model="gemini-9.9-imaginary", input_tokens=1_000),
+ run="r", stage="judge", ledger=tmp_path / "l.jsonl")
+ assert unknown.price_source == "unknown" and unknown.price_approximate is True
diff --git a/harness/tests/cost/test_profiles.py b/harness/tests/cost/test_profiles.py
new file mode 100644
index 00000000..22f0deb7
--- /dev/null
+++ b/harness/tests/cost/test_profiles.py
@@ -0,0 +1,215 @@
+"""The economy / balanced / quality dial: one name, the whole cost/quality setting."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from codeverse.cli.main import app
+from codeverse.config import Settings, get_settings
+from codeverse.cost.profiles import PROFILE_NAMES, PROFILES, get_profile, profile_table
+
+runner = CliRunner()
+
+
[email protected](autouse=True)
+def _fresh_settings():
+ """A profile mutates the process-wide settings singleton; never leak that."""
+ get_settings.cache_clear()
+ yield
+ get_settings.cache_clear()
+
+
+def test_the_three_profiles_are_a_monotone_dial():
+ usd = [PROFILES[n].expected_usd for n in PROFILE_NAMES]
+ assert usd == sorted(usd) and usd[0] < usd[-1]
+ rounds = [PROFILES[n].rounds for n in PROFILE_NAMES]
+ assert rounds == sorted(rounds)
+ # the judge payload is NOT part of the dial: two independent draws of the
+ # 2-crop vs 1-crop experiment disagreed by 0.048 = 1.6x the pro judge's σ,
+ # so no profile trades a fifth of a cent for that (docs/COST.md §14)
+ assert {PROFILES[n].judge_detail_crops for n in PROFILE_NAMES} == {2}
+ assert {PROFILES[n].judge_max_px for n in PROFILE_NAMES} == {1024}
+ assert PROFILES["quality"].candidates == 2 and PROFILES["quality"].texture is True
+ assert PROFILES["economy"].texture is False and PROFILES["economy"].candidates == 1
+ # every profile explains itself and cites what it was measured at
+ for p in PROFILES.values():
+ assert p.expected_score and p.note and p.generator and p.judge
+
+
+def test_balanced_is_todays_defaults():
+ p = PROFILES["balanced"]
+ s = Settings()
+ assert (p.generator, p.planner, p.judge) == (s.default_generator, s.default_planner, s.default_judge)
+ assert p.rounds == 4 and p.candidates == s.default_candidates and p.judge_samples == 1
+ assert p.max_usd == 5.0 and p.max_minutes == 60.0 and p.judge_max_px == 1024
+
+
+def test_economy_is_single_shot_flash_and_two_rounds_with_no_turn_cap_of_its_own():
+ p = PROFILES["economy"]
+ assert p.generator.startswith("single-shot:") and "flash" in p.judge
+ assert p.rounds == 2 and p.judge_samples == 2
+ # a single-shot generator opens no agent session, so economy's old 20-turn cap
+ # could never fire — and the cap it was modelled on lost its own A/B (+$0.02,
+ # −0.205 score, docs/COST.md §17), so no profile sets one
+ assert p.max_turns == 0 and all(PROFILES[n].max_turns == 0 for n in PROFILE_NAMES)
+
+
+def test_apply_profile_sets_the_whole_dial():
+ s = Settings()
+ p = s.apply_profile("quality")
+ assert p.name == "quality" and s.profile == "quality"
+ assert s.default_judge == p.judge and s.default_candidates == 2
+ assert s.judge.samples == 3 and s.judge.max_px == 1024
+ s2 = Settings()
+ s2.apply_profile("economy")
+ assert s2.judge.samples == 2 and s2.limits.agent_max_turns == 0
+ # payload untouched: 768 px bills the same on Gemini and is noisier, and the
+ # crop cut did not survive a second draw (docs/COST.md §14)
+ assert s2.judge.detail_crops == 2 and s2.judge.max_px == 1024
+
+
+def test_a_value_the_user_configured_survives_the_profile_unless_forced():
+ s = Settings(default_generator="codex:gpt-5.6-sol")
+ s.apply_profile("economy")
+ assert s.default_generator == "codex:gpt-5.6-sol" # config.yaml / CV3D_* wins over a default dial
+ assert s.default_judge == "gemini:gemini-3.7-flash" # everything unstated still moves
+ s.apply_profile("economy", force=True)
+ assert s.default_generator == "single-shot:gemini:gemini-3.7-flash"
+
+
+def test_unknown_profile_is_a_clear_error():
+ with pytest.raises(ValueError, match="unknown profile"):
+ get_profile("cheapest")
+
+
+def test_profile_table_and_cli():
+ rows = profile_table()
+ assert len(rows) == 3 and rows[0][0] == "economy"
+ r = runner.invoke(app, ["cost", "profiles"])
+ assert r.exit_code == 0 and "economy" in r.stdout and "quality" in r.stdout
+
+
+def test_make_profile_writes_the_whole_shape_onto_the_spec(tmp_path: Path):
+ runs = tmp_path / "runs"
+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run", "--profile", "quality"])
+ assert r.exit_code == 0, r.output
+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())
+ assert spec["options"] == {"candidates": 2, "texture": True, "profile": "quality"}
+ assert spec["budget"]["max_rounds"] == 4 and spec["budget"]["max_usd"] == 8.0
+ assert spec["backends"]["judge"] == "gemini:gemini-3.1-pro-preview"
+
+
+def test_an_explicit_flag_beats_the_profile(tmp_path: Path):
+ runs = tmp_path / "runs"
+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run",
+ "--profile", "economy", "--rounds", "4", "--max-usd", "9",
+ "--generator", "api-agent:gemini:gemini-3.7-flash"])
+ assert r.exit_code == 0, r.output
+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())
+ assert spec["budget"]["max_rounds"] == 4 and spec["budget"]["max_usd"] == 9.0
+ assert spec["backends"]["generator"] == "api-agent:gemini:gemini-3.7-flash"
+ assert spec["backends"]["judge"] == "gemini:gemini-3.7-flash" # unstated → still the profile's
+
+
+def test_no_profile_flag_leaves_the_defaults_alone(tmp_path: Path):
+ runs = tmp_path / "runs"
+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run"])
+ assert r.exit_code == 0, r.output
+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())
+ # the resolved dial is recorded whichever way it was named, so `3dcv resume`
+ # reproduces it; with no flag and no env that dial is the default, balanced
+ assert spec["options"]["profile"] == "balanced" and spec["options"]["texture"] is False
+ assert spec["budget"]["max_rounds"] == 4 and spec["budget"]["max_usd"] == 5.0
+
+
+def test_judge_samples_reach_the_round_policy_only_when_a_profile_asks(tmp_path: Path):
+ from codeverse.cli import _common as C
+ from codeverse.contracts.common import Budget, Language, Track
+ from codeverse.contracts.spec import Spec
+
+ spec = Spec(id="x", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="p",
+ budget=Budget(max_rounds=3))
+ s = Settings()
+ assert C.round_policy_options(spec, s) == {} # n=1: the track keeps its own policy
+ s.apply_profile("quality", force=True)
+ opts = C.round_policy_options(spec, s)
+ assert opts["policy"].judge_samples == 3 and opts["policy"].max_rounds == 3
+ # the rubric threshold is bound here because injecting a policy skips BaseTrack's own binding
+ assert opts["policy"].target == pytest.approx(0.72)
+
+
+# --------------------------------------------------------------- flag == env var
+def _dial_from_flag(name: str):
+ """The dial `3dcv make --profile <name>` resolves to."""
+ from codeverse.cli import _common as C
+
+ return C.resolve_dial(Settings(), name)
+
+
+def _dial_from_env(name: str, monkeypatch):
+ """The dial `CV3D_PROFILE=<name> 3dcv make` resolves to (the real settings path)."""
+ from codeverse.cli import _common as C
+
+ monkeypatch.setenv("CV3D_PROFILE", name)
+ get_settings.cache_clear()
+ try:
+ return C.resolve_dial(get_settings(), None)
+ finally:
+ get_settings.cache_clear()
+
+
[email protected]("name", PROFILE_NAMES)
+def test_the_flag_and_the_env_var_resolve_to_the_same_dial(name, monkeypatch):
+ """`--profile X` and `CV3D_PROFILE=X` must set the WHOLE dial, not two thirds
+ of it: the verifier found the env path skipped candidates and the texture pass
+ because the CLI read those off the flag instead of the resolved profile."""
+ assert _dial_from_flag(name) == _dial_from_env(name, monkeypatch)
+
+
+ ("name", "expected"),
+ [
+ ("economy", {"generator": "single-shot:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.7-flash", "judge_samples": 2,
+ "rounds": 2, "candidates": 1, "texture": False,
+ "judge_max_px": 1024, "judge_montages": 3, "judge_detail_crops": 2,
+ "agent_max_turns": 0, "max_usd": 1.50, "max_minutes": 30.0}),
+ ("balanced", {"generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview", "judge_samples": 1,
+ "rounds": 4, "candidates": 1, "texture": False,
+ "judge_max_px": 1024, "judge_montages": 3, "judge_detail_crops": 2,
+ "agent_max_turns": 0, "max_usd": 5.0, "max_minutes": 60.0}),
+ ("quality", {"generator": "api-agent:gemini:gemini-3.7-flash",
+ "judge": "gemini:gemini-3.1-pro-preview", "judge_samples": 3,
+ "rounds": 4, "candidates": 2, "texture": True,
+ "judge_max_px": 1024, "judge_montages": 3, "judge_detail_crops": 2,
+ "agent_max_turns": 0, "max_usd": 8.0, "max_minutes": 90.0}),
+ ],
+)
+def test_each_profile_resolves_to_its_documented_dial(name, expected, monkeypatch):
+ """Every dial value docs/COST.md §15 promises, from BOTH entry points."""
+ for dial in (_dial_from_flag(name), _dial_from_env(name, monkeypatch)):
+ assert dial.profile == name
+ for field, want in expected.items():
+ assert getattr(dial, field) == want, f"{name}.{field}"
+
+
[email protected]("name", PROFILE_NAMES)
+def test_the_env_var_reaches_the_spec_a_make_writes(name, tmp_path: Path, monkeypatch):
+ """End to end: CV3D_PROFILE alone must produce the same spec shape as --profile."""
+ from codeverse.cost.profiles import PROFILES
+
+ p = PROFILES[name]
+ monkeypatch.setenv("CV3D_PROFILE", name)
+ get_settings.cache_clear()
+ runs = tmp_path / "runs"
+ r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run"])
+ assert r.exit_code == 0, r.output
+ spec = json.loads(next(runs.iterdir()).joinpath("spec.json").read_text())
+ assert spec["options"] == {"candidates": p.candidates, "texture": p.texture, "profile": name}
+ assert spec["budget"]["max_rounds"] == p.rounds and spec["budget"]["max_usd"] == p.max_usd
+ assert spec["backends"]["judge"] == p.judge and spec["backends"]["generator"] == p.generator
diff --git a/harness/tests/cost/test_reconstruct.py b/harness/tests/cost/test_reconstruct.py
index e24b3a91..028396d0 100644
--- a/harness/tests/cost/test_reconstruct.py
+++ b/harness/tests/cost/test_reconstruct.py
@@ -29,6 +29,47 @@ def test_generate_event_does_not_double_count_its_session(fake_run: Path):
assert not [r for r in led.rows if r.source == "event:generate.done"]
+def test_a_retried_session_does_not_double_count_its_generate_event(fake_run: Path, tmp_path: Path):
+ """The de-duplication bug behind docs/COST.md §6.
+
+ A retry records itself as ``<label>.a2`` while the ``generate.done`` event
+ carries the plain job label, so keying the "already covered" pool on the
+ session's own label left the retry's dollars in a different bucket, the pool
+ ran dry and the event was counted a SECOND time — $3.58 of the reconstructed
+ $89.88 across the 61 recorded runs."""
+ import shutil
+
+ ws = tmp_path / "retried"
+ shutil.copytree(fake_run, ws)
+ # split the baseline into a first attempt and a retry, as the harness records them
+ first = ws / "trajectories" / "baseline_r00"
+ retry = ws / "trajectories" / "baseline.a2_r00"
+ shutil.copytree(first, retry)
+ res = json.loads((retry / "result.json").read_text())
+ res["label"] = "baseline.a2"
+ for key, factor in (("input_tokens", 0.5), ("output_tokens", 0.5), ("cached_tokens", 0.5),
+ ("cost_usd", 0.5)):
+ res["usage"][key] = type(res["usage"][key])(res["usage"][key] * factor)
+ (retry / "result.json").write_text(json.dumps(res))
+ turns = [json.loads(line) for line in (retry / "transcript.jsonl").read_text().splitlines()]
+ for t in turns:
+ for key in ("input_tokens", "output_tokens", "cached_tokens", "cost_usd"):
+ t["usage"][key] = type(t["usage"][key])(t["usage"][key] * 0.5)
+ (retry / "transcript.jsonl").write_text("\n".join(json.dumps(t) for t in turns))
+ # the event summarises the WHOLE job (first attempt + retry) under the job label
+ events = [json.loads(line) for line in (ws / "events.jsonl").read_text().splitlines()]
+ for ev in events:
+ if ev.get("event") == "generate.done":
+ ev["cost_usd"] = round(ev["cost_usd"] * 1.5, 6)
+ (ws / "events.jsonl").write_text("\n".join(json.dumps(e) for e in events))
+
+ led = reconstruct(ws)
+ assert not [r for r in led.rows if r.source == "event:generate.done"], \
+ "the retry's dollars cover the job event; nothing may be added on top"
+ gen = [r for r in led.rows if r.stage is Stage.BASELINE]
+ assert len(gen) == 4 and all(r.source == "transcript" for r in gen) # 2 turns x 2 attempts
+
+
def test_judge_row_carries_the_judge_model(fake_run: Path):
row = next(r for r in reconstruct(fake_run).rows if r.role is Role.JUDGE)
assert row.model == "gemini-3.1-pro-preview" and row.provider == "gemini"
@@ -75,6 +116,23 @@ def test_reference_runs_reconcile():
assert led.notes, f"{led.run}: unexplained extra ${gap:.4f}"
[email protected](not (REPO / "bench" / "out" / "articulated_v1_flash").is_dir(),
+ reason="the recorded bench batteries are not present")
+ ("path", "record_usd"),
+ [("bench/out/articulated_v1_flash/runs/art_hard_door_handle", 3.0690),
+ ("bench/out/static_v1_flash/runs/tool_med_hand_drill", 4.0210),
+ ("bench/out/articulated_v1_flash/runs/art_easy_laptop", 1.1223)],
+)
+def test_runs_with_retried_sessions_reconcile_instead_of_looking_off_record(path: str, record_usd: float):
+ """The three worst rows of the old docs/COST.md §6 table ($1.38 / $0.50 / $0.31
+ "off record") were the retry de-duplication bug, not lost money."""
+ led = reconstruct(REPO / path)
+ assert led.recorded_usd == pytest.approx(record_usd, abs=0.001)
+ found = sum(r.recorded_usd for r in led.rows if r.source != "residual")
+ assert found <= led.recorded_usd + 0.005, f"{path}: ${found - led.recorded_usd:.4f} counted twice"
+
+
@pytest.mark.skipif(not (REPO / "runs" / "e2e_chair_blender" / "record.json").is_file(),
reason="recorded reference runs are not present")
def test_gemini_cli_recheck_finds_the_stale_parse():
diff --git a/harness/tests/flywheel_cli/test_bench.py b/harness/tests/flywheel_cli/test_bench.py
index 51804067..405a4946 100644
--- a/harness/tests/flywheel_cli/test_bench.py
+++ b/harness/tests/flywheel_cli/test_bench.py
@@ -83,3 +83,38 @@ def test_run_battery_resume_and_report(tmp_path: Path):
page = (out / "report.html").read_text()
assert "data:image/jpeg;base64," in page and "furn_easy_stool" in page # self-contained gallery
assert "veh_easy_toy_car" in page and "boom" in page # errored prompt still gets a card
+
+
+def test_every_bench_prompt_opens_its_own_run_ledger(tmp_path: Path):
+ """The batteries produce most of the runs; without a ledger their per-call rows
+ went to the per-process fallback log instead of the run (docs/COST.md §12)."""
+ from codeverse.contracts.chat import ChatMessage, ChatRequest, ChatResponse
+ from codeverse.contracts.common import Usage
+ from codeverse.cost.instrument import MeteredChatModel
+ from codeverse.cost.ledger import load_ledger
+
+ class FakeChat:
+ provider, model, id = "gemini", "gemini-3.7-flash", "gemini:gemini-3.7-flash"
+
+ def supports_vision(self) -> bool:
+ return True
+
+ def generate(self, request: ChatRequest) -> ChatResponse:
+ return ChatResponse(text="ok", usage=Usage(backend="gemini", model="gemini-3.7-flash",
+ input_tokens=1000, output_tokens=10))
+
+ inner = _fake_run_fn({})
+
+ def run(spec, ws: Workspace, resume: bool) -> RunRecord:
+ MeteredChatModel(FakeChat()).generate(
+ ChatRequest(messages=[ChatMessage.user("x")], label="api-agent:baseline:t0"))
+ return inner(spec, ws, resume)
+
+ battery = REPO / "bench" / "prompts" / "static_objects_v1.yaml"
+ out = tmp_path / "bench_out"
+ res = run_battery(battery, out, BenchOptions(parallel=2, limit=3), run_fn=run)
+ assert len(res) == 3
+ for r in res:
+ rows = load_ledger(Path(r.workspace))
+ assert len(rows) == 1, f"{r.id}: {rows}"
+ assert rows[0].run == r.id and rows[0].cost_usd > 0 # and not a sibling's row
diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.py
index 90951b0e..f5dc6926 100644
--- a/harness/tests/judges/test_vlm_judge.py
+++ b/harness/tests/judges/test_vlm_judge.py
@@ -31,7 +31,7 @@ def test_full_path_single_sample(judge_input, cache_dir):
assert "[verdict: overall 0.80 vs threshold 0.72 → PASS]" in j.summary
req = model.requests[0]
assert req.response_schema is not None and "Criteria" in req.response_schema["$defs"]
- assert req.label == "judge:static_object_v1:s0" and req.temperature == 0.2
+ assert req.label == "judge:static_object_v1:r00:s0" and req.temperature == 0.2
assert len(image_parts(req)) == 3 # 2×2 montage + 2 detail crops
assert "BLIND JUDGE" in req.system and "DEFECT CHECKLIST" in req.system
assert "Defects" in req.response_schema["$defs"]
@@ -163,3 +163,27 @@ def test_no_acceptance_items(judge_input, cache_dir):
j = _judge(model, cache_dir=cache_dir).judge(judge_input)
assert j.passed and j.acceptance_results == {}
assert "acceptance" not in model.requests[0].response_schema["properties"]
+
+
+# --------------------------------------------------------------- payload size (docs/COST.md §3)
+def test_judge_payload_size_comes_from_the_settings_dial(monkeypatch):
+ """``Settings.judge`` (set coherently by a cost profile) sizes the verdict's
+ images unless the caller states its own."""
+ from codeverse.config import Settings, get_settings
+ from codeverse.judges.vlm_judge import VlmJudge
+
+ s = Settings()
+ s.apply_profile("economy", force=True)
+ monkeypatch.setattr("codeverse.judges.vlm_judge.get_settings", lambda: s)
+ j = VlmJudge(rubric="static_object_v1", model_id="fake:fake-1")
+ # no profile shrinks the payload: 768 px bills the same as 1024 on Gemini and is
+ # noisier, and the 2→1 crop cut did not survive a second draw (docs/COST.md §14)
+ assert j.max_px == 1024 and j.detail_crops == 2 and j.max_montages == 3
+ # a caller that states a size still gets it — the dial is the default, not a cap
+ explicit = VlmJudge(rubric="static_object_v1", model_id="fake:fake-1", max_px=768, detail_crops=1)
+ assert explicit.max_px == 768 and explicit.detail_crops == 1
+ s2 = Settings(judge={"detail_crops": 0, "max_px": 512})
+ monkeypatch.setattr("codeverse.judges.vlm_judge.get_settings", lambda: s2)
+ s2.apply_profile("economy") # a stated payload survives a profile
+ assert VlmJudge(rubric="static_object_v1", model_id="fake:fake-1").detail_crops == 0
+ get_settings.cache_clear()
diff --git a/harness/tests/orchestrator_tracks/data/offrecord_runs.json b/harness/tests/orchestrator_tracks/data/offrecord_runs.json
new file mode 100644
index 00000000..b70e0f73
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/data/offrecord_runs.json
@@ -0,0 +1,534 @@
+{
+ "_source": "recorded telemetry of three runs from the docs/COST.md \u00a76 off-record list (bench/out/*/runs/<slug>); read-only reference data, copied here as fixtures",
+ "runs": {
+ "art_hard_door_handle": {
+ "status": "plateau",
+ "stop_reason": "max_rounds",
+ "recorded_total_usd": 3.068995,
+ "recorded_total_input_tokens": 9946812,
+ "rounds_in_record": [
+ 0,
+ 1,
+ 2,
+ 3
+ ],
+ "round_usd": {
+ "0": 0.771663,
+ "1": 0.474745,
+ "2": 0.331447,
+ "3": 0.750977
+ },
+ "sessions": [
+ {
+ "dir": "baseline.a2_r00",
+ "label": "baseline.a2",
+ "job_label": "baseline",
+ "round": 0,
+ "attempt": 2,
+ "exit_reason": "completed",
+ "turns": 82,
+ "cost_usd": 0.663884,
+ "input_tokens": 2500987,
+ "output_tokens": 66902,
+ "cached_tokens": 2167020,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "baseline_r00",
+ "label": "baseline",
+ "job_label": "baseline",
+ "round": 0,
+ "attempt": 1,
+ "exit_reason": "error",
+ "turns": 9,
+ "cost_usd": 0.035494,
+ "input_tokens": 53988,
+ "output_tokens": 59,
+ "cached_tokens": 8119,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine.a2_r01",
+ "label": "refine.a2",
+ "job_label": "refine",
+ "round": 1,
+ "attempt": 2,
+ "exit_reason": "completed",
+ "turns": 82,
+ "cost_usd": 0.357542,
+ "input_tokens": 1529537,
+ "output_tokens": 15943,
+ "cached_tokens": 1258366,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine.a2_r03",
+ "label": "refine.a2",
+ "job_label": "refine",
+ "round": 3,
+ "attempt": 2,
+ "exit_reason": "completed",
+ "turns": 84,
+ "cost_usd": 0.683181,
+ "input_tokens": 2226412,
+ "output_tokens": 51036,
+ "cached_tokens": 1745205,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_r01",
+ "label": "refine",
+ "job_label": "refine",
+ "round": 1,
+ "attempt": 1,
+ "exit_reason": "error",
+ "turns": 13,
+ "cost_usd": 0.045647,
+ "input_tokens": 103904,
+ "output_tokens": 79,
+ "cached_tokens": 48796,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_r02",
+ "label": "refine",
+ "job_label": "refine",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 48,
+ "cost_usd": 0.257921,
+ "input_tokens": 829760,
+ "output_tokens": 20677,
+ "cached_tokens": 654722,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_r03",
+ "label": "refine",
+ "job_label": "refine",
+ "round": 3,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 96,
+ "cost_usd": 0.726272,
+ "input_tokens": 2653708,
+ "output_tokens": 46718,
+ "cached_tokens": 2132151,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ }
+ ],
+ "judges": [
+ {
+ "round": 0,
+ "rubric": "articulated_v1",
+ "cost_usd": 0.072284,
+ "input_tokens": 11668,
+ "output_tokens": 2335
+ },
+ {
+ "round": 1,
+ "rubric": "articulated_v1",
+ "cost_usd": 0.071556,
+ "input_tokens": 12102,
+ "output_tokens": 2260
+ },
+ {
+ "round": 2,
+ "rubric": "articulated_v1",
+ "cost_usd": 0.073526,
+ "input_tokens": 10681,
+ "output_tokens": 1649
+ },
+ {
+ "round": 3,
+ "rubric": "articulated_v1",
+ "cost_usd": 0.067796,
+ "input_tokens": 10558,
+ "output_tokens": 1868
+ }
+ ],
+ "post_hoc_texture_passes": []
+ },
+ "tool_med_hand_drill": {
+ "status": "budget",
+ "stop_reason": "budget",
+ "recorded_total_usd": 4.021042,
+ "recorded_total_input_tokens": 16463068,
+ "rounds_in_record": [
+ 0,
+ 1
+ ],
+ "round_usd": {
+ "0": 0.624707,
+ "1": 1.095167
+ },
+ "sessions": [
+ {
+ "dir": "baseline_r00",
+ "label": "baseline",
+ "job_label": "baseline",
+ "round": 0,
+ "attempt": 1,
+ "exit_reason": "budget",
+ "turns": 121,
+ "cost_usd": 0.580225,
+ "input_tokens": 3012928,
+ "output_tokens": 39110,
+ "cached_tokens": 2705383,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_assembly_r02",
+ "label": "refine_assembly",
+ "job_label": "refine_assembly",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 50,
+ "cost_usd": 0.181432,
+ "input_tokens": 637080,
+ "output_tokens": 5733,
+ "cached_tokens": 470929,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_drill_bit.a2_r01",
+ "label": "refine_drill_bit.a2",
+ "job_label": "refine_drill_bit",
+ "round": 1,
+ "attempt": 2,
+ "exit_reason": "completed",
+ "turns": 16,
+ "cost_usd": 0.082922,
+ "input_tokens": 110928,
+ "output_tokens": 657,
+ "cached_tokens": 4056,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_drill_bit_r01",
+ "label": "refine_drill_bit",
+ "job_label": "refine_drill_bit",
+ "round": 1,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 20,
+ "cost_usd": 0.097178,
+ "input_tokens": 144813,
+ "output_tokens": 597,
+ "cached_tokens": 20253,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_drill_bit_r02",
+ "label": "refine_drill_bit",
+ "job_label": "refine_drill_bit",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 24,
+ "cost_usd": 0.07962,
+ "input_tokens": 208685,
+ "output_tokens": 676,
+ "cached_tokens": 117672,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_main_housing.a2_r01",
+ "label": "refine_main_housing.a2",
+ "job_label": "refine_main_housing",
+ "round": 1,
+ "attempt": 2,
+ "exit_reason": "completed",
+ "turns": 32,
+ "cost_usd": 0.120601,
+ "input_tokens": 330206,
+ "output_tokens": 3374,
+ "cached_tokens": 206972,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_main_housing_r01",
+ "label": "refine_main_housing",
+ "job_label": "refine_main_housing",
+ "round": 1,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 40,
+ "cost_usd": 0.20125,
+ "input_tokens": 411341,
+ "output_tokens": 2065,
+ "cached_tokens": 170369,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_main_housing_r02",
+ "label": "refine_main_housing",
+ "job_label": "refine_main_housing",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 20,
+ "cost_usd": 0.107131,
+ "input_tokens": 191298,
+ "output_tokens": 2745,
+ "cached_tokens": 69091,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_overall.a2_r01",
+ "label": "refine_overall.a2",
+ "job_label": "refine_overall",
+ "round": 1,
+ "attempt": 2,
+ "exit_reason": "completed",
+ "turns": 64,
+ "cost_usd": 0.300438,
+ "input_tokens": 1045197,
+ "output_tokens": 7017,
+ "cached_tokens": 755220,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_overall_r01",
+ "label": "refine_overall",
+ "job_label": "refine_overall",
+ "round": 1,
+ "attempt": 1,
+ "exit_reason": "budget",
+ "turns": 121,
+ "cost_usd": 0.680978,
+ "input_tokens": 3165991,
+ "output_tokens": 49693,
+ "cached_tokens": 2784983,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_rubber_overmold.a2_r01",
+ "label": "refine_rubber_overmold.a2",
+ "job_label": "refine_rubber_overmold",
+ "round": 1,
+ "attempt": 2,
+ "exit_reason": "completed",
+ "turns": 106,
+ "cost_usd": 0.48986,
+ "input_tokens": 2358423,
+ "output_tokens": 22979,
+ "cached_tokens": 2022413,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_rubber_overmold_r01",
+ "label": "refine_rubber_overmold",
+ "job_label": "refine_rubber_overmold",
+ "round": 1,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 92,
+ "cost_usd": 0.444199,
+ "input_tokens": 2197274,
+ "output_tokens": 25188,
+ "cached_tokens": 1923276,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_rubber_overmold_r02",
+ "label": "refine_rubber_overmold",
+ "job_label": "refine_rubber_overmold",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "budget",
+ "turns": 121,
+ "cost_usd": 0.496051,
+ "input_tokens": 2612152,
+ "output_tokens": 23083,
+ "cached_tokens": 2295740,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ }
+ ],
+ "judges": [
+ {
+ "round": 0,
+ "rubric": "static_object_v1",
+ "cost_usd": 0.044482,
+ "input_tokens": 10802,
+ "output_tokens": 1956
+ },
+ {
+ "round": 1,
+ "rubric": "static_object_v1",
+ "cost_usd": 0.062174,
+ "input_tokens": 10957,
+ "output_tokens": 2020
+ }
+ ],
+ "post_hoc_texture_passes": []
+ },
+ "furn_hard_rolltop_desk": {
+ "status": "passed",
+ "stop_reason": "pass",
+ "recorded_total_usd": 1.216518,
+ "recorded_total_input_tokens": 3381927,
+ "rounds_in_record": [
+ 0,
+ 1,
+ 2
+ ],
+ "round_usd": {
+ "0": 0.479199,
+ "1": 0.327666,
+ "2": 0.398044
+ },
+ "sessions": [
+ {
+ "dir": "baseline_r00",
+ "label": "baseline",
+ "job_label": "baseline",
+ "round": 0,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 108,
+ "cost_usd": 0.425861,
+ "input_tokens": 2144051,
+ "output_tokens": 21382,
+ "cached_tokens": 1870162,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_desk_top_r02",
+ "label": "refine_desk_top",
+ "job_label": "refine_desk_top",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 16,
+ "cost_usd": 0.079404,
+ "input_tokens": 99297,
+ "output_tokens": 1315,
+ "cached_tokens": 0,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_drawer_front_r02",
+ "label": "refine_drawer_front",
+ "job_label": "refine_drawer_front",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 44,
+ "cost_usd": 0.151169,
+ "input_tokens": 471835,
+ "output_tokens": 3576,
+ "cached_tokens": 320173,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_leather_inlay_r01",
+ "label": "refine_leather_inlay",
+ "job_label": "refine_leather_inlay",
+ "round": 1,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 24,
+ "cost_usd": 0.121015,
+ "input_tokens": 172309,
+ "output_tokens": 1460,
+ "cached_tokens": 20284,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_overall_r01",
+ "label": "refine_overall",
+ "job_label": "refine_overall",
+ "round": 1,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 40,
+ "cost_usd": 0.144453,
+ "input_tokens": 309989,
+ "output_tokens": 504,
+ "cached_tokens": 133228,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ },
+ {
+ "dir": "refine_overall_r02",
+ "label": "refine_overall",
+ "job_label": "refine_overall",
+ "round": 2,
+ "attempt": 1,
+ "exit_reason": "completed",
+ "turns": 22,
+ "cost_usd": 0.110066,
+ "input_tokens": 148847,
+ "output_tokens": 309,
+ "cached_tokens": 4041,
+ "backend": "api-agent",
+ "model": "gemini:gemini-3.7-flash"
+ }
+ ],
+ "judges": [
+ {
+ "round": 0,
+ "rubric": "static_object_v1",
+ "cost_usd": 0.053338,
+ "input_tokens": 10601,
+ "output_tokens": 1639
+ },
+ {
+ "round": 1,
+ "rubric": "static_object_v1",
+ "cost_usd": 0.062198,
+ "input_tokens": 10705,
+ "output_tokens": 1721
+ },
+ {
+ "round": 2,
+ "rubric": "static_object_v1",
+ "cost_usd": 0.057404,
+ "input_tokens": 10750,
+ "output_tokens": 1679
+ }
+ ],
+ "post_hoc_texture_passes": [
+ {
+ "cost_usd": 0.294,
+ "shipped": false
+ },
+ {
+ "cost_usd": 0.0919,
+ "shipped": false
+ },
+ {
+ "cost_usd": 0.2703,
+ "shipped": true
+ }
+ ]
+ }
+ }
+}
diff --git a/harness/tests/orchestrator_tracks/test_cost_accounting.py b/harness/tests/orchestrator_tracks/test_cost_accounting.py
new file mode 100644
index 00000000..c969369a
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/test_cost_accounting.py
@@ -0,0 +1,146 @@
+"""Closing the accounting holes docs/COST.md §6 found, and the money stops of §5.
+
+The audit measured $4.80 of real spend that never reached ``record.total_usage``
+(and therefore never reached ``BudgetGuard``): rounds the budget cut after the
+work was done, retried ``.a2`` agent sessions, post-hoc texture passes. These
+tests replay the *recorded* usage of three of those runs
+(``data/offrecord_runs.json``, taken from the read-only bench runs) through the
+production accounting objects and assert that every dollar now lands in the
+guard, in one ledger row, and in a round bucket.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from codeverse.contracts.agent import AgentJob, AgentResult
+from codeverse.contracts.common import Budget, Usage
+from codeverse.cost.ledger import load_ledger
+from codeverse.orchestrator.budget import BudgetExceeded, BudgetGuard, usage_delta
+from codeverse.tracks.generation import GenerationTask, _SessionAcc
+
+FIXTURES = json.loads((Path(__file__).parent / "data" / "offrecord_runs.json").read_text())["runs"]
+
+
+def _usage(row: dict) -> Usage:
+ return Usage(backend=row.get("backend") or "api-agent", model=row.get("model") or "gemini:gemini-3.7-flash",
+ cost_usd=row["cost_usd"], input_tokens=row["input_tokens"], output_tokens=row["output_tokens"],
+ cached_tokens=row.get("cached_tokens", 0))
+
+
+class _RecordedAgent:
+ """Replays the recorded sessions of one task, attempt by attempt."""
+
+ kind = "api-agent"
+ model = "gemini:gemini-3.7-flash"
+
+ def __init__(self, sessions: list[dict]):
+ self.sessions = list(sessions)
+
+ def run(self, job: AgentJob) -> AgentResult:
+ s = self.sessions.pop(0)
+ return AgentResult(ok=True, exit_reason=s["exit_reason"] or "completed", usage=_usage(s))
+
+
+def replay(name: str, ledger: Path | None, *, texture: bool = True) -> BudgetGuard:
+ """Spend one recorded run's money through today's accounting path."""
+ fx = FIXTURES[name]
+ guard = BudgetGuard(Budget(max_usd=10_000, max_minutes=10_000), run=name, ledger=ledger)
+ tasks: dict[tuple[str, int], list[dict]] = {}
+ for s in fx["sessions"]:
+ tasks.setdefault((s["job_label"] or s["label"], int(s["round"] or 0)), []).append(s)
+ for (label, rnd), sessions in tasks.items():
+ sessions.sort(key=lambda s: s["attempt"])
+ task = GenerationTask(label=label, prompt="p", round=rnd,
+ kind="baseline" if label.startswith("baseline") else "refine")
+ acc = _SessionAcc(task=task, budget=guard)
+ agent = _RecordedAgent(sessions)
+ for _ in sessions: # attempt 1, then the .a2 retry — exactly as run_agent_task does
+ acc.run(agent, AgentJob(workspace="/tmp", prompt="p", label=label, round=rnd))
+ for j in fx["judges"]: # steps.run_round
+ guard.add(Usage(cost_usd=j["cost_usd"], input_tokens=j["input_tokens"], output_tokens=j["output_tokens"]),
+ stage="judge", role="judge", round_index=j["round"], label=j["rubric"])
+ if texture:
+ for t in fx["post_hoc_texture_passes"]: # lifecycle._texture_pass
+ guard.add(Usage(cost_usd=t["cost_usd"]), stage="texture", role="image", label="texture_pass")
+ return guard
+
+
[email protected]("name", sorted(FIXTURES))
+def test_every_recorded_dollar_reaches_the_guard_and_the_ledger(name, tmp_path):
+ fx = FIXTURES[name]
+ ledger = tmp_path / "cost_ledger.jsonl"
+ guard = replay(name, ledger)
+ sessions = sum(s["cost_usd"] for s in fx["sessions"])
+ judges = sum(j["cost_usd"] for j in fx["judges"])
+ texture = sum(t["cost_usd"] for t in fx["post_hoc_texture_passes"])
+ # nothing is lost between the sessions and the guard — retries included
+ assert guard.spent.cost_usd == pytest.approx(sessions + judges + texture, abs=1e-6)
+ # and the ledger the run ships is the same number, one row per call
+ rows = load_ledger(ledger)
+ assert len(rows) == len(fx["sessions"]) + len(fx["judges"]) + len(fx["post_hoc_texture_passes"])
+ assert sum(r.cost_usd for r in rows) == pytest.approx(guard.spent.cost_usd, abs=1e-6)
+ # a retried session is its OWN row, labelled .a2 — the reconstructor had to guess
+ # this from a trajectory directory name, and double-counted when it guessed wrong
+ retried = [s for s in fx["sessions"] if s["attempt"] > 1]
+ assert sum(1 for r in rows if r.label.endswith(".a2")) == len(retried)
+
+
+def test_the_round_the_budget_cut_is_still_reported():
+ """tool_med_hand_drill spent $0.86 on r02 and then stopped: the round is not in
+ ``record.rounds`` at all, so its dollars used to be attributable to nothing."""
+ fx = FIXTURES["tool_med_hand_drill"]
+ assert fx["rounds_in_record"] == [0, 1]
+ cut = [s for s in fx["sessions"] if s["round"] == 2]
+ assert cut, "fixture must contain the round the budget cut"
+ guard = replay("tool_med_hand_drill", None)
+ burned = sum(s["cost_usd"] for s in cut)
+ assert guard.round_costs(2)["refine"] == pytest.approx(burned, abs=1e-6)
+ assert burned > 0.8 # $0.86 that record.rounds never mentioned
+
+
+def test_post_hoc_texture_passes_are_inside_the_total_now():
+ """furn_hard_rolltop_desk ran four texture passes for $0.656; ``record.total_usage``
+ said $1.2165. Charged through the guard the run reports $1.87."""
+ fx = FIXTURES["furn_hard_rolltop_desk"]
+ with_tex = replay("furn_hard_rolltop_desk", None)
+ without = replay("furn_hard_rolltop_desk", None, texture=False)
+ off_record = with_tex.spent.cost_usd - without.spent.cost_usd
+ assert off_record == pytest.approx(0.6562, abs=1e-3)
+ assert without.spent.cost_usd == pytest.approx(fx["recorded_total_usd"], abs=0.02)
+ assert with_tex.by_stage["texture"] == pytest.approx(0.6562, abs=1e-3)
+
+
+# ----------------------------------------------------------------------------- guard mechanics
+def test_spend_buckets_by_stage_and_round_and_only_charge_enforces():
+ g = BudgetGuard(Budget(max_usd=1.0, max_minutes=60))
+ g.charge(Usage(cost_usd=0.4), stage="baseline", round_index=0, label="baseline")
+ g.add(Usage(cost_usd=0.05), stage="judge", role="judge", round_index=0)
+ assert g.by_stage == {"baseline": pytest.approx(0.4), "judge": pytest.approx(0.05)}
+ assert g.round_costs(0)["judge"] == pytest.approx(0.05) and g.round_costs(9) == {}
+ g.add(Usage(cost_usd=0.9), stage="texture", role="image") # over the ceiling, but never raises
+ assert not g.ok() and g.spent.cost_usd == pytest.approx(1.35)
+ with pytest.raises(BudgetExceeded):
+ g.charge(Usage(cost_usd=0.01), stage="refine")
+ assert g.spent.cost_usd == pytest.approx(1.36) # the money is counted even when it raises
+ assert list(g.stage_summary())[0] == "texture" # biggest bucket first
+
+
+def test_usage_delta_reports_what_a_round_burned():
+ before = Usage(cost_usd=1.0, input_tokens=100, output_tokens=10)
+ after = Usage(cost_usd=1.75, input_tokens=350, output_tokens=40)
+ d = usage_delta(after, before)
+ assert d.cost_usd == pytest.approx(0.75) and d.input_tokens == 250 and d.output_tokens == 30
+ assert usage_delta(before, after).cost_usd == 0.0 # never negative
+
+
+def test_the_ledger_is_optional_and_never_breaks_a_run(tmp_path):
+ g = BudgetGuard(Budget(max_usd=1.0, max_minutes=60), ledger=tmp_path / "sub" / "dir" / "l.jsonl")
+ g.charge(Usage(cost_usd=0.1), stage="plan", role="planner")
+ assert len(load_ledger(tmp_path / "sub" / "dir" / "l.jsonl")) == 1
+ quiet = BudgetGuard(Budget(max_usd=1.0, max_minutes=60)) # no ledger configured
+ quiet.charge(Usage(cost_usd=0.1))
+ assert quiet.spent.cost_usd == pytest.approx(0.1)
diff --git a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
index 44610ecf..adaebf7d 100644
--- a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
+++ b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
@@ -92,7 +92,8 @@ def test_generate_single_shot_writes_files_and_charges(tmp_ws):
res = generate(tmp_ws, agent_id="single-shot:gemini:x", task=task, model=model, budget=budget, events=events)
assert res.ok and [c.path for c in res.files_changed] == ["src/model.py"] and (tmp_ws.src / "model.py").read_text() == "import bpy\n"
assert budget.spent.cost_usd == pytest.approx(0.002)
- assert "=== FILE:" in model.requests[0].system and model.requests[0].label == "baseline"
+ # the label carries the round so the cost ledger can attribute a single-shot call
+ assert "=== FILE:" in model.requests[0].system and model.requests[0].label == "baseline:r00"
assert (tmp_ws.trajectories / "baseline_r00" / "response.md").is_file()
# parse failure → ok=False, no raise
bad = FakeChatModel(lambda req: "nope")
diff --git a/harness/tests/orchestrator_tracks/test_orchestrator.py b/harness/tests/orchestrator_tracks/test_orchestrator.py
index 44a4ea74..2dd4ffca 100644
--- a/harness/tests/orchestrator_tracks/test_orchestrator.py
+++ b/harness/tests/orchestrator_tracks/test_orchestrator.py
@@ -124,7 +124,11 @@ def test_stop_policy_decisions():
assert sp.decide([_round(0, 0.5)]) == "continue"
assert sp.decide([_round(0, 0.5)], budget_ok=False) == "budget"
assert sp.decide([_round(0, 0.5), _round(1, 0.85)]) == "pass"
- assert sp.decide([_round(0, 0.5), _round(1, 0.51), _round(2, 0.515)]) == "plateau"
+ # three flat rounds: from r03 on the marginal-value stop answers first (both stop;
+ # "diminishing_returns" is the more precise reason — see test_round_economics.py)
+ assert sp.decide([_round(0, 0.5), _round(1, 0.51), _round(2, 0.515)]) == "diminishing_returns"
+ flat = StopPolicy(RoundPolicy(max_rounds=3, plateau_window=2, min_delta=0.02, target=0.8, marginal_from_round=99))
+ assert flat.decide([_round(0, 0.5), _round(1, 0.51), _round(2, 0.515)]) == "plateau"
assert sp.decide([_round(0, 0.5), _round(1, 0.6), _round(2, 0.7)]) == "continue"
assert sp.decide([_round(0, 0.5), _round(1, 0.6), _round(2, 0.7), _round(3, 0.75)]) == "max_rounds"
# unscored rounds (build failed) do not count as plateau evidence
diff --git a/harness/tests/orchestrator_tracks/test_round_economics.py b/harness/tests/orchestrator_tracks/test_round_economics.py
new file mode 100644
index 00000000..a29e8005
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/test_round_economics.py
@@ -0,0 +1,510 @@
+"""The round-level cost controls — the ones that survived measurement, and the
+proof for the ones that did not.
+
+* a round that scores below the best by more than the judge's noise does not
+ buy another round of the same shape (``regression`` / strategy switch);
+* r03+ only starts when the previous round gained more than 1.5 σ
+ (``diminishing_returns``);
+* agent sessions are **uncapped by default** — a 28-turn cap was A/B-tested and
+ rejected ($1.381 / 0.479 capped vs $1.360 / 0.684 uncapped, n=3 per arm,
+ docs/COST.md §17) — but an explicitly requested cap still lands gracefully;
+* the judge is skipped only where the verdict is provably never bought at all,
+ and every round emits one ``cost.round`` event with what it burned.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from codeverse.contracts.agent import AgentJob, AgentResult
+from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Severity
+from codeverse.contracts.common import Language, Usage
+from codeverse.contracts.judgment import Judgment
+from codeverse.contracts.run import RoundRecord, RunStatus
+from codeverse.events import EventLog
+from codeverse.orchestrator.rounds import (
+ DEFAULT_JUDGE_SIGMA,
+ REWRITE_KIND,
+ RoundPolicy,
+ StopPolicy,
+ best_score,
+ judge_sigma,
+ last_gain,
+)
+from codeverse.tracks.generation import (
+ DEFAULT_AGENT_MAX_TURNS,
+ GenerationTask,
+ agent_max_turns,
+ run_agent_task,
+ turn_capped,
+)
+from codeverse.tracks.static_object import StaticObjectTrack
+from codeverse.workspace import Workspace
+from tests.orchestrator_tracks.conftest import make_spec
+from tests.orchestrator_tracks.fakes import (
+ FakeAgent,
+ FakeChatModel,
+ FakeJudge,
+ FakeRuntime,
+ FakeServices,
+)
+
+
+def _round(i: int, score: float | None, kind: str = "refine") -> RoundRecord:
+ j = Judgment(rubric="r", scores={}, overall=score, passed=bool(score and score >= 0.8)) if score is not None else None
+ return RoundRecord(index=i, kind="baseline" if i == 0 else kind, judgment=j,
+ build=BuildResult(ok=True, language="l"), commit=f"c{i}")
+
+
+# ----------------------------------------------------------------------------- judge noise table
+def test_judge_sigma_comes_from_the_one_measured_table():
+ from codeverse.cost.routing import JUDGE_NOISE
+
+ assert judge_sigma("gemini:gemini-3.1-pro-preview") == pytest.approx(JUDGE_NOISE["gemini-3.1-pro-preview"][0])
+ assert judge_sigma("gemini:gemini-3.7-flash") == pytest.approx(0.083)
+ assert judge_sigma("gemini:gemini-3.7-flash-002") == pytest.approx(0.083) # version suffix
+ assert judge_sigma("") == DEFAULT_JUDGE_SIGMA and judge_sigma("who:knows") == DEFAULT_JUDGE_SIGMA
+ pol = RoundPolicy(judge_model="gemini:gemini-3.7-flash")
+ assert pol.sigma == pytest.approx(0.083)
+ assert pol.regression_delta == pytest.approx(0.083) and pol.marginal_delta == pytest.approx(0.1245)
+ assert RoundPolicy().with_judge("gemini:gemini-3.7-flash").sigma == pytest.approx(0.083)
+ assert RoundPolicy().with_judge("") is not None
+
+
+def test_best_score_and_last_gain():
+ h = [_round(0, 0.5), _round(1, 0.62), _round(2, 0.55)]
+ assert best_score(h) == pytest.approx(0.62) and last_gain(h) == pytest.approx(-0.07)
+ assert best_score([]) is None and last_gain([_round(0, 0.5)]) is None
+ assert last_gain([_round(0, 0.5), _round(1, None)]) is None
+
+
+# ----------------------------------------------------------------------------- regression
+def test_a_regression_buys_a_change_of_shape_then_stops():
+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, judge_model="gemini:gemini-3.1-pro-preview",
+ marginal_from_round=99))
+ inside = sp.evaluate([_round(0, 0.60), _round(1, 0.58)]) # −0.02 < σ 0.030 → judge noise
+ assert inside.reason == "continue" and inside.strategy == "same"
+ first = sp.evaluate([_round(0, 0.60), _round(1, 0.52)]) # −0.08: a real regression
+ assert first.reason == "continue" and first.strategy == "switch" and "0.520" in first.detail
+ again = sp.evaluate([_round(0, 0.60), _round(1, 0.52), _round(2, 0.50, kind=REWRITE_KIND)])
+ assert again.reason == "regression" and "changed strategy" in again.detail
+ twice = sp.evaluate([_round(0, 0.60), _round(1, 0.52), _round(2, 0.55)]) # two regressions vs best
+ assert twice.reason == "regression"
+
+
+def test_regression_can_be_configured_to_stop_outright():
+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99, regression_allow_switch=False))
+ d = sp.evaluate([_round(0, 0.60), _round(1, 0.52)])
+ assert d.reason == "regression" and d.strategy == "same"
+
+
+def test_a_noisy_judge_widens_the_regression_band():
+ flash = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99,
+ judge_model="gemini:gemini-3.7-flash"))
+ # −0.06 is a regression for pro (σ 0.030) but inside flash's own noise (σ 0.083)
+ assert flash.evaluate([_round(0, 0.60), _round(1, 0.54)]).strategy == "same"
+ pro = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99,
+ judge_model="gemini:gemini-3.1-pro-preview"))
+ assert pro.evaluate([_round(0, 0.60), _round(1, 0.54)]).strategy == "switch"
+
+
+# ----------------------------------------------------------------------------- diminishing returns
+def test_r03_only_runs_when_the_last_round_paid_for_itself():
+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, judge_model="gemini:gemini-3.1-pro-preview"))
+ # r02 gained +0.10 ≫ 1.5σ = 0.045 → r03 is worth starting
+ assert sp.evaluate([_round(0, 0.4), _round(1, 0.5), _round(2, 0.6)]).reason == "continue"
+ # r02 gained +0.03 < 0.045 → not worth another round
+ d = sp.evaluate([_round(0, 0.40), _round(1, 0.55), _round(2, 0.58)])
+ assert d.reason == "diminishing_returns" and "1.5 × judge σ" in d.detail
+ # ... and the rule does not apply to r01/r02, which the audit shows are the good buys
+ assert sp.evaluate([_round(0, 0.40), _round(1, 0.42)]).reason == "continue"
+ # scoring at/above target without passing (a failed must-item) is not worth
+ # another refine round either — the score is not what is missing
+ above = RoundRecord(index=2, kind="refine", build=BuildResult(ok=True, language="l"),
+ judgment=Judgment(rubric="r", scores={}, overall=0.95, passed=False))
+ at_target = sp.evaluate([_round(0, 0.4), _round(1, 0.6), above])
+ assert at_target.reason == "diminishing_returns" and "target" in at_target.detail
+
+
+def test_the_marginal_round_is_configurable():
+ """The audit's recommendation #1 (stop after r01) is this knob, not a new rule."""
+ sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=2,
+ judge_model="gemini:gemini-3.1-pro-preview"))
+ assert sp.evaluate([_round(0, 0.40), _round(1, 0.42)]).reason == "diminishing_returns"
+
+
+def test_the_stop_order_is_exhausted_regression_then_marginal_then_switch_then_plateau():
+ """One case per branch, each one where the NEXT branch would answer differently —
+ the order in ``StopPolicy.evaluate`` is the economics, so it is pinned here.
+
+ (Replayed over the 107 recorded rounds this shipped default cuts 2 runs /
+ $1.14 with 0 best rounds lost — waste-and-accounting/replay_stops.py.)"""
+ pro = dict(max_rounds=4, target=0.9, judge_model="gemini:gemini-3.1-pro-preview")
+ sp = StopPolicy(RoundPolicy(**pro))
+ # 1. the switch is already spent (a rewrite regressed again) — outranks everything,
+ # even though the marginal test would also stop here (last gain −0.10)
+ spent = [_round(0, 0.60), _round(1, 0.52), _round(2, 0.50, kind=REWRITE_KIND)]
+ assert sp.evaluate(spent).reason == "regression" and "changed strategy" in sp.evaluate(spent).detail
+ # 2. at r03 the marginal test beats a FIRST regression: no shape is worth $0.5 there
+ marginal = [_round(0, 0.60), _round(1, 0.65), _round(2, 0.55)]
+ assert sp.evaluate(marginal).reason == "diminishing_returns"
+ # 3. before r03 the same first regression buys exactly one change of shape
+ switch = sp.evaluate([_round(0, 0.60), _round(1, 0.52)])
+ assert (switch.reason, switch.strategy, switch.stop) == ("continue", "switch", False)
+ # 4. plateau is the fallback: no regression, marginal test not in force, no gain
+ flat = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99, plateau_window=2,
+ min_delta=0.02, judge_model="gemini:gemini-3.1-pro-preview"))
+ assert flat.evaluate([_round(0, 0.50), _round(1, 0.505), _round(2, 0.51)]).reason == "plateau"
+ # ... and a run that is still climbing is never stopped by any of the four
+ assert flat.evaluate([_round(0, 0.50), _round(1, 0.60), _round(2, 0.70)]).reason == "continue"
+
+
+# ----------------------------------------------------------------------------- turn budget
+class _TurnAgent:
+ """Burns turns; reports the api-agent's own 'max_turns (N) reached' when it runs out."""
+
+ kind = "api-agent"
+ model = "gemini:gemini-3.7-flash"
+
+ def __init__(self, *, writes_on: int = 2, cost: float = 0.2):
+ self.jobs: list[AgentJob] = []
+ self.writes_on = writes_on
+ self.cost = cost
+
+ def run(self, job: AgentJob) -> AgentResult:
+ self.jobs.append(job)
+ ws = Workspace(job.workspace)
+ wrote = []
+ if len(self.jobs) >= self.writes_on:
+ (ws.src).mkdir(parents=True, exist_ok=True)
+ (ws.src / "object.js").write_text(f"// session {len(self.jobs)}\n")
+ wrote = [{"path": "src/object.js", "status": "modified"}]
+ exhausted = len(self.jobs) == 1
+ return AgentResult(ok=bool(wrote), exit_reason="budget" if exhausted else "completed",
+ files_changed=wrote, errors=[f"max_turns ({job.max_turns}) reached"] if exhausted else [],
+ usage=Usage(backend="api-agent", cost_usd=self.cost, input_tokens=1000))
+
+
+def test_agent_sessions_are_uncapped_by_default(tmp_ws, monkeypatch):
+ """Measured (docs/COST.md §17): cap 28 cost $1.381 at score 0.479, uncapped
+ $1.360 at 0.684 — n=3 per arm, same prompt/generator/judge/budget. The cap
+ saved nothing and lost 0.205 of a score point, so nothing sets one by default."""
+ monkeypatch.delenv("CV3D_AGENT_MAX_TURNS", raising=False)
+ assert DEFAULT_AGENT_MAX_TURNS == 0 and RoundPolicy().agent_max_turns == 0
+ agent = _TurnAgent(writes_on=1)
+ task = GenerationTask(label="baseline", prompt="p", round=0, kind="baseline")
+ run_agent_task(tmp_ws, agent=agent, task=task)
+ assert agent.jobs[0].max_turns == AgentJob(workspace="w", prompt="p").max_turns # the backend's own default
+ assert agent_max_turns() == 0 # "no cap asked for"
+ # ... and every explicit way of asking for one still works
+ run_agent_task(tmp_ws, agent=agent, task=task, max_turns=12)
+ assert agent.jobs[-1].max_turns == 12
+ run_agent_task(tmp_ws, agent=agent, task=task.model_copy(update={"max_turns": 7}), max_turns=12)
+ assert agent.jobs[-1].max_turns == 7 # the task wins over the policy
+ monkeypatch.setenv("CV3D_AGENT_MAX_TURNS", "9")
+ assert agent_max_turns() == 9
+ run_agent_task(tmp_ws, agent=agent, task=task)
+ assert agent.jobs[-1].max_turns == 9
+ monkeypatch.setenv("CV3D_AGENT_MAX_TURNS", "nonsense")
+ assert agent_max_turns() == DEFAULT_AGENT_MAX_TURNS == 0
+
+
+def test_the_policy_cap_is_plumbed_through_the_round(tmp_path, spec, settings):
+ """A caller that DOES set one (a cost profile, a bench arm) still reaches the job."""
+ from codeverse.tracks.steps import run_generation_tasks
+
+ agent = _TurnAgent(writes_on=1)
+ ctx = _ctx(tmp_path, spec, settings, policy=RoundPolicy(agent_max_turns=15), agent=agent)
+ run_generation_tasks(ctx, [GenerationTask(label="baseline", prompt="p", round=0, kind="baseline")])
+ assert agent.jobs[0].max_turns == 15
+ assert agent.jobs[1].max_turns == RoundPolicy().agent_wrapup_turns # ... and the landing session
+
+
+def test_hitting_the_cap_asks_for_a_landing_instead_of_killing_the_session(tmp_ws):
+ events = EventLog(tmp_ws.events_path)
+ agent = _TurnAgent(writes_on=2) # session 1 burns its turns and writes nothing
+ guard = _guard()
+ res = run_agent_task(tmp_ws, agent=agent, task=GenerationTask(label="refine", prompt="do the work", round=1,
+ kind="refine", max_turns=28),
+ budget=guard, events=events, wrapup_turns=5)
+ assert len(agent.jobs) == 2 and turn_capped(_result_of(agent, 0))
+ wrap = agent.jobs[1]
+ assert wrap.max_turns == 5 and "LAST session" in wrap.prompt and "do the work" in wrap.prompt
+ assert res.ok and res.turn_capped and res.sessions == 2
+ # both sessions are paid for and both are in the guard, under their own labels
+ assert res.usage.cost_usd == pytest.approx(0.4) and guard.spent.cost_usd == pytest.approx(0.4)
+ assert guard.round_costs(1)["refine"] == pytest.approx(0.4)
+ kinds = [e["event"] for e in events.read()]
+ assert "generate.turn_cap" in kinds
+ done = next(e for e in events.read() if e["event"] == "generate.done")
+ assert done["turn_capped"] is True and done["sessions"] == 2
+
+
+def test_a_crashing_second_session_never_erases_what_the_first_one_spent(tmp_ws):
+ class _Flaky(_TurnAgent):
+ def run(self, job: AgentJob) -> AgentResult:
+ if self.jobs:
+ self.jobs.append(job)
+ raise RuntimeError("503 storm")
+ return super().run(job)
+
+ guard = _guard()
+ agent = _Flaky(writes_on=99)
+ res = run_agent_task(tmp_ws, agent=agent, task=GenerationTask(label="refine", prompt="p", round=2, kind="refine"),
+ budget=guard)
+ assert len(agent.jobs) == 2 and not res.ok
+ assert guard.spent.cost_usd == pytest.approx(0.2) and res.usage.cost_usd == pytest.approx(0.2)
+ assert "503 storm" in res.notes and guard.round_costs(2)["refine"] == pytest.approx(0.2)
+
+
+def _result_of(agent, i):
+ return AgentResult(ok=False, exit_reason="budget", errors=[f"max_turns ({agent.jobs[i].max_turns}) reached"])
+
+
+def _guard():
+ from codeverse.contracts.common import Budget
+ from codeverse.orchestrator.budget import BudgetGuard
+
+ return BudgetGuard(Budget(max_usd=100, max_minutes=100))
+
+
+def test_task_stage_names_the_cost_bucket_a_task_spends_in():
+ from codeverse.tracks.generation import task_stage
+
+ def t(kind: str, label: str = "x") -> str:
+ return task_stage(GenerationTask(label=label, prompt="p", kind=kind))
+
+ assert t("baseline") == "baseline" and t("refine") == "refine" and t("repair") == "repair"
+ assert t("rebuild") == "repair" # regenerating after a failed build IS repair money
+ assert t("zone") == "zones" and t("asset") == "assets" and t("asset_fix") == "assets"
+ assert t("compose") == "assemble" and t("env") == "env"
+ # an unknown kind falls back to the label, which record_call maps by prefix
+ assert t("something_new", "asset_koi") == "asset_koi"
+
+
+# ----------------------------------------------------------------------------- judge skipping
+def _renders():
+ from codeverse.contracts.artifacts import RenderSet, RenderView
+
+ return RenderSet(views=[RenderView(name="front", path="x.png")], renderer="fake")
+
+
+def _gates(errors: int = 0):
+ return [GateReport(gate="lint", passed=not errors,
+ findings=[GateFinding(gate="lint", severity=Severity.ERROR, message="e")] * errors)]
+
+
+def test_skip_judge_reasons_are_only_states_where_the_verdict_is_never_bought(tmp_path, spec, settings):
+ """Every branch left in ``skip_judge_reason`` must save a REAL verdict.
+
+ The two wave-2 branches that did not are gone: "no file change" was
+ unreachable and "build not repaired within the repair budget" was bought back
+ by ``rejudge_round`` on the next loop iteration (both reproduced below)."""
+ from codeverse.tracks.steps import skip_judge_reason
+
+ ctx = _ctx(tmp_path, spec, settings)
+ assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == ""
+ # a round with gate errors IS judged by default — the gates say what is broken,
+ # the verdict says whether the shape is right, and only the verdict can promote it
+ assert skip_judge_reason(ctx, gates=_gates(3), renders=_renders()) == ""
+ assert skip_judge_reason(ctx, gates=_gates(), renders=None) == "no renders"
+ # the caller can switch that off; then the SAME rule holds in the rejudge path
+ strict = _ctx(tmp_path, spec, settings, policy=RoundPolicy(judge_on_gate_errors=False), name="strict")
+ assert skip_judge_reason(strict, gates=_gates(3), renders=_renders()) == "gate errors"
+ assert skip_judge_reason(strict, gates=_gates(0), renders=_renders()) == ""
+ # ... the budget/clock stop: the loop's next budget_ok check ends the run, so this
+ # verdict cannot promote anything (audit: 2 verdicts / $0.09 bought past the clock)
+ ctx.budget.add(Usage(cost_usd=99.0), stage="refine")
+ assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == "budget already exceeded"
+ assert StopPolicy(ctx.policy).evaluate([_round(0, 0.5)], budget_ok=ctx.budget.ok()).reason == "budget"
+ ctx.judge = None
+ assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == "no judge configured"
+
+
+def test_a_round_that_changed_no_file_never_reaches_the_judge_question(tmp_path, spec, settings):
+ """Proof the deleted "no file change" branch was unreachable: a task that writes
+ nothing is not ``ok``, and ``run_generation_tasks`` raises before any verdict."""
+ from codeverse.tracks.steps import RoundFailed, run_generation_tasks
+
+ class _Idle:
+ kind, model = "api-agent", "m"
+
+ def run(self, job: AgentJob) -> AgentResult:
+ return AgentResult(ok=True, exit_reason="completed", files_changed=[], usage=Usage(cost_usd=0.01))
+
+ ctx = _ctx(tmp_path, spec, settings, agent=_Idle(), name="nochange")
+ with pytest.raises(RoundFailed):
+ run_generation_tasks(ctx, [GenerationTask(label="refine", prompt="p", round=1, kind="refine")])
+
+
+def test_a_skipped_verdict_is_not_bought_back_by_the_rejudge_path(tmp_path, spec, settings):
+ """``rejudge_round`` exists for a judge that FAILED, not for one we chose not to
+ call: without this the "gate errors" skip is a ``judge.retry``, not a saving."""
+ from codeverse.tracks.steps import rejudge_round
+
+ ctx = _ctx(tmp_path, spec, settings, policy=RoundPolicy(judge_on_gate_errors=False), name="norebuy")
+ rec = RoundRecord(index=1, kind="refine", build=BuildResult(ok=True, language="l"),
+ gates=_gates(2), renders=_renders())
+ assert rejudge_round(ctx, _pipeline(), rec) is False
+ assert ctx.judge.calls == [] and ctx.budget.spent.cost_usd == 0.0
+ # the same round with clean gates (an outage, not a policy skip) IS re-judged
+ rec2 = RoundRecord(index=2, kind="refine", build=BuildResult(ok=True, language="l"),
+ gates=_gates(0), renders=_renders())
+ assert rejudge_round(ctx, _pipeline(), rec2) is True and len(ctx.judge.calls) == 1
+
+
+def test_a_round_that_broke_the_gates_does_not_displace_a_clean_one():
+ """The unscored fallback: never promote a round that does not build, and never
+ let a gate-breaking round displace the previous artifact just by being last."""
+ from codeverse.orchestrator.rounds import BestSelector
+
+ def r(i, *, build_ok=True, errors=0):
+ return RoundRecord(index=i, kind="refine", build=BuildResult(ok=build_ok, language="l"),
+ gates=_gates(errors), commit=f"c{i}")
+
+ assert BestSelector().pick([r(0), r(1, build_ok=False)]) == 0 # broken build never wins
+ assert BestSelector().pick([r(0), r(1, errors=4)]) == 0 # clean r00 keeps the crown
+ assert BestSelector().pick([r(0, errors=4), r(1)]) == 1
+ assert BestSelector().pick([r(0), r(1)]) == 1 # tie → the later one
+ assert BestSelector().pick([r(0, build_ok=False)]) is None
+
+
+class _Pipeline:
+ """The minimum ``RoundPipeline`` the judge path touches."""
+
+ def measure(self, ctx, build): return None
+ def gates(self, ctx, i, build, m): return []
+ def render(self, ctx, i, build, m): return _renders()
+ def plan_summary(self, ctx): return "plan"
+ def judge_context(self, ws, plan, i, build, gates): return ""
+
+
+def _pipeline():
+ return _Pipeline()
+
+
+def _ctx(tmp_path, spec, settings, *, policy: RoundPolicy | None = None, agent=None, name: str = "skip"):
+ from codeverse.contracts.common import Budget
+ from codeverse.orchestrator.budget import BudgetGuard
+ from codeverse.orchestrator.state import RunState
+ from codeverse.tracks.common import RunContext
+
+ ws = Workspace(tmp_path / "runs" / name)
+ ws.create()
+ return RunContext(spec=spec, ws=ws, events=EventLog(ws.events_path), settings=settings,
+ budget=BudgetGuard(Budget(max_usd=1.0, max_minutes=60)), runtime=FakeRuntime(Language.THREEJS),
+ services=FakeServices(), state=RunState(), policy=policy or RoundPolicy(), track=spec.track,
+ rubric="static_object_v1", agent_id="fake:fake-model", judge=FakeJudge(), agent=agent)
+
+
+# ----------------------------------------------------------------------------- end to end
+def _planner(plan_dict):
+ return FakeChatModel(lambda req: plan_dict)
+
+
+def _writer(job, ws):
+ return {"src/object.js": f"// {job.label} r{job.round}\nexport function build(THREE) {{}}\n"}
+
+
+def test_the_loop_switches_shape_after_a_regression_and_emits_cost_rounds(tmp_path, chair_plan, settings):
+ """r01 regresses hard → r02 is ONE whole-object rewrite, not another per-part fan-out."""
+ spec = make_spec(max_rounds=3)
+ ws = Workspace(tmp_path / "runs" / "regress")
+ agent = FakeAgent(_writer, cost=0.02)
+ judge = FakeJudge(scores=(0.60, 0.30, 0.35))
+ track = StaticObjectTrack(services=FakeServices(contract_errors=2), judge=judge, agent=agent,
+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,
+ runtime=FakeRuntime(Language.THREEJS),
+ policy=RoundPolicy(max_rounds=3, target=0.8, judge_model="gemini:gemini-3.1-pro-preview"))
+ rec = track.run(spec, ws)
+ assert [r.kind for r in rec.rounds] == ["baseline", "refine", REWRITE_KIND]
+ assert rec.extra["stop_reason"] == "regression" and rec.status is RunStatus.PLATEAU
+ events = EventLog(ws.events_path).read()
+ switch = next(e for e in events if e["event"] == "strategy.switch")
+ assert switch["reason"] == "regression"
+ rewrite = [j for j in agent.jobs if "rewrite" in j.prompt.lower() or "made the artifact WORSE" in j.prompt]
+ assert rewrite, "the rewrite round must tell the agent the last round made it worse"
+ assert len([j for j in agent.jobs if j.round == 2]) == 1 # ONE task, not a per-part fan-out
+ # every round reports what it burned
+ costs = [e for e in events if e["event"] == "cost.round"]
+ assert len(costs) == 3 and all("generate" in c["stages"] for c in costs)
+ assert costs[0]["agent_turns"] == 0 # the fake agent keeps no transcript
+ assert costs[1]["wasted"] is True and costs[1]["waste_reason"] == "regression"
+ assert costs[2]["judge_usd"] > 0 and costs[2]["run_usd"] >= costs[1]["run_usd"]
+ assert rec.extra["cost_by_stage"]["judge"] > 0 and rec.extra["cost_by_stage"]["refine"] > 0
+ # ... and the run ships its own priced ledger
+ from codeverse.cost.ledger import load_ledger
+
+ rows = load_ledger(ws.root / "cost_ledger.jsonl")
+ assert {str(r.stage) for r in rows} >= {"baseline", "refine", "judge"}
+ assert sum(r.cost_usd for r in rows) == pytest.approx(rec.total_usage.cost_usd, abs=1e-6)
+
+
+def test_a_lint_stuck_run_keeps_every_score_instead_of_deferring_the_verdict(tmp_path, chair_plan, settings):
+ """The verifier's reproduction of the removed "build not repaired" skip.
+
+ Every round here builds but leaves a lint error the repair budget cannot clear.
+ Wave 2 skipped the judge on all three rounds and then bought two of the verdicts
+ back one loop iteration later (``judge.retry``) — no money saved — while the LAST
+ round, the best one (0.7), was left without a score and could not be promoted.
+ Now each round is judged once, in the round, and r02 wins."""
+ def _writer_lint(job, ws):
+ return {"src/object.js": f"// {job.label} r{job.round}\nLINT_ERROR\nexport function build(THREE) {{}}\n"}
+
+ spec = make_spec(max_rounds=2)
+ ws = Workspace(tmp_path / "runs" / "lintstuck")
+ judge = FakeJudge(scores=(0.5, 0.6, 0.7))
+ track = StaticObjectTrack(services=FakeServices(), judge=judge, agent=FakeAgent(_writer_lint),
+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,
+ runtime=FakeRuntime(Language.THREEJS),
+ policy=RoundPolicy(max_rounds=2, target=0.9))
+ rec = track.run(spec, ws)
+ events = EventLog(ws.events_path).read()
+ assert [e for e in events if e["event"] == "judge.skipped"] == []
+ assert [e for e in events if e["event"] == "judge.retry"] == []
+ assert [r.score for r in rec.rounds] == [0.5, 0.6, 0.7]
+ assert rec.best_round == 2 and rec.final_score == pytest.approx(0.7)
+ # one verdict per round, bought once
+ assert len([e for e in events if e["event"] == "judge.done"]) == len(rec.rounds) == len(judge.calls)
+
+
+def test_a_round_that_raises_still_reports_what_it_burned(tmp_path, chair_plan, settings):
+ spec = make_spec(max_rounds=2, max_usd=0.05)
+ ws = Workspace(tmp_path / "runs" / "cut")
+
+ class _Expensive(FakeAgent):
+ def run(self, job):
+ res = super().run(job)
+ return res.model_copy(update={"usage": Usage(backend="fake", cost_usd=0.04, input_tokens=1000)})
+
+ track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.5, 0.6)), agent=_Expensive(_writer),
+ planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings,
+ runtime=FakeRuntime(Language.THREEJS), policy=RoundPolicy(max_rounds=2, target=0.9))
+ rec = track.run(spec, ws)
+ assert rec.status is RunStatus.BUDGET
+ aborted = json.loads((ws.root / "rounds" / "aborted_r01.json").read_text())
+ assert aborted["index"] == 1 and aborted["usage"]["cost_usd"] > 0 and "BudgetExceeded" in aborted["notes"]
+ assert rec.extra["aborted_rounds"][0]["index"] == 1
+ # the aborted round is NOT resumable state, and its money is in the total
+ assert [r.index for r in rec.rounds] == [0]
+ assert rec.total_usage.cost_usd >= sum(r.usage.cost_usd for r in rec.rounds)
+ cut = next(e for e in EventLog(ws.events_path).read() if e["event"] == "cost.round" and e["round"] == 1)
+ assert cut["wasted"] is True and cut["waste_reason"] == "aborted" and cut["total_usd"] > 0
+
+
+def test_fan_out_workers_inherit_the_callers_context():
+ """Ledger attribution rides on contextvars: a parallel judge sample, a
+ best-of-N candidate or a bench cell must not fall back to the process
+ default just because it ran on a pool thread."""
+ import contextvars
+
+ from codeverse.fanout import fan_out
+
+ var: contextvars.ContextVar[str] = contextvars.ContextVar("attr", default="process-default")
+ var.set("run-42")
+ seen = fan_out(range(6), lambda _i: var.get(), max_workers=4, label="attr")
+ assert set(seen) == {"run-42"}
+ # the caller's own context is untouched by the workers
+ assert var.get() == "run-42"
diff --git a/harness/tests/orchestrator_tracks/test_tracks.py b/harness/tests/orchestrator_tracks/test_tracks.py
index 7f37f3e6..3b8c6db3 100644
--- a/harness/tests/orchestrator_tracks/test_tracks.py
+++ b/harness/tests/orchestrator_tracks/test_tracks.py
@@ -91,7 +91,7 @@ def test_static_track_single_shot_with_repair_and_budget_stop(tmp_path, chair_pl
def responder(req):
n["gen"] += 1
- if req.label == "baseline": # first build fails, the repair fixes it
+ if req.label.startswith("baseline"): # first build fails, the repair fixes it
return f"=== FILE: src/object.js ===\n// {FAIL_MARK}\nexport function build(){{}}\n=== END FILE ==="
return "=== FILE: src/object.js ===\nexport function build(THREE) { return new THREE.Group(); }\n=== END FILE ==="
@@ -99,7 +99,7 @@ def test_static_track_single_shot_with_repair_and_budget_stop(tmp_path, chair_pl
track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.5, 0.55, 0.6, 0.62)), model=model,
planner_model=_planner(chair_plan.model_dump(mode="json")), settings=settings, runtime=FakeRuntime(Language.THREEJS))
rec = track.run(spec, ws)
- assert rec.status is RunStatus.BUDGET and rec.rounds[0].build.ok and "repair attempts: 1 (fixed)" in rec.rounds[0].notes
+ assert rec.status is RunStatus.BUDGET and rec.rounds[0].build.ok and "repair attempts: 1/2 (fixed)" in rec.rounds[0].notes
assert rec.rounds[0].score == pytest.approx(0.5)
assert any(r.label.startswith("r00_baseline_repair") for r in model.requests) or any("Repair" in r.messages[0].text for r in model.requests)
assert rec.total_usage.cost_usd > 0.03 and ws.record_path.is_file()
diff --git a/harness/tests/texturing/test_tools_cli.py b/harness/tests/texturing/test_tools_cli.py
index 5df0cfd8..3c1c74d6 100644
--- a/harness/tests/texturing/test_tools_cli.py
+++ b/harness/tests/texturing/test_tools_cli.py
@@ -14,9 +14,12 @@ from codeverse.workspace import Workspace
from tests.texturing.conftest import FakeJudge, fake_render
-def _ws(tmp_path: Path, chair_glb, chair_spec, chair_plan) -> Workspace:
+def _ws(tmp_path: Path, chair_glb, chair_spec, chair_plan, *, texture: bool = True) -> Workspace:
ws = Workspace(tmp_path / "run").create()
- ws.write_json(ws.spec_path, chair_spec)
+ # texturing.run.texture_requested is the ONE owner of "does this run texture?" —
+ # the tool refuses in a run whose spec says no, so the spec has to say yes here
+ spec = chair_spec.model_copy(update={"options": chair_spec.options.model_copy(update={"texture": texture})})
+ ws.write_json(ws.spec_path, spec)
ws.write_json(ws.plan_path, chair_plan)
shutil.copy(chair_glb, ws.artifacts / "object.glb")
return ws
@@ -54,6 +57,21 @@ def test_texture_pass_tool_without_glb_is_usage_error(tmp_path, chair_spec, chai
assert not obs.ok and "build" in obs.text
+def test_texture_pass_tool_refuses_when_the_run_did_not_ask_for_texturing(
+ tmp_path, chair_glb, chair_spec, chair_plan):
+ """`texture: false` must actually prevent the pass. The tool is registered for every
+ object track, so an agent used to be able to buy a texture pass inside any run
+ (docs/COST.md §15: the quality run paid for two)."""
+ ws = _ws(tmp_path, chair_glb, chair_spec, chair_plan, texture=False)
+ services = TextureServices(image_model=FakeImageModel(), judge_obj=FakeJudge([(0.7, {})]),
+ render=fake_render, cache_dir=tmp_path / "c")
+ obs = get_tool("texture_pass").call(
+ ToolContext(workspace=ws, track="static_object", language="blender",
+ extra={"texture_services": services}), {})
+ assert not obs.ok and "did not ask for texturing" in obs.text
+ assert not (ws.artifacts / "object_textured.glb").exists()
+
+
def test_cli_texture_show_and_help(tmp_path, chair_glb, chair_spec, chair_plan):
from codeverse.cli.main import app
from codeverse.texturing.run import texture_pass