Skip to content

Official Codebase of 3DCodeVerse

Python

cost: the ledger is the only record of money — the guard becomes a clock, the total is the ledger's sum

gaoypeng5e5495faf0ce
diff --git a/eval/docs/COMPLEXITY.md b/eval/docs/COMPLEXITY.md
index 1b80792c..f70b3ff4 100644
--- a/eval/docs/COMPLEXITY.md
+++ b/eval/docs/COMPLEXITY.md
@@ -13,8 +13,8 @@ the second number existed the harness could not tell them apart.
Reconciled against the code and the recorded corpus on **2026-08-24**.
* the vector: `codeverse3d/spatial/complexity.py`
-* where it is stored: `Measurement.extra["complexity"]`, `record.extra["complexity"]`,
- `record.extra["rounds_summary"][i]["complexity"]`, the gallery card + detail page
+* where it is stored: `Measurement.extra["complexity"]` (every round's own: `record.record.round_complexity`),
+ `record.extra["complexity"]`, the gallery card + detail page
* the study: `bench/complexity_report.py`
* the ladder battery: `bench/prompts/complexity_v3.yaml`
diff --git a/eval/tests/test_bench.py b/eval/tests/test_bench.py
index 0cc0fbaa..0b621df6 100644
--- a/eval/tests/test_bench.py
+++ b/eval/tests/test_bench.py
@@ -119,7 +119,8 @@ def test_every_bench_prompt_opens_its_own_run_ledger(tmp_path: Path):
res = run_battery(battery, out, BenchOptions(parallel=2, limit=3), run_fn=run)
assert len(res) == 3
for r in res:
- rows = load_ledger(Path(r.workspace))
+ # the fake run's own rows (make_fake_run writes a ledger like a run) + the metered call
+ rows = [row for row in load_ledger(Path(r.workspace)) if row.label == "api-agent:baseline:t0"]
assert len(rows) == 1, f"{r.id}: {rows}"
assert rows[0].run == r.id and rows[0].cost_usd > 0 # and not a sibling's row
diff --git a/harness/codeverse3d/addons/gallery/detail.py b/harness/codeverse3d/addons/gallery/detail.py
index 7cf819ec..4b806a99 100644
--- a/harness/codeverse3d/addons/gallery/detail.py
+++ b/harness/codeverse3d/addons/gallery/detail.py
@@ -284,8 +284,6 @@ def _cost_panel(entry: RunEntry, ws: Workspace, rec: RunRecord) -> str:
_kv("calls", str(cost.n_calls)),
_kv("minutes", "—" if rec.minutes is None else f"{rec.minutes:.1f}"),
- _kv("unattributed", f"${cost.unattributed_usd:.4f}"),
- _kv("post-run", f"${cost.post_run_usd:.4f}"),
])
by_model = "".join(_kv(k, f"${v:.4f}") for k, v in sorted(cost.by_model.items(), key=lambda kv: -kv[1]))
by_role = "".join(_kv(k, f"${v:.4f}") for k, v in sorted(cost.by_role.items(), key=lambda kv: -kv[1]))
diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.py
index a8c47b39..3a2edb82 100644
--- a/harness/codeverse3d/addons/select.py
+++ b/harness/codeverse3d/addons/select.py
@@ -219,14 +219,19 @@ def package(run_dir: Path | str, round_index: int, *, texture: bool = False, met
``texture=True`` runs the texture pass on that round's own GLB first, unless a pass
already started from those exact bytes (a paid pass is never bought twice); a pack
- ships into the deliverable only when its judge gate shipped it."""
+ ships into the deliverable only when its judge gate shipped it. The record is then
+ re-packaged, so its total takes in what the pick paid for (a texture pass, a pairwise
+ verdict): the run's money is its ledger's (``record.package_run``)."""
ws = Workspace(run_dir)
rec = load_record(ws)
rnd = next((r for r in rec.rounds if r.index == round_index), None)
if rnd is None:
raise ValueError(f"no round {round_index} in {ws.root} (rounds: {[r.index for r in rec.rounds]})")
if texture:
- rec = _texture(ws, rec, round_index, image_model=image_model)
+ _texture(ws, rec, round_index, image_model=image_model)
+ rec = load_record(ws) # the pass wrote extra["texturing"]
+ package_run(ws, rec)
+ ws.write_json(ws.record_path, rec)
manifest = build_deliverable(ws, rec, round_index)
sel = Selection(round=round_index, method=method, scores={r.index: r.score for r in round_rows(ws.root, record=rec)},
textured=any(f.path == "deliverable/object_textured.glb" for f in manifest.files))
@@ -236,8 +241,8 @@ def package(run_dir: Path | str, round_index: int, *, texture: bool = False, met
return ws.deliverable
-def _texture(ws: Workspace, rec: RunRecord, index: int, *, image_model: Any | None) -> RunRecord:
- """The texture pass on round ``index``'s GLB; returns the record as the pass left it."""
+def _texture(ws: Workspace, rec: RunRecord, index: int, *, image_model: Any | None) -> None:
+ """The texture pass on round ``index``'s GLB (it records itself on ``record.json``)."""
from codeverse3d.texturing.run import texture_pass, texture_supported
if not texture_supported(rec.spec.track):
@@ -249,7 +254,7 @@ def _texture(ws: Workspace, rec: RunRecord, index: int, *, image_model: Any | No
raise ValueError(f"round {index} kept no object.glb to texture (artifacts/r{index:02d}/)")
if texture_report_for(ws, glb) is not None:
EventLog(ws.events_path).emit("texture.skipped", reason="already_textured_this_artifact", round=index)
- return rec
+ return
sheet = ws.rebase(rnd.renders.contact_sheet) if rnd.renders is not None and rnd.renders.contact_sheet else None
events = EventLog(ws.events_path)
try:
@@ -258,8 +263,3 @@ def _texture(ws: Workspace, rec: RunRecord, index: int, *, image_model: Any | No
except Exception as e: # noqa: BLE001 — a derived asset pack: the hand-over goes out without it
log.warning("texture pass failed on round %d: %s", index, e)
events.emit("texture.failed", round=index, error=f"{type(e).__name__}: {e}")
- return rec
- rec = load_record(ws) # the pass wrote extra["texturing"]; its spend is post-run: refresh telemetry/
- package_run(ws, rec)
- ws.write_json(ws.record_path, rec)
- return rec
diff --git a/harness/codeverse3d/cli/layout_cmd.py b/harness/codeverse3d/cli/layout_cmd.py
index f59451a0..6e2a1bf3 100644
--- a/harness/codeverse3d/cli/layout_cmd.py
+++ b/harness/codeverse3d/cli/layout_cmd.py
@@ -237,18 +237,13 @@ def print_cost_and_settings(ws: Workspace, record: RunRecord) -> None:
cost, settings = tele.cost, tele.settings
if cost is not None:
u = cost.tokens
- budget_line = fmt_usd(cost.total_usd)
console.print(kv_table("cost", {
- "spent / budget": budget_line,
+ "spent": f"{fmt_usd(cost.total_usd)} (the ledger, list price)",
"minutes": ("-" if record.minutes is None else f"{record.minutes:.1f}")
+ f" (the clock's ceiling: {cost.max_minutes:.0f} min)",
"tokens": f"in {u.input_tokens:,} (cached {u.cached_tokens:,}) · out {u.output_tokens:,} · "
f"thoughts {u.thoughts_tokens:,} · {u.tool_calls} tool calls",
"model calls": cost.n_calls,
- **({"post-run calls": f"{fmt_usd(cost.post_run_usd)} priced outside the run total (texture pass …)"}
- if cost.post_run_usd >= 0.0005 else {}),
- **({"unattributed": f"{fmt_usd(cost.unattributed_usd)} with no per-call row (ledger residual)"}
- if cost.unattributed_usd >= 0.0005 else {}),
"by role": " · ".join(f"{r} {fmt_usd(v)}" for r, v in sorted(cost.by_role.items(), key=lambda kv: -kv[1])) or "-",
"by model": " · ".join(f"{m or '(unknown)'} {fmt_usd(v)}" for m, v in sorted(cost.by_model.items(), key=lambda kv: -kv[1])),
"rounds": " · ".join(f"r{r['index']} {r['kind']} {fmt_usd(r['cost_usd'])}" for r in cost.by_round),
diff --git a/harness/codeverse3d/config.py b/harness/codeverse3d/config.py
index 9cd16cd1..baa36b61 100644
--- a/harness/codeverse3d/config.py
+++ b/harness/codeverse3d/config.py
@@ -253,8 +253,6 @@ class Settings(BaseSettings):
"a 55 800-token plan needs ~930 s and died at the socket, its tokens billed and discarded. "
"Owner's rule 2026-08-27: time may be generous, errors may not.")
profile: str = Field(default="balanced", description="economy | balanced | quality")
- cost_ledger: bool = Field(default=True, description="append one priced row per model call "
- "to the run's telemetry/cost.jsonl (or a per-process log)")
# ------------------------------------------------------------------ switches
# The A/B switches and runtime overrides, each read as C3D_<NAME> at the call that uses
diff --git a/harness/codeverse3d/contracts/run.py b/harness/codeverse3d/contracts/run.py
index 06af7f35..161e42bb 100644
--- a/harness/codeverse3d/contracts/run.py
+++ b/harness/codeverse3d/contracts/run.py
@@ -303,10 +303,13 @@ class StageCost(BaseModel):
class CostSummary(BaseModel):
- """The money ledger: what the run cost, where, and against which budget."""
+ """``telemetry/cost.json``: the run's ledger rows summarised — what the run cost, where, and
+ its minutes against ``max_minutes``. A summary written before 2026-09-22 also carries
+ ``ledger_usd`` / ``unattributed_usd`` / ``post_run_usd`` (the record's total and the ledger
+ were two numbers then) and a span as ``wall_clock_s``: unknown keys, ignored."""
schema_version: int = 1
- total_usd: float = 0.0
+ total_usd: float = Field(default=0.0, description="the ledger's sum at list price = record.total_usage")
minutes: float | None = Field(default=None, description="RunRecord.minutes when the record was written")
max_minutes: float = 0.0
n_calls: int = 0
@@ -314,13 +317,7 @@ class CostSummary(BaseModel):
by_stage: list[StageCost] = Field(default_factory=list)
by_role: dict[str, float] = Field(default_factory=dict, description="planner/generator/judge/… → USD")
by_model: dict[str, float] = Field(default_factory=dict, description="model → USD")
- by_round: list[dict[str, Any]] = Field(default_factory=list, description="index / kind / cost_usd / seconds / score")
- ledger_usd: float = Field(default=0.0, description="sum of the per-call rows in telemetry/usage.jsonl")
- unattributed_usd: float = Field(
- default=0.0, description="total_usd - ledger_usd when positive: money with no per-call row")
- post_run_usd: float = Field(
- default=0.0, description="ledger_usd - total_usd when positive: priced calls the run total does not "
- "include (a texture pass that ran after the loop, …)")
+ by_round: list[dict[str, Any]] = Field(default_factory=list, description="index / kind / cost_usd / minutes / score")
class RunTelemetry(BaseModel):
diff --git a/harness/codeverse3d/cost/billing.py b/harness/codeverse3d/cost/billing.py
deleted file mode 100644
index 602a589b..00000000
--- a/harness/codeverse3d/cost/billing.py
+++ /dev/null
@@ -1,34 +0,0 @@
-"""Which backends actually take dollars out of an account.
-
-``Usage.cost_usd`` answers "what would these tokens cost at list price?". That is the
-right number for a report, a $/complexity-point, or a flywheel record: it is comparable
-across backends and it does not depend on who is paying. It is the WRONG number for
-"what did this run bill", because a backend on a local subscription bills no dollars at
-all — measured 2026-08-25 on ``tsr_scn_temple_night`` (``codex:gpt-5.6-sol``): two
-Blender hero sessions priced at OpenAI list rates read as $7.712 over a bill of exactly
-$0.00 (docs/COST.md §25).
-
-So the split is: the LEDGER and the reports price everything, and :func:`bills_usd`
-shapes the BILLED figure — ``BudgetGuard.billed_usd`` (``record.json`` / the report's
-``spent_usd`` beside ``notional_usd``) and the resume reconcile in
-``tracks/lifecycle.py``. Nothing enforces a dollar ceiling any more (removed in
-fbf89a5); the only ceiling is ``max_minutes``, which ``BudgetGuard.timeout_s`` also
-clips individual sessions against.
-"""
-
-from __future__ import annotations
-
-#: Backends that run on a flat-rate local subscription on this machine, so their
-#: ``cost_usd`` is notional (see ``CLAUDE.md`` "Environment": codex / claude / agy run on
-#: local subscriptions). ``gemini-cli`` is deliberately NOT here: it authenticates with
-#: an API key, so its tokens draw on a real per-token quota even when that quota is free.
-SUBSCRIPTION_BACKENDS = frozenset({"codex", "claude-code", "agy", "antigravity"})
-
-
-def bills_usd(backend: str | None) -> bool:
- """True when this backend's ``cost_usd`` is money someone is actually charged.
-
- Unknown backends bill: a new API provider that nobody remembered to classify must be
- reported as spend, not exempted.
- """
- return (backend or "").strip().lower() not in SUBSCRIPTION_BACKENDS
diff --git a/harness/codeverse3d/cost/instrument.py b/harness/codeverse3d/cost/instrument.py
index 80fd2037..8ad9c786 100644
--- a/harness/codeverse3d/cost/instrument.py
+++ b/harness/codeverse3d/cost/instrument.py
@@ -262,28 +262,19 @@ class MeteredAgent:
# --------------------------------------------------------------------------- factories
def metered_chat_model(model: Any) -> Any:
- """Wrap ``model`` unless it is already metered or metering is off."""
- if model is None or isinstance(model, MeteredChatModel) or not metering_enabled():
+ """Wrap ``model`` unless it is already metered. Metering cannot be switched off: the
+ ledger is the only record of money (the ``C3D_COST_LEDGER=off`` switch went 2026-09-22)."""
+ if model is None or isinstance(model, MeteredChatModel):
return model
return MeteredChatModel(model)
def metered_agent(agent: Any) -> Any:
- if agent is None or isinstance(agent, MeteredAgent) or not metering_enabled():
+ if agent is None or isinstance(agent, MeteredAgent):
return agent
return MeteredAgent(agent)
-def metering_enabled() -> bool:
- """``Settings.cost_ledger`` (env ``C3D_COST_LEDGER=off`` also disables it)."""
- try:
- from codeverse3d.config import get_settings
-
- return bool(get_settings().cost_ledger)
- except Exception: # pragma: no cover
- return True
-
-
# --------------------------------------------------------------------------- run activation
@contextmanager
def run_ledger(workspace: str | Path, *, run: str = "", create: bool = True) -> Iterator[CostLedger | None]:
@@ -303,9 +294,6 @@ def run_ledger(workspace: str | Path, *, run: str = "", create: bool = True) ->
harness run inside it keeps both, and ``bench.run_bench`` can run N prompts in
N threads without their rows mixing — a save-and-restore by value could not,
because a fresh worker saved whatever a sibling had published last."""
- if not metering_enabled():
- yield None
- return
ws = Path(workspace)
if not create and existing_ledger_path(ws) is None:
with bound_run(run or ws.name):
@@ -316,5 +304,4 @@ def run_ledger(workspace: str | Path, *, run: str = "", create: bool = True) ->
yield ledger
-__all__ = ["MeteredAgent", "MeteredChatModel", "metered_agent", "metered_chat_model",
- "metering_enabled", "run_ledger"]
+__all__ = ["MeteredAgent", "MeteredChatModel", "metered_agent", "metered_chat_model", "run_ledger"]
diff --git a/harness/codeverse3d/cost/ledger.py b/harness/codeverse3d/cost/ledger.py
index 7b063630..61b1385d 100644
--- a/harness/codeverse3d/cost/ledger.py
+++ b/harness/codeverse3d/cost/ledger.py
@@ -53,11 +53,6 @@ log = logging.getLogger(__name__)
TELEMETRY_DIR = "telemetry"
TELEMETRY_LEDGER = f"{TELEMETRY_DIR}/cost.jsonl"
-#: env var that points ``record_call`` at a ledger when no path is passed
-#: (``off`` / ``0`` / ``none`` disables ledger writing for the process)
-LEDGER_ENV = "C3D_COST_LEDGER"
-_LEDGER_OFF = frozenset({"off", "0", "no", "none", "false"})
-
DIMENSIONS = ("run", "stage", "role", "backend", "model", "provider", "round", "outcome")
#: backends that bill us themselves (a subscription CLI reporting ``total_cost_usd``).
@@ -88,7 +83,6 @@ class CostLedger:
_fallback_lock = threading.Lock()
_fallback: CostLedger | None = None
-_fallback_read = False
#: sentinel: "this execution context did not state a ledger"
_UNSET: Any = object()
@@ -113,22 +107,21 @@ def bound_ledger(path: str | Path) -> Iterator[CostLedger]:
_default_var.reset(token)
-def default_ledger() -> CostLedger | None:
- """The ledger :func:`record_call` writes to when no ``ledger=`` is given:
- whatever :func:`bound_ledger` bound here, else ``$C3D_COST_LEDGER``, else the
- per-process fallback log (:func:`process_ledger_path`) so a call made outside
- any run — ``3dcode judge``, a bench script, a notebook — is still accounted for.
- ``C3D_COST_LEDGER=off`` turns writing off entirely."""
- global _fallback, _fallback_read
+def default_ledger() -> CostLedger:
+ """The ledger :func:`record_call` writes to when no ``ledger=`` is given: whatever
+ :func:`bound_ledger` bound here, else the per-process fallback log
+ (:func:`process_ledger_path`), so a call made outside any run — ``3dcode judge``, a
+ bench script, a notebook — is still accounted for. There is no off switch: the
+ ledger is the only record of money (``C3D_COST_LEDGER``, which read ``off`` or a path —
+ and, as a Settings field, made every command die on anything but a boolean — went
+ 2026-09-22)."""
+ global _fallback
led = _default_var.get()
if led is not _UNSET:
return led
with _fallback_lock:
- if _fallback_read:
- return _fallback
- env = os.environ.get(LEDGER_ENV, "").strip()
- _fallback = None if env.lower() in _LEDGER_OFF else CostLedger(env or process_ledger_path())
- _fallback_read = True
+ if _fallback is None:
+ _fallback = CostLedger(process_ledger_path())
return _fallback
@@ -292,9 +285,7 @@ def record_call(
**price_fields,
**extra,
)
- target = ledger if isinstance(ledger, CostLedger) else (CostLedger(ledger) if ledger else default_ledger())
- if target is not None:
- target.append(row)
+ (ledger if isinstance(ledger, CostLedger) else CostLedger(ledger) if ledger else default_ledger()).append(row)
if source != "attempt": # an attempt row's tokens are already on its call's own row
book_usage(Usage(backend=row.backend, model=row.model, input_tokens=row.input_tokens,
output_tokens=row.output_tokens, cached_tokens=row.cached_tokens,
@@ -341,6 +332,15 @@ def _key(row: CallCost, dim: str) -> str:
return str(value) if value != "" else "(none)"
+def ledger_usage(rows: Iterable[CallCost]) -> Usage:
+ """The money and tokens of ``rows`` as one ``Usage`` — how a run's ``total_usage`` is read
+ off its ledger (list price; attempt rows are already out: ``load_ledger``)."""
+ t = summarise(rows, dimensions=()).total
+ return Usage(input_tokens=t.input_tokens, output_tokens=t.output_tokens, cached_tokens=t.cached_tokens,
+ thoughts_tokens=t.thoughts_tokens, tool_calls=t.tool_calls, cost_usd=t.cost_usd,
+ latency_ms=t.latency_ms)
+
+
def summarise(rows: Iterable[CallCost], *, dimensions: Sequence[str] = DIMENSIONS) -> Summary:
"""Total + per-dimension buckets. ``dimensions`` may name any scalar field
of :class:`CallCost` (defaults to :data:`DIMENSIONS`)."""
diff --git a/harness/codeverse3d/cost/types.py b/harness/codeverse3d/cost/types.py
index ad8b37d5..44b988ac 100644
--- a/harness/codeverse3d/cost/types.py
+++ b/harness/codeverse3d/cost/types.py
@@ -54,11 +54,10 @@ class Role(StrEnum):
OTHER = "other"
-#: a bare ``GenerationTask.kind`` whose money belongs to a differently-named stage. ONE
-#: vocabulary for both spend paths: ``MeteredAgent.run`` files the session row by
-#: ``job.kind`` and ``tracks.generation.task_stage`` buckets the guard by the same kind —
-#: until 2026-08-29 only the tracks side knew these, so every scene zone / compose /
-#: asset / rebuild session landed in the ledger as ``other``.
+#: a bare ``GenerationTask.kind`` whose money belongs to a differently-named stage:
+#: ``MeteredAgent.run`` files the session row by ``job.kind`` — until 2026-08-29 only the
+#: tracks side knew these, so every scene zone / compose / asset / rebuild session landed in
+#: the ledger as ``other``.
_KIND_STAGES: dict[str, Stage] = {
"generate": Stage.BASELINE,
"rebuild": Stage.REPAIR,
diff --git a/harness/codeverse3d/models/registry.py b/harness/codeverse3d/models/registry.py
index 9e40fe66..2db022f6 100644
--- a/harness/codeverse3d/models/registry.py
+++ b/harness/codeverse3d/models/registry.py
@@ -2,8 +2,7 @@
Every model handed out here is wrapped by ``codeverse3d.cost.instrument`` so each
call lands in the cost ledger (the run's ``telemetry/cost.jsonl`` when a run is
-active, otherwise a per-process log). ``C3D_COST_LEDGER=off`` /
-``Settings.cost_ledger=false`` returns the bare model."""
+active, otherwise a per-process log) — always: the ledger is the only record of money."""
from __future__ import annotations
diff --git a/harness/codeverse3d/orchestrator.py b/harness/codeverse3d/orchestrator.py
index eed26e4e..eb2d79cb 100644
--- a/harness/codeverse3d/orchestrator.py
+++ b/harness/codeverse3d/orchestrator.py
@@ -1,5 +1,5 @@
"""Orchestration: the stage runner with resume, run state, the round policy
-(RoundPolicy), refine-task compilation and the budget.
+(RoundPolicy), refine-task compilation and the budget clock.
One module since 2026-08-28: the seven-file package predates the 1,500-line cap,
nobody ever imported the package itself, and refine_tasks existed only to be
@@ -23,11 +23,10 @@ from typing import Any, Literal, TypeVar
from pydantic import BaseModel, Field, ValidationError
from codeverse3d.contracts.artifacts import GateReport, Judgment
-from codeverse3d.contracts.common import Budget, Usage
+from codeverse3d.contracts.common import Budget
from codeverse3d.contracts.plan import AcceptanceItem, Plan
from codeverse3d.contracts.run import RoundRecord, RunStatus, StepTime
from codeverse3d.conventions import to_snake
-from codeverse3d.cost.billing import bills_usd
from codeverse3d.cost.tally import timed
from codeverse3d.proc import EventLog
from codeverse3d.workspace import Workspace
@@ -448,26 +447,22 @@ def gate_error_count(r: RoundRecord) -> int:
# ===================================================================== budget
-
-#: stage label used when a caller does not say where the money went
-OTHER_STAGE = "other"
-
-
class BudgetExceeded(RuntimeError):
- """Raised when a run crosses its USD or wall-clock ceiling."""
+ """Raised when a run crosses its wall-clock ceiling (``max_minutes``)."""
- def __init__(self, reason: str, *, spent_usd: float):
+ def __init__(self, reason: str):
super().__init__(reason)
self.reason = reason
- self.spent_usd = spent_usd
class BudgetGuard:
- """Thread-safe accumulator of ``Usage`` against a ``Budget``.
+ """The run's clock against ``Budget.max_minutes`` — a LIMIT, not a statistic.
- ``charge`` adds usage and then calls ``check``; ``check`` raises
- ``BudgetExceeded`` when ``max_minutes`` is exceeded. Cost is accumulated for the
- ledger and the run record; the wall clock is the ceiling.
+ It counts every active second, provider errors included, because it has to stop a run
+ even in a 503 storm; the run's minutes (``RunRecord.minutes``, docs/COST.md §31) leave
+ those errors out and never stop anything. ``check`` raises ``BudgetExceeded`` past the
+ ceiling; ``ok`` is its non-raising twin. Money is not the guard's (2026-09-22): the
+ ledger, ``telemetry/cost.jsonl``, is the one record of it.
"""
def __init__(
@@ -482,89 +477,27 @@ class BudgetGuard:
#: ACTIVE seconds carried over from previous sessions of this run (``restore``
#: sets it from the snapshot); downtime between sessions never counts.
self._active_s: float = 0.0
- self.spent = Usage()
- #: the share of ``spent.cost_usd`` that is money someone is actually charged.
- #: The ceilings enforce THIS, not ``spent.cost_usd`` — a backend on a flat-rate
- #: subscription prices its tokens notionally and must not consume a spend guard.
- #: See ``cost/billing.py`` for why, and what still bounds a subscription run.
- self.billed_usd = 0.0
self._lock = threading.Lock()
- self.calls = 0
- #: fraction of the hard ceilings the *baseline* may use (1.0 = no soft cap)
+ #: fraction of the hard ceiling the *baseline* may use (1.0 = no soft cap)
self.soft_fraction = min(1.0, max(0.0, float(soft_fraction)))
#: one-off extension of the HARD wall-clock ceiling (finalise/salvage headroom)
self.grace_minutes = 0.0
- #: stage -> USD, so a stop can say where the money went
- self.by_stage: dict[str, float] = {}
-
- # ----------------------------------------------------------------- accounting
- def charge(self, usage: Usage | None, *, stage: str = OTHER_STAGE, enforce: bool = True) -> None:
- """THE door every dollar goes through — for bucketing and enforcement only.
-
- Accumulates ``usage``, buckets it by ``stage`` and — when ``enforce`` — raises
- ``BudgetExceeded`` if a hard ceiling is now crossed.
- The ledger row is NOT written here: ``cost.instrument`` (MeteredAgent /
- MeteredChatModel) is the one writer, one priced row per real call. Until
- 2026-08-29 this method kept a second, aggregate writer that fired whenever no
- ``run_ledger()`` was active — a direct ``track.run()`` without the CLI, or any
- run with ``C3D_COST_LEDGER=off`` (the guard's own ledger was opened
- unconditionally), so ledger-off runs of that era carry one unpriced row per
- charge; a ledger-off run now writes nothing at all. Role, label and round
- went with that second writer (2026-08-30): ``CallCost`` carries all three per
- call, so the guard bucketing them a second time fed nothing but itself."""
- if usage is None:
- return
- with self._lock:
- self.spent = self.spent + usage
- if bills_usd(getattr(usage, "backend", None)):
- self.billed_usd += float(usage.cost_usd)
- self.calls += 1
- key = stage or OTHER_STAGE
- self.by_stage[key] = self.by_stage.get(key, 0.0) + float(usage.cost_usd)
- if enforce:
- self.check()
-
- def add(self, usage: Usage | None, *, stage: str = OTHER_STAGE) -> None:
- """Account for ``usage`` WITHOUT enforcing the ceilings.
-
- For work that is already done and persisted (a completed judge verdict,
- a pairwise tie-break, the texture pass): the money is spent either way,
- and raising here would throw away a finished, paid-for result. The
- round loop stops at its next ``ok()`` check instead. The dollar is
- still bucketed — "not enforced" never means "not seen"."""
- self.charge(usage, stage=stage, enforce=False)
-
- def mark(self) -> Usage:
- """Snapshot of the running total — diff it with :func:`usage_delta` to see
- what a round burned even when the round itself raised half-way."""
- return self.spent.model_copy(deep=True)
# ----------------------------------------------------------------- resume snapshot
def snapshot(self) -> BudgetSnapshot:
- """Point-in-time guard state for ``run_state.json`` so a resume continues the
- SAME budget instead of a fresh one. Grace and config are deliberately absent:
- grace is one attempt's salvage headroom — persisting it would ratchet the hard
- ceiling run over run, and the ceilings always come from the current spec."""
+ """The clock for ``run_state.json``, so a resume continues the SAME budget instead of
+ a fresh one. Grace and config are deliberately absent: grace is one attempt's salvage
+ headroom — persisting it would ratchet the hard ceiling run over run, and the ceiling
+ always comes from the current spec."""
with self._lock:
- return BudgetSnapshot(
- spent=self.spent.model_copy(deep=True),
- billed_usd=self.billed_usd,
- calls=self.calls,
- by_stage=dict(self.by_stage),
- active_s=self._active_s + (time.time() - self.start_time),
- )
+ return BudgetSnapshot(active_s=self._active_s + (time.time() - self.start_time))
def restore(self, snap: BudgetSnapshot) -> None:
- """Adopt a snapshot: money, calls and buckets keep counting; ``start_time``
- stays *now*, so ``elapsed_minutes`` is prior ACTIVE seconds plus this session
- — never the downtime in between. The ceilings come from the (possibly raised)
- spec budget, so a raised cap (``--max-minutes`` / ``--rounds``) grants exactly
+ """Adopt a snapshot: ``start_time`` stays *now*, so ``elapsed_minutes`` is prior ACTIVE
+ seconds plus this session — never the downtime in between. The ceiling comes from
+ the (possibly raised) spec budget, so a raised cap (``--max-minutes``) grants exactly
the difference, never a fresh full cap."""
with self._lock:
- self.spent = snap.spent.model_copy(deep=True)
- self.billed_usd = float(snap.billed_usd)
- self.calls = int(snap.calls)
- self.by_stage = dict(snap.by_stage)
self._active_s = float(snap.active_s)
self.start_time = time.time()
@@ -586,14 +519,10 @@ class BudgetGuard:
self.grace_minutes = max(self.grace_minutes, max(0.0, float(minutes)))
def check(self) -> None:
- """Raise ``BudgetExceeded`` if any hard ceiling has been crossed."""
- spent = self.billed_usd
+ """Raise ``BudgetExceeded`` if the hard ceiling has been crossed."""
elapsed = self.elapsed_minutes()
if elapsed > self.hard_minutes:
- raise BudgetExceeded(
- f"elapsed {elapsed:.1f} min exceeds max_minutes {self.hard_minutes:.1f}",
- spent_usd=spent,
- )
+ raise BudgetExceeded(f"elapsed {elapsed:.1f} min exceeds max_minutes {self.hard_minutes:.1f}")
# ----------------------------------------------------------------- soft cap
def soft_minutes(self) -> float:
@@ -629,51 +558,14 @@ class BudgetGuard:
left = max(0.0, (self.soft_minutes() if soft else self.hard_minutes) - self.elapsed_minutes()) * 60.0
return int(max(floor_s, min(float(want_s), left) if left > 0 else floor_s))
- def summary(self) -> dict[str, float | int]:
- return {
- "spent_usd": round(self.billed_usd, 4),
- # what it WOULD have cost at list price; equal to spent_usd unless a
- # subscription backend ran (cost/billing.py)
- "notional_usd": round(self.spent.cost_usd, 4),
- "elapsed_min": round(self.elapsed_minutes(), 2),
- "soft_fraction": round(self.soft_fraction, 3),
- "calls": self.calls,
- "input_tokens": self.spent.input_tokens,
- "output_tokens": self.spent.output_tokens,
- }
-
- def stage_summary(self) -> dict[str, float]:
- """``{stage: USD}`` over the whole run (what the ``cost.round`` events add up to)."""
- return {k: round(v, 6) for k, v in sorted(self.by_stage.items(), key=lambda kv: -kv[1])}
+ def summary(self) -> dict[str, float]:
+ """``record.extra["budget"]``: the clock at the end, beside its ceiling."""
+ return {"elapsed_min": round(self.elapsed_minutes(), 2), "max_minutes": self.budget.max_minutes}
class BudgetSnapshot(BaseModel):
- """What survives a resume (``run_state.extra["budget_snapshot"]``).
-
- The four accumulator fields of :class:`BudgetGuard` plus cumulative ACTIVE
- seconds. Grace (``grace_minutes``) and config (ceilings, soft fraction) are
- EXCLUDED on purpose: grace is per-attempt salvage headroom — persisting it
- would ratchet the hard ceiling — and config always comes from the current
- spec/settings."""
-
- spent: Usage
- billed_usd: float
- calls: int
- by_stage: dict[str, float]
- active_s: float
-
-
-def usage_delta(after: Usage, before: Usage) -> Usage:
- """``after - before`` field by field (never negative). Used to report what a
- round burned when it raised before it could fold its own usage together."""
- return Usage(
- backend=after.backend or before.backend,
- model=after.model or before.model,
- input_tokens=max(0, after.input_tokens - before.input_tokens),
- output_tokens=max(0, after.output_tokens - before.output_tokens),
- cached_tokens=max(0, after.cached_tokens - before.cached_tokens),
- thoughts_tokens=max(0, after.thoughts_tokens - before.thoughts_tokens),
- tool_calls=max(0, after.tool_calls - before.tool_calls),
- cost_usd=max(0.0, after.cost_usd - before.cost_usd),
- latency_ms=max(0, after.latency_ms - before.latency_ms),
- )
+ """What survives a resume (``run_state.extra["budget_snapshot"]``): cumulative ACTIVE
+ seconds. A snapshot written before 2026-09-22 also carries the guard's money
+ (``spent`` / ``billed_usd`` / ``calls`` / ``by_stage``): unknown keys, ignored."""
+
+ active_s: float = 0.0
diff --git a/harness/codeverse3d/record/record.py b/harness/codeverse3d/record/record.py
index 8b780193..95037ced 100644
--- a/harness/codeverse3d/record/record.py
+++ b/harness/codeverse3d/record/record.py
@@ -1,9 +1,9 @@
"""Run records: finalise / load / iterate.
``finalize_record`` is the last step of every track run: it fills the
-environment (tool versions + harness git sha) and the totals when the track left
-them empty, adds a compact per-round summary under ``record.extra["rounds_summary"]``,
-writes ``telemetry/`` (see docs/RUN_LAYOUT.md) and writes ``record.json`` atomically. ``deliverable/`` is not
+environment (tool versions + harness git sha), reads the run's total off its cost ledger
+(``telemetry/cost.jsonl``, the one record of money), writes ``telemetry/`` (see
+docs/RUN_LAYOUT.md) and writes ``record.json`` atomically. ``deliverable/`` is not
the run's: which round to hand over is ``codeverse3d.addons.select``'s question
(2026-09-22). Telemetry is best-effort: a run is never failed by it.
"""
@@ -22,8 +22,8 @@ from typing import Any, NamedTuple
from codeverse3d import __version__
from codeverse3d.config import get_settings
-from codeverse3d.contracts.common import Usage
from codeverse3d.contracts.run import RoundRecord, RunId, RunRecord
+from codeverse3d.cost.ledger import existing_ledger_path, ledger_usage, load_ledger
from codeverse3d.proc import version_line
from codeverse3d.workspace import Workspace
@@ -149,7 +149,7 @@ def round_complexity(r: RoundRecord) -> dict[str, Any] | None:
def round_summary(r: RoundRecord) -> dict[str, Any]:
- """Compact per-round digest (for record.extra, dashboards and the CLI).
+ """Compact per-round digest (the dataset sample's ``rounds_summary``).
A degraded judgment shows up as score/passed None + ``judge_degraded``."""
j = effective_judgment(r)
@@ -166,28 +166,19 @@ def round_summary(r: RoundRecord) -> dict[str, Any]:
"issues": len(j.issues) if j else 0,
"judge_degraded": r.judgment is not None and j is None,
"cost_usd": round(r.usage.cost_usd, 6),
- "duration_s": round(r.duration_s, 1),
+ "minutes": round(r.minutes, 2),
"complexity": cx.get("index") if cx else None,
}
-def _sum_usage(rounds: list[RoundRecord]) -> Usage:
- """Round usage already includes the round's judge call (tracks/steps.py) — do not add it twice."""
- return sum((r.usage for r in rounds), Usage())
-
-
def fill_derived(record: RunRecord) -> RunRecord:
- """Fill total_usage/finished_at/environment when absent, the per-round summary and the
- complexity block. No best round, baseline or final score: which round counts is
- ``addons.select``'s question (2026-09-22)."""
- if record.total_usage.cost_usd == 0 and record.total_usage.input_tokens == 0 and record.rounds:
- record.total_usage = _sum_usage(record.rounds)
+ """Fill finished_at/environment when absent and the complexity block. No best round,
+ baseline or final score: which round counts is ``addons.select``'s question (2026-09-22)."""
if record.finished_at is None:
record.finished_at = datetime.now(UTC)
env = dict(environment_versions())
env.update(record.environment) # the track's own entries win
record.environment = env
- record.extra["rounds_summary"] = [round_summary(r) for r in record.rounds]
block = complexity_block(record)
if block is not None:
record.extra["complexity"] = block
@@ -216,12 +207,18 @@ def complexity_block(record: RunRecord) -> dict[str, Any] | None:
# --------------------------------------------------------------------------- io
def package_run(ws: Workspace, record: RunRecord) -> None:
- """Materialise ``telemetry/`` and mirror it onto the record.
+ """Read the run's total off its ledger, materialise ``telemetry/`` and mirror it onto the
+ record. ``record.total_usage`` is the sum of ``telemetry/cost.jsonl`` at list price — every
+ billed row once: the planner, an aborted round, a retried session, a round-trip the
+ provider billed and the call discarded, and post-run work (a pick's texture pass) once it
+ joined the ledger. A run with no ledger keeps what it has.
Best-effort by contract: a failure is logged and the run still gets its
``record.json`` (the old layout is always enough to read a run)."""
from codeverse3d.record.telemetry import build_telemetry
+ if existing_ledger_path(ws.root) is not None:
+ record.total_usage = ledger_usage(load_ledger(ws.root))
try:
ws.ensure_layout()
except OSError as e: # read-only mount, exotic filesystem
diff --git a/harness/codeverse3d/record/telemetry.py b/harness/codeverse3d/record/telemetry.py
index 097e3c78..e9bceae2 100644
--- a/harness/codeverse3d/record/telemetry.py
+++ b/harness/codeverse3d/record/telemetry.py
@@ -33,8 +33,8 @@ from codeverse3d.contracts.run import (
SettingsSnapshot,
StageCost,
)
-from codeverse3d.cost.ledger import TELEMETRY_LEDGER, load_ledger
-from codeverse3d.cost.types import Stage
+from codeverse3d.cost.ledger import TELEMETRY_LEDGER, load_ledger, summarise
+from codeverse3d.cost.types import CallCost, Stage
from codeverse3d.proc import read_json_or_none, write_json_atomic
from codeverse3d.workspace import Workspace
@@ -187,68 +187,37 @@ def stage_order() -> tuple[str, ...]:
return tuple(s.value for s in Stage)
-def ledger_rows(ws: Workspace) -> list[dict[str, Any]]:
- """The run's priced per-call rows, ``telemetry/cost.jsonl`` (none when it has no ledger).
- The cost package owns the pricing and the double-counting rules — this bucket never
- re-implements them."""
+def ledger_rows(ws: Workspace) -> list[CallCost]:
+ """The run's priced per-call rows, ``telemetry/cost.jsonl`` (none when it has no ledger)."""
try:
- return [r.model_dump(mode="json") for r in load_ledger(ws.root)]
+ return load_ledger(ws.root)
except Exception as e: # noqa: BLE001 - accounting must never fail a run
log.warning("cost ledger unreadable for %s: %s", ws.root, e)
return []
-def _num(row: dict[str, Any], key: str) -> float:
- v = row.get(key)
- return float(v) if isinstance(v, (int, float)) else 0.0
-
-
-def cost_summary(record: RunRecord, rows: list[dict[str, Any]]) -> CostSummary:
- """Money + tokens: totals, per stage, per role, per model, per round, budget, wall clock."""
- by_stage: dict[str, StageCost] = {}
- by_model: dict[str, float] = {}
- by_role: dict[str, float] = {}
- residual = 0.0
- for row in rows:
- stage = str(row.get("stage") or "other")
- sc = by_stage.setdefault(stage, StageCost(stage=stage))
- sc.calls += int(row.get("n_calls") or 1)
- sc.input_tokens += int(_num(row, "input_tokens"))
- sc.output_tokens += int(_num(row, "output_tokens"))
- sc.cached_tokens += int(_num(row, "cached_tokens"))
- sc.thoughts_tokens += int(_num(row, "thoughts_tokens"))
- sc.cost_usd += _num(row, "cost_usd")
- sc.seconds += _num(row, "latency_ms") / 1000.0
- model = str(row.get("model") or row.get("model_id") or "(unknown)")
- by_model[model] = round(by_model.get(model, 0.0) + _num(row, "cost_usd"), 6)
- role = str(row.get("role") or "other")
- by_role[role] = round(by_role.get(role, 0.0) + _num(row, "cost_usd"), 6)
- if str(row.get("source")) == "residual" or str(row.get("label")) == "unattributed":
- residual += _num(row, "cost_usd")
- for sc in by_stage.values():
- sc.cost_usd = round(sc.cost_usd, 6)
- sc.seconds = round(sc.seconds, 1)
- order = stage_order()
- ordered = [by_stage[s] for s in order if s in by_stage]
- ordered += [sc for s, sc in sorted(by_stage.items()) if s not in order]
- total = float(record.total_usage.cost_usd)
- ledger = round(sum(_num(r, "cost_usd") for r in rows), 6)
+def cost_summary(record: RunRecord, rows: list[CallCost]) -> CostSummary:
+ """The ledger rows as the run layout shows them: total, per stage / role / model (the
+ ledger's own ``summarise`` — money by stage is computed there and nowhere else), per round,
+ and the run's minutes against ``max_minutes``."""
+ summ = summarise(rows, dimensions=("stage", "role", "model"))
+ rank = {s: i for i, s in enumerate(stage_order())}
+ by_stage = [StageCost(stage=b.key, calls=b.n_calls, input_tokens=b.input_tokens, output_tokens=b.output_tokens,
+ cached_tokens=b.cached_tokens, thoughts_tokens=b.thoughts_tokens,
+ cost_usd=round(b.cost_usd, 6), seconds=round(b.latency_ms / 1000.0, 1))
+ for b in sorted(summ.dimension("stage").values(), key=lambda b: (rank.get(b.key, len(rank)), b.key))]
minutes = record.minutes
- budget = record.spec.budget
return CostSummary(
- total_usd=round(total, 6),
+ total_usd=round(summ.total.cost_usd, 6),
minutes=None if minutes is None else round(minutes, 2),
- max_minutes=budget.max_minutes,
- n_calls=sum(int(r.get("n_calls") or 1) for r in rows),
+ max_minutes=record.spec.budget.max_minutes,
+ n_calls=summ.total.n_calls,
tokens=record.total_usage,
- by_stage=ordered,
- by_role=by_role,
- by_model=by_model,
+ by_stage=by_stage,
+ by_role={k: round(b.cost_usd, 6) for k, b in summ.dimension("role").items()},
+ by_model={k: round(b.cost_usd, 6) for k, b in summ.dimension("model").items()},
by_round=[{"index": r.index, "kind": r.kind, "cost_usd": round(r.usage.cost_usd, 6),
- "seconds": round(r.duration_s, 1), "score": r.score} for r in record.rounds],
- ledger_usd=ledger,
- unattributed_usd=round(residual, 6),
- post_run_usd=round(max(0.0, ledger - total - residual), 6),
+ "minutes": round(r.minutes, 2), "score": r.score} for r in record.rounds],
)
diff --git a/harness/codeverse3d/tracks/candidates.py b/harness/codeverse3d/tracks/candidates.py
index f72fe629..feb4cd32 100644
--- a/harness/codeverse3d/tracks/candidates.py
+++ b/harness/codeverse3d/tracks/candidates.py
@@ -29,7 +29,7 @@ from codeverse3d.contracts.artifacts import BuildResult, Measurement, RenderSet
from codeverse3d.contracts.common import Usage
from codeverse3d.contracts.run import RoundRecord
from codeverse3d.conventions import OBJECT_VIEWS_QUICK
-from codeverse3d.cost.tally import timed
+from codeverse3d.cost.tally import tally, timed
from codeverse3d.orchestrator import BudgetExceeded, gate_error_count
from codeverse3d.proc import EventLog, fan_out
from codeverse3d.tracks.common import RunContext
@@ -62,8 +62,10 @@ def run_best_of_n(track: Any, ctx: RunContext, tasks: Sequence[GenerationTask],
ctx.events.emit("candidate.start", candidate=k, workspace=str(sub.ws.root))
# kind="candidate" is what the cost ledger files the sessions under (Stage.CANDIDATE)
tasks_k = [t.model_copy(update={"label": f"{t.label}_{label}", "kind": "candidate"}) for t in tasks]
- rec = _run_round(sub, index=0, kind="candidate", tasks=tasks_k, pipeline=pipeline, files_hint=list(files_hint),
- render=partial(quick_render, pipeline=pipeline), geometry_views=False)
+ with tally() as spent: # the candidate's cost: the ledger rows it books
+ rec = _run_round(sub, index=0, kind="candidate", tasks=tasks_k, pipeline=pipeline,
+ files_hint=list(files_hint), render=partial(quick_render, pipeline=pipeline),
+ geometry_views=False, spent=spent)
build_ok = rec.build is not None and rec.build.ok
cand = CandidateRecord(index=k, label=label, workspace=str(sub.ws.root), commit=rec.commit, build_ok=build_ok,
score=rec.score, gate_errors=gate_error_count(rec), usage=rec.usage,
diff --git a/harness/codeverse3d/tracks/generation.py b/harness/codeverse3d/tracks/generation.py
index 67b7a30f..780f1343 100644
--- a/harness/codeverse3d/tracks/generation.py
+++ b/harness/codeverse3d/tracks/generation.py
@@ -24,11 +24,10 @@ So there is **no default cap** (``DEFAULT_AGENT_MAX_TURNS`` is 0 → the backend
stays **graceful**: the session is not killed but asked, in a short wrap-up
(``DEFAULT_WRAPUP_TURNS``), for one last build + summary.
-**Every dollar is charged.** Both strategies spend through ``BudgetGuard.charge`` in
-the stage the task belongs to, so a retried session (``<label>.a2``) and a wrap-up
-session are visible to the guard, the record and the ledger even when the *next*
-attempt then raises. The label, round and outcome of each call are on the ledger row
-``cost.instrument`` writes, not on the guard (2026-08-30).
+**Every dollar is on the ledger.** Both strategies spend through the metered model or
+agent (``cost.instrument``), one row per call or session — a retried session
+(``<label>.a2``) and a wrap-up session included, even when the *next* attempt then
+raises. The run's clock (``BudgetGuard``) is checked at the session boundary.
"""
from __future__ import annotations
@@ -414,13 +413,11 @@ def generate_files(
)
resp = model.generate(req)
# + every paid round-trip the winner does not represent (a billed-but-invalid
- # response, a late hedge loser): the ledger records those as source="extra";
- # without this the guard never sees them (measured $0.18 on one call).
+ # response, a late hedge loser): the ledger records those as source="extra"
usage = resp.usage + (resp.raw.get("wasted_usage") or Usage())
- # book the money WITHOUT enforcing: the response is already paid for, and raising
- # here would discard it before transcript/parse/write_files. The ceiling is
- # enforced at the round's phase boundary instead (steps._run_phase).
- _charge(budget, usage, task=task, enforce=False)
+ # the clock is not enforced here: the response is already paid for, and raising would
+ # discard it before transcript/parse/write_files — the round's phase boundary does
+ # (steps._run_phase)
retry = False
if _is_truncated(resp):
# cut off by max_output_tokens: the envelope is unterminated — one retry with a
@@ -443,8 +440,6 @@ def generate_files(
"max_wait_s": _deadline_preflight(budget, RETRY_DEADLINE_S)})
resp = model.generate(req)
usage = usage + resp.usage
- _charge(budget, resp.usage, task=task,
- enforce=False)
traj = ws.trajectory_dir(task.label.replace("/", "_"), task.round)
(traj / "prompt.md").write_text(f"# system\n{system}\n\n# user\n{task.prompt}\n")
(traj / "response.md").write_text(resp.text or "")
@@ -506,32 +501,6 @@ def generate_files(
)
-def task_stage(task: GenerationTask) -> str:
- """The cost stage one generation task spends in: its kind through
- ``cost.types.stage_for_label`` — the SAME vocabulary ``MeteredAgent`` files the
- session row by — else the label, which ``record_call`` maps the same way
- (``asset_koi`` → assets, ``r00_baseline_repair1`` → repair)."""
- from codeverse3d.cost.types import Stage, stage_for_label
-
- stage = stage_for_label(task.kind)
- return str(stage) if stage is not Stage.OTHER else (task.label or "other")
-
-
-def _charge(budget: Any | None, usage: Usage, *, task: GenerationTask, enforce: bool = True) -> None:
- """Spend through the guard in the stage this task belongs to.
-
- ``enforce=False`` books the dollar without raising — the single-shot path uses it
- so a paid response is still parsed and written to disk; the ceiling is enforced at
- the round's phase boundary (``steps._run_phase``) after the work is persisted.
-
- Label, round and outcome are NOT passed on (2026-08-30): the guard buckets by stage
- and enforces, and the per-call record with all three is the ledger row
- ``cost.instrument`` writes."""
- if budget is None:
- return
- budget.charge(usage, stage=task_stage(task), enforce=enforce)
-
-
def _deadline_preflight(budget: Any | None, wait_s: float, *, soft: bool = False,
floor_s: float = 120.0) -> float:
"""Gate a session on the run clock: refuse to start past the HARD ceiling (a
@@ -686,10 +655,11 @@ def run_agent_task(
class _SessionAcc:
"""Runs the agent sessions of ONE task and keeps their accounting.
- Each session is charged the moment it returns (label ``<task>``/``<task>.a2``
- for the retry, ``<task>.wrapup`` for the landing session), so a crash in a
- later session can never erase what an earlier one already spent — the ``.a2``
- hole of docs/COST.md §6."""
+ Each session is a ledger row the moment it returns (label ``<task>``/``<task>.a2`` for
+ the retry, ``<task>.wrapup`` for the landing session — ``cost.instrument.MeteredAgent``),
+ so a crash in a later session can never erase what an earlier one already spent — the
+ ``.a2`` hole of docs/COST.md §6. The run's clock is checked after each session that
+ did any work, as the guard's charge did before money left it."""
def __init__(self, *, task: GenerationTask, budget: Any | None):
self.task = task
@@ -722,8 +692,8 @@ class _SessionAcc:
return None
self.usage = self.usage + res.usage
self.turns += session_turns(res)
- if res.usage.cost_usd or res.usage.input_tokens or res.usage.output_tokens: # an empty session books no call
- _charge(self.budget, res.usage, task=self.task)
+ if self.budget is not None and (res.usage.cost_usd or res.usage.input_tokens or res.usage.output_tokens):
+ self.budget.check()
return res
diff --git a/harness/codeverse3d/tracks/lifecycle.py b/harness/codeverse3d/tracks/lifecycle.py
index bf2dc4e0..2ca015be 100644
--- a/harness/codeverse3d/tracks/lifecycle.py
+++ b/harness/codeverse3d/tracks/lifecycle.py
@@ -26,7 +26,8 @@ from codeverse3d.contracts.plan import Plan
from codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus
from codeverse3d.contracts.spec import Spec
from codeverse3d.cost.context import run_binding
-from codeverse3d.cost.instrument import run_ledger
+from codeverse3d.cost.instrument import metered_agent, metered_chat_model, run_ledger
+from codeverse3d.cost.ledger import ledger_usage, load_ledger
from codeverse3d.orchestrator import (
BudgetExceeded,
BudgetGuard,
@@ -59,7 +60,6 @@ from codeverse3d.tracks.steps import (
load_round_journal,
rejudge_round,
run_round,
- sum_usage,
)
from codeverse3d.workspace import Workspace
@@ -68,35 +68,6 @@ log = logging.getLogger(__name__)
REFERENCE_RUBRIC = "reference_v1"
-def _reconcile_billed_from_ledger(budget: BudgetGuard, ws: Workspace, events: EventLog) -> None:
- """Adopt the run ledger's total when it exceeds the restored snapshot.
-
- The snapshot is saved at boundaries; the ledger is appended per CALL, so a crash
- between a round's spend and its save loses that money from the snapshot but never
- from the ledger. Taking the larger of the two makes resume charge for everything
- the provider actually billed — silently under-counting is how a resumed run walks
- past its ceiling. Attempt rows stay excluded (the winner is already on the logical
- row), and so do subscription-backend rows: their ``cost_usd`` is notional, not money
- (the same ``bills_usd`` predicate the live path uses — ``BudgetGuard.charge``), so a
- codex/claude/agy run resumed offline keeps billed at $0. A missing or unreadable
- ledger simply leaves the snapshot alone. ``load_ledger(root)`` reads the one ledger
- the metered models/agents write, ``telemetry/cost.jsonl``; the root-level
- ``cost_ledger.jsonl`` leg went on 2026-08-30 (an audit of every run under $HOME found
- 677 symlinks to that name and zero real files — ``LEDGER_NAME`` is the alias only)."""
- try:
- from codeverse3d.cost.billing import bills_usd
- from codeverse3d.cost.ledger import load_ledger
-
- billed = sum(float(r.cost_usd or 0.0) for r in load_ledger(ws.root) if bills_usd(r.backend))
- except Exception as e: # noqa: BLE001 — accounting repair must never block a resume
- log.debug("ledger reconcile skipped: %s", e)
- return
- if billed > budget.billed_usd + 1e-9:
- events.emit("resume.billed_reconciled", snapshot_usd=round(budget.billed_usd, 6),
- ledger_usd=round(billed, 6))
- budget.billed_usd = billed
-
-
def plan_stage_inputs(spec: Spec) -> dict[str, Any]:
"""Only the spec fields the PLAN depends on: hashing the whole Spec meant that
raising ``budget.max_minutes`` in spec.json (the only way to continue a BUDGET-stopped
@@ -273,13 +244,13 @@ class BaseTrack:
self.prepare(ctx, runner)
stop = self._round_loop(ctx, rounds)
except BudgetExceeded as e:
- events.emit("budget.exceeded", reason=e.reason, spent_usd=round(e.spent_usd, 4))
+ events.emit("budget.exceeded", reason=e.reason)
stop, error = RunStatus.BUDGET, e.reason
self._salvage_baseline(ctx, rounds)
except Exception as e: # noqa: BLE001 — persist a FAILED record, then fail loud
error = f"{type(e).__name__}: {e}"
events.emit("run.failed", error=error, traceback=traceback.format_exc()[-3000:])
- self._save_budget(ctx) # mid-round charges must survive for resume
+ self._save_budget(ctx) # the clock and the step log must survive for resume
state.status, state.error = RunStatus.FAILED, error
state.save(ws)
rec = self._record(ctx, rounds, RunStatus.FAILED, error=error)
@@ -377,11 +348,9 @@ class BaseTrack:
budget = BudgetGuard(spec.budget, soft_fraction=self.soft_budget_fraction)
snap = state.extra.get("budget_snapshot")
if snap:
- # full guard state: money, calls, buckets AND active minutes keep counting,
- # so a raised cap (--max-minutes / --rounds) grants only the difference,
- # never a fresh full cap.
+ # the active minutes keep counting, so a raised cap (--max-minutes / --rounds)
+ # grants only the difference, never a fresh full cap
budget.restore(BudgetSnapshot.model_validate(snap))
- _reconcile_billed_from_ledger(budget, ws, events)
policy = self._policy or RoundPolicy(max_rounds=spec.budget.max_rounds)
policy = replace(policy, n_candidates=self._resolve_candidates(spec, settings))
# reference images: static objects are scored with reference_v1 (adds the measured silhouette
@@ -389,7 +358,10 @@ class BaseTrack:
rubric = REFERENCE_RUBRIC if spec.references and self.track is Track.STATIC_OBJECT else self.rubric
ctx = RunContext(spec=spec, ws=ws, events=events, settings=settings, budget=budget, runtime=runtime,
services=self.services, state=state, policy=policy, track=self.track, rubric=rubric,
- agent_id=spec.backends.generator, agent=self._agent, model=self._model)
+ agent_id=spec.backends.generator,
+ # metered like the ones services hand out: an injected agent or model spends
+ # money too, and the ledger is the only record of it
+ agent=metered_agent(self._agent), model=metered_chat_model(self._model))
ctx.contract_text = language_text(spec.language, "contract.md")
ctx.cookbook_rel = language_prompt(spec.language, "cookbook.md")
ctx.cookbook_text = language_text(spec.language, "cookbook.md")
@@ -438,10 +410,10 @@ class BaseTrack:
ctx.state.status = RunStatus.PLANNING
ctx.state.save(ctx.ws)
try:
- plan = run_planner(ctx.spec, ctx.spec.backends.planner, self.plan_model, ctx.ws, model=self._planner_model,
- events=ctx.events, budget=ctx.budget)
+ plan = run_planner(ctx.spec, ctx.spec.backends.planner, self.plan_model, ctx.ws,
+ model=metered_chat_model(self._planner_model), events=ctx.events, budget=ctx.budget)
finally:
- self._save_budget(ctx) # charged on success AND PlanningError
+ self._save_budget(ctx) # the clock and the plan step, on success AND PlanningError
return plan
# ------------------------------------------------------------------ stages
@@ -602,7 +574,7 @@ class BaseTrack:
cut mid-way can leave src/ past it, so it is put back and rebuilt — the only restore a
run does (every round's own build is kept under artifacts/rNN/, and choosing a round
to hand over is ``codeverse3d.addons.select``'s job, after the run)."""
- self._save_budget(ctx) # aborted-round / stage charges must survive for resume
+ self._save_budget(ctx) # the clock and an aborted round's steps must survive for resume
last = rounds[-1] if rounds else None
rebuild_err = ""
rebuild_ok = False
@@ -634,7 +606,7 @@ class BaseTrack:
self.services.finalize_record(ctx.ws, rec)
ctx.events.emit("run.done", status=status.value, rounds=len(rounds),
last_score=last.score if last is not None else None,
- cost_usd=round(rec.total_usage.cost_usd, 4))
+ cost_usd=round(rec.total_usage.cost_usd, 4), minutes=rec.minutes)
return rec
@staticmethod
@@ -650,21 +622,18 @@ class BaseTrack:
return False
def _record(self, ctx: RunContext, rounds: list[RoundRecord], status: RunStatus, *, error: str) -> RunRecord:
- total = sum_usage(rounds)
- # planner, aborted rounds, retried sessions: all in the guard,
- # none of them in ``rounds`` — the guard is the honest total (docs/COST.md §6).
- if ctx.budget.spent.cost_usd > total.cost_usd:
- total = ctx.budget.spent
+ """The run record. Its ``total_usage`` is the sum of the run's ledger at list price —
+ every billed row once: the planner, the scene stages, aborted rounds, retried sessions,
+ round-trips the provider billed and the call discarded — not a sum kept here."""
cands = ctx.ws.root / "rounds" / "candidates.json" # best-of-N summary: the file is the one copy
extra: dict[str, Any] = {"stop_reason": status.value, "rubric": ctx.rubric, "budget": ctx.budget.summary(),
"n_candidates": ctx.policy.n_candidates,
- "candidates": ctx.ws.read_json(cands) if cands.is_file() else None,
- "cost_by_stage": ctx.budget.stage_summary()}
+ "candidates": ctx.ws.read_json(cands) if cands.is_file() else None}
if ctx.extra.get("finalise_rebuild_failed"):
extra["finalise_rebuild_failed"] = ctx.extra["finalise_rebuild_failed"]
if ctx.extra.get("aborted_rounds"):
# rounds the budget (or a crash) cut after the money was spent: they are not
- # in ``rounds``, so name them here and keep their dollars in total_usage.
+ # in ``rounds``, so name them here (their dollars are in the ledger, the total).
extra["aborted_rounds"] = ctx.extra["aborted_rounds"]
# A resume rewrite must not drop what earlier sessions / other packages put on
# the record post-hoc (flywheel captions, texturing/run.record_texturing, prior
@@ -681,8 +650,8 @@ class BaseTrack:
extra["aborted_rounds"] = prior_aborted + [a for a in ctx.extra["aborted_rounds"]
if a not in prior_aborted]
return RunRecord(
- spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, total_usage=total,
- steps=list(ctx.state.steps),
+ spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds,
+ total_usage=ledger_usage(load_ledger(ctx.ws.root)), steps=list(ctx.state.steps),
environment={"python": platform.python_version(), "host": platform.node(), "track": self.track.value,
"language": ctx.language.value, "generator": ctx.agent_id},
prompt_hashes={**prior_hashes, **dict(ctx.prompt_hashes)}, started_at=ctx.state.started_at,
diff --git a/harness/codeverse3d/tracks/planner.py b/harness/codeverse3d/tracks/planner.py
index 538cf4dc..73d94195 100644
--- a/harness/codeverse3d/tracks/planner.py
+++ b/harness/codeverse3d/tracks/planner.py
@@ -827,12 +827,10 @@ def plan[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P], ws: Works
against the same ceiling.
``budget`` (the run's ``BudgetGuard``; ``guard`` below, because ``budget`` becomes this
- request's PlanBudget) is booked with a non-enforcing ``add`` the moment each call is
- PAID — brief, first call, every re-ask — so success, ``PlanningError`` and a crash on a
- LATER attempt all leave the earlier dollars in the guard. ``PlanningError`` still
- carries the total usage for guard-less callers; when a guard is given those dollars are
- already booked and the caller must NOT charge them again. The ceilings are enforced
- once, after the plan is written."""
+ request's PlanBudget) only clips each call's wait to the clock left; every call is a
+ ledger row of its own (the metered model), so success, ``PlanningError`` and a crash on
+ a LATER attempt all leave the earlier dollars on the ledger. ``PlanningError`` carries
+ the total usage for a caller that wants it."""
guard = budget
if model is None:
from codeverse3d.models import get_chat_model
@@ -841,8 +839,6 @@ def plan[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P], ws: Works
brief, usage = (None, Usage())
if brief_enabled(spec):
brief, usage = expand_brief(spec, model_id, model=model, events=events)
- if guard is not None and (usage.cost_usd or usage.input_tokens or usage.output_tokens):
- guard.add(usage, stage="plan")
budget = plan_budget(spec, brief)
unit = "passes" if spec.track is Track.GRAPHICS else "parts"
system = build_system_prompt(spec, plan_model, budget=budget)
@@ -880,9 +876,6 @@ def plan[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P], ws: Works
events.emit("plan.truncated", attempt=attempt, max_output_tokens=tokens, thinking=thinking)
continue
usage = usage + resp.usage
- if guard is not None:
- # booked where it is paid: a later attempt that raises cannot erase this dollar
- guard.add(resp.usage, stage="plan")
try:
raw = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
except ValueError: # JsonParseError: nothing parsed — the re-ask below says so
diff --git a/harness/codeverse3d/tracks/scene.py b/harness/codeverse3d/tracks/scene.py
index d8cd93de..2a0f187e 100644
--- a/harness/codeverse3d/tracks/scene.py
+++ b/harness/codeverse3d/tracks/scene.py
@@ -255,8 +255,7 @@ class SceneTrack(BaseTrack):
notes: list[str] = ctx.extra.setdefault("degraded", [])
if note not in notes:
notes.append(note)
- ctx.events.emit("budget.degraded", note=note, spent_usd=round(ctx.budget.spent.cost_usd, 4),
- elapsed_min=round(ctx.budget.elapsed_minutes(), 2))
+ ctx.events.emit("budget.degraded", note=note, elapsed_min=round(ctx.budget.elapsed_minutes(), 2))
# ---- the stage session ------------------------------------------------
def _stage_session(self, ctx: RunContext, stage: str, kind: str,
@@ -299,9 +298,6 @@ class SceneTrack(BaseTrack):
ctx.events.emit("textures.stage_failed", error=f"{type(e).__name__}: {e}"[:300])
return {}
manifest = pack.manifest()
- # post-hoc, like the object texture pass (lifecycle): the images are on disk either
- # way, and the round loop's next ok() check is where a crossed ceiling stops the run
- ctx.budget.add(pack.usage, stage="texture")
ctx.ws.commit("textures")
ctx.events.emit("textures.done", n=len(manifest), cost_usd=round(pack.usage.cost_usd, 4), source=pack.source)
return manifest
diff --git a/harness/codeverse3d/tracks/scene_assets.py b/harness/codeverse3d/tracks/scene_assets.py
index c0ac0a55..ad7cedfe 100644
--- a/harness/codeverse3d/tracks/scene_assets.py
+++ b/harness/codeverse3d/tracks/scene_assets.py
@@ -630,10 +630,6 @@ def _judge_and_fix(ctx: RunContext, asset: AssetPlan, result: AssetResult, rende
except Exception as e: # noqa: BLE001
events.emit("asset.judge_failed", asset=asset.name, error=f"{type(e).__name__}: {e}")
return None
- # book the money non-enforcing like every other judge site (steps.py, candidates.py):
- # the verdict is already paid for, and raising here would discard it. The stage
- # boundary enforces the ceiling.
- ctx.budget.add(verdict.usage, stage="judge")
write_json_atomic(d / "judge.json", verdict.model_dump(mode="json")) # replayable, like a round's judge/rNN.json
if is_degraded(verdict):
# a degraded verdict is no verdict: score stays None and `judged` False, and the
diff --git a/harness/codeverse3d/tracks/steps.py b/harness/codeverse3d/tracks/steps.py
index 1dbaf6f1..c85f88a7 100644
--- a/harness/codeverse3d/tracks/steps.py
+++ b/harness/codeverse3d/tracks/steps.py
@@ -13,10 +13,10 @@ Two cost rules live here (docs/COST.md §5, §6):
:func:`rejudge_round` re-buys one only after the judge was TRIED and failed, so
a skip is a saving and not a deferral. Two wave-2 branches were removed for
failing exactly that test (docs/COST.md §17).
-* **every round reports what it burned** — :func:`emit_round_cost` writes one
- ``cost.round`` event ({stage → $}, judge $, agent turns, wasted flag) at the
- end of the round *and* when the round raises half-way, so the ledger never has
- to reconstruct a round the budget cut.
+* **a round's cost is its ledger rows** — the round runs inside a ``cost.tally``, so
+ ``RoundRecord.usage`` is exactly what ``telemetry/cost.jsonl`` booked while it ran, and a
+ round that raises half-way still reports it (``rounds/aborted_rNN.json``). Each step it
+ runs is timed into ``RoundRecord.steps`` (docs/COST.md §31).
"""
from __future__ import annotations
@@ -38,10 +38,10 @@ from codeverse3d.contracts.artifacts import (
from codeverse3d.contracts.common import Usage
from codeverse3d.contracts.plan import AcceptanceItem, Plan
from codeverse3d.contracts.run import RoundRecord, StepTime
-from codeverse3d.cost.tally import timed
+from codeverse3d.cost.tally import Tally, tally, timed
from codeverse3d.judges.base import round_input
from codeverse3d.judges.rubrics import is_degraded
-from codeverse3d.orchestrator import BudgetExceeded, usage_delta
+from codeverse3d.orchestrator import BudgetExceeded
from codeverse3d.proc import fan_out
from codeverse3d.record.deliverable import keep_round_artifacts
from codeverse3d.spatial.render import RenderError
@@ -206,18 +206,18 @@ def run_round(
``render`` (replaces ``pipeline.render``) and ``geometry_views=False`` (skips the
clay views): the two knobs a best-of-N candidate turns (``candidates.run_best_of_n``
calls it directly for each candidate in its sub-workspace)."""
- mark = ctx.budget.mark()
steps: list[StepTime] = []
- try:
- return _run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,
- previous=previous, files_hint=files_hint, extra_usage=extra_usage,
- extra_notes=extra_notes, steps=steps)
- except BaseException as e:
- # the round died half-way (budget stop, 503 storm, RoundFailed). Whatever it
- # burned is already in the guard: report it so the round is not invisible.
- burned = usage_delta(ctx.budget.spent, mark)
- record_aborted_round(ctx, index=index, kind=kind, usage=burned, error=f"{type(e).__name__}: {e}", steps=steps)
- raise
+ with tally() as spent:
+ try:
+ return _run_round(ctx, index=index, kind=kind, tasks=tasks, pipeline=pipeline, instructions=instructions,
+ previous=previous, files_hint=files_hint, extra_usage=extra_usage,
+ extra_notes=extra_notes, steps=steps, spent=spent)
+ except BaseException as e:
+ # the round died half-way (budget stop, 503 storm, RoundFailed): what it burned
+ # is in its tally — the ledger rows booked while it ran — so it is not invisible
+ record_aborted_round(ctx, index=index, kind=kind, usage=spent.usage,
+ error=f"{type(e).__name__}: {e}", steps=steps)
+ raise
def _run_round(
@@ -235,17 +235,16 @@ def _run_round(
render: RenderFn | None = None,
geometry_views: bool = True,
steps: list[StepTime] | None = None,
+ spent: Tally,
) -> RoundRecord:
+ """The round's body. ``spent`` is the tally the caller opened around it: the round's
+ cost is what it booked (+ ``extra_usage``, work paid before the round began)."""
t0 = time.time()
steps = [] if steps is None else steps
ctx.events.emit("round.start", round=index, kind=kind, n_tasks=len(tasks))
rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, instructions=list(instructions),
started_at=datetime.now(UTC))
- usage = extra_usage or Usage()
notes: list[str] = list(extra_notes)
- cost: dict[str, float] = {}
- if extra_usage is not None and extra_usage.cost_usd:
- cost["candidates"] = round(extra_usage.cost_usd, 6)
# a candidate IS a baseline written in a sub-workspace: route its skills as baseline work,
# or every kind-gated bundle (the *-forms / *-joints routes) drops out and the winner is
@@ -253,22 +252,12 @@ def _run_round(
skill_kind = "baseline" if kind == "candidate" else kind
skills_hook.attach_for_round(ctx, index=index, kind=skill_kind)
gens = run_generation_tasks(ctx, skills_hook.with_inlined_skill(ctx, tasks), steps=steps)
- turns = 0
- for g in gens:
- usage = usage + g.usage
- turns += g.turns
- if not g.ok:
- notes.append(f"{g.label}: {g.notes}")
- if gens:
- cost["generate"] = round(sum(g.usage.cost_usd for g in gens), 6)
+ notes += [f"{g.label}: {g.notes}" for g in gens if not g.ok]
ctx.ws.commit(f"r{index:02d} {kind}: generated")
with timed("build", steps, round_index=index):
outcome: RepairOutcome = build_with_repair(ctx, round_index=index, label=f"r{index:02d}_{kind}",
files_hint=list(files_hint))
- usage = usage + outcome.usage
- if outcome.usage.cost_usd:
- cost["repair"] = round(outcome.usage.cost_usd, 6)
rec.build = outcome.build
gates: list[GateReport] = [outcome.lint]
if outcome.attempts:
@@ -310,19 +299,8 @@ def _run_round(
ctx.events.emit("judge.skipped", round=index, reason=skip)
else:
with timed("judge", steps, round_index=index):
- rec.judgment, judge_usage = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous, notes,
- geometry_views=geometry_views)
- if rec.judgment is not None or judge_usage.cost_usd:
- # a degraded/crashed verdict was still PAID: fold its usage into the
- # round's usage and cost["judge"] so rec.usage and the cost.round event
- # see the money even when judged=False.
- usage = usage + judge_usage
- cost["judge"] = round(judge_usage.cost_usd, 6)
- if rec.judgment is not None:
- # add, never charge: the verdict exists and is paid for — raising here
- # would drop a fully judged round before it is committed/recorded
- # (the loop stops at its next budget check instead).
- ctx.budget.add(rec.judgment.usage, stage="judge")
+ rec.judgment = _judge(ctx, pipeline, index, outcome.build, gates, rec, previous, notes,
+ geometry_views=geometry_views)
else:
# a failed build's own reports say why (a shader error lives in shader_preflight)
gates.extend(outcome.build.gates)
@@ -335,15 +313,13 @@ def _run_round(
if outcome.build.ok:
# every round keeps what a hand-over of it needs: the next build replaces artifacts/
keep_round_artifacts(ctx.ws, index)
- rec.usage = usage
+ rec.usage = spent.usage + (extra_usage or Usage())
rec.duration_s = round(time.time() - t0, 2)
rec.steps = steps
rec.notes = "; ".join(notes)
ctx.ws.write_json(round_record_path(ctx, index), rec)
ctx.events.emit("round.done", round=index, kind=kind, commit=rec.commit[:10], score=rec.score,
- build_ok=outcome.build.ok, duration_s=rec.duration_s, cost_usd=round(usage.cost_usd, 4))
- emit_round_cost(ctx, index=index, kind=kind, cost=cost, usage=usage, turns=turns, score=rec.score,
- build_ok=outcome.build.ok, judged=rec.judgment is not None)
+ build_ok=outcome.build.ok, duration_s=rec.duration_s, cost_usd=round(rec.usage.cost_usd, 4))
return rec
@@ -369,62 +345,16 @@ def skip_judge_reason(ctx: RunContext, *, renders: RenderSet | None) -> str:
return ""
-def emit_round_cost(
- ctx: RunContext,
- *,
- index: int,
- kind: str,
- cost: dict[str, float],
- usage: Usage,
- turns: int,
- score: float | None,
- build_ok: bool,
- judged: bool,
- aborted: str = "",
- corrected: bool = False,
-) -> None:
- """One ``cost.round`` event per round: ``{stage → $}``, judge $, agent turns and
- whether the money bought a usable round. The ledger/audit reads this instead of
- reconstructing the round from trajectories and events. ``corrected=True``
- marks a re-emission after ``rejudge_round`` recovered a verdict — the LAST
- event per round is the truth."""
- wasted, why = _waste_flag(build_ok=build_ok, judged=judged and score is not None, aborted=aborted)
- ctx.events.emit(
- "cost.round", round=index, kind=kind,
- stages={k: round(v, 6) for k, v in cost.items()},
- judge_usd=round(cost.get("judge", 0.0), 6),
- total_usd=round(usage.cost_usd, 6),
- agent_turns=turns,
- input_tokens=usage.input_tokens, output_tokens=usage.output_tokens, cached_tokens=usage.cached_tokens,
- score=score, wasted=wasted, waste_reason=why, aborted=aborted,
- corrected=corrected,
- run_usd=round(ctx.budget.spent.cost_usd, 6),
- )
-
-
-def _waste_flag(*, build_ok: bool, judged: bool, aborted: str) -> tuple[bool, str]:
- """Did this round buy a usable, judged round? (docs/COST.md §5 vocabulary.) A round that
- scored below an earlier one is NOT waste any more (2026-09-22): every round is kept, and
- any of them can be the one handed over (``codeverse3d.addons.select``)."""
- if aborted:
- return True, "aborted"
- if not build_ok:
- return True, "build_failed"
- if not judged:
- return True, "unjudged"
- return False, ""
-
-
def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage, error: str,
steps: Sequence[StepTime] = ()) -> None:
"""A round that raised half-way still reports what it burned.
Its record is written next to the round records as ``aborted_rNN.json`` (never
``rNN.json``: the round did not happen, and ``load_round_journal`` must not
- resume from it), a ``cost.round`` event is emitted, the usage is remembered
- in ``ctx.extra`` so the run record can carry it, and the steps it did run join the
- run-level steps (``RunState.steps``): their minutes were spent. Accounting a stop
- must never raise on top of the stop that is already happening."""
+ resume from it), its cost is remembered in ``ctx.extra`` so the run record can carry
+ it, and the steps it did run join the run-level steps (``RunState.steps``): their
+ minutes were spent. Accounting a stop must never raise on top of the stop that is
+ already happening."""
try:
ctx.state.steps.extend(steps)
rec = RoundRecord(index=index, kind=kind, agent_backend=ctx.agent_id, usage=usage, steps=list(steps),
@@ -434,23 +364,18 @@ def record_aborted_round(ctx: RunContext, *, index: int, kind: str, usage: Usage
ctx.ws.write_json(path, rec)
aborted = ctx.extra.setdefault("aborted_rounds", [])
aborted.append({"index": index, "kind": kind, "cost_usd": round(usage.cost_usd, 6), "error": error[:300]})
- emit_round_cost(ctx, index=index, kind=kind, cost={"aborted": round(usage.cost_usd, 6)}, usage=usage,
- turns=0, score=None, build_ok=False, judged=False, aborted=error[:300])
except Exception as e: # noqa: BLE001 — never mask the exception that stopped the round
log.warning("could not record aborted round r%02d: %s", index, e)
def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildResult, gates: list[GateReport],
rec: RoundRecord, previous: Judgment | None, notes: list[str], *,
- geometry_views: bool = True) -> tuple[Judgment | None, Usage]:
- """Judge one round → ``(judgment, paid_usage)``.
-
- ``judgment`` is ``None`` when the verdict is unusable (the judge crashed or
- returned a degraded non-score); ``paid_usage`` is what the attempt cost
- EITHER WAY, so a failed verdict's dollars still reach the round record and
- its ``cost.round`` event. Failure notes are appended to the CALLER's
- ``notes`` list — ``_run_round`` joins that list into ``rec.notes`` at the
- end of the round, so writing to ``rec.notes`` here was silently lost."""
+ geometry_views: bool = True) -> Judgment | None:
+ """Judge one round; ``None`` when the verdict is unusable (the judge crashed or returned
+ a degraded non-score). What the attempt cost reaches the round either way: its calls
+ are ledger rows, booked into the round's tally. Failure notes are appended to the
+ CALLER's ``notes`` list — ``_run_round`` joins that list into ``rec.notes`` at the end
+ of the round, so writing to ``rec.notes`` here was silently lost."""
renders = rec.renders
jv = getattr(pipeline, "judge_views", None)
if callable(jv) and renders is not None:
@@ -476,27 +401,22 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildRes
except Exception as e: # noqa: BLE001 — a judge outage must not destroy the run's code/commits
log.exception("judge failed in round %d", index)
paid = getattr(e, "usage", None)
- paid = paid if isinstance(paid, Usage) else Usage()
- if paid.cost_usd:
- # the provider billed the failed attempt (ModelError.usage): book it
- ctx.budget.add(paid, stage="judge")
ctx.events.emit("judge.failed", round=index, error=f"{type(e).__name__}: {e}",
- cost_usd=round(paid.cost_usd, 4))
+ cost_usd=round(paid.cost_usd, 4) if isinstance(paid, Usage) else 0.0)
notes.append(f"judge failed: {type(e).__name__}: {e}")
- return None, paid
+ return None
ctx.ws.write_json(ctx.ws.judge_path(index), judgment)
if is_degraded(judgment):
- # a glitch, never a score: keep the raw verdict on disk, pay for it, but do not
- # let 0.0 poison a pick or a refine plan (judges/rubrics.degraded_judgment contract)
- ctx.budget.add(judgment.usage, stage="judge")
+ # a glitch, never a score: keep the raw verdict on disk, but do not let 0.0
+ # poison a pick or a refine plan (judges/rubrics.degraded_judgment contract)
ctx.events.emit("judge.degraded", round=index, error=judgment.summary[:300],
cost_usd=round(judgment.usage.cost_usd, 4))
notes.append(f"judge degraded: {judgment.summary[:200]}")
- return None, judgment.usage
+ return None
ctx.events.emit("judge.done", round=index, overall=round(judgment.overall, 3), passed=judgment.passed,
n_issues=len(judgment.issues), n_plan=len(judgment.improvement_plan),
duration_s=round(time.time() - t0, 1), cost_usd=round(judgment.usage.cost_usd, 4))
- return judgment, judgment.usage
+ return judgment
def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, previous: Judgment | None = None) -> bool:
@@ -510,25 +430,15 @@ def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, pr
ctx.events.emit("judge.retry", round=rec.index)
notes: list[str] = [rec.notes] if rec.notes else []
# a round recorded before step timing keeps counting its own clock (RoundRecord.minutes)
- with timed("judge", rec.steps if rec.steps is not None else [], round_index=rec.index):
- judgment, judge_usage = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)
- if judgment is None:
- # the retry failed AGAIN — but it was still paid for: keep the money and the
- # failure note on the persisted record so totals and resume can see them.
- rec.usage = rec.usage + judge_usage
- rec.notes = "; ".join(notes)
- ctx.ws.write_json(round_record_path(ctx, rec.index), rec)
- return False
- rec.judgment = judgment
- rec.usage = rec.usage + judgment.usage
+ with timed("judge", rec.steps if rec.steps is not None else [], round_index=rec.index) as spent:
+ judgment = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)
+ # paid either way: a retry that failed AGAIN keeps its money and its note on the record
+ rec.usage = rec.usage + spent.usage
+ if judgment is not None:
+ rec.judgment = judgment
rec.notes = "; ".join(notes)
- ctx.budget.add(judgment.usage, stage="judge")
ctx.ws.write_json(round_record_path(ctx, rec.index), rec)
- # the round's original cost.round went out as wasted=unjudged; emit the corrected
- # one so the audit stream stops counting a now-scored round as wasted.
- emit_round_cost(ctx, index=rec.index, kind=rec.kind, cost={"judge": round(judgment.usage.cost_usd, 6)},
- usage=rec.usage, turns=0, score=rec.score, build_ok=True, judged=True, corrected=True)
- return True
+ return judgment is not None
def _write_gate_reports(ctx: RunContext, index: int, gates: list[GateReport]) -> None:
@@ -545,7 +455,3 @@ def failed_acceptance(ctx: RunContext, judgment: Judgment | None) -> list[Accept
return []
items = {a.id: a for a in (getattr(ctx.plan, "acceptance", []) or [])}
return [items[k] for k, ok in judgment.acceptance_results.items() if not ok and k in items]
-
-
-def sum_usage(rounds: Sequence[RoundRecord]) -> Usage:
- return sum((r.usage for r in rounds), Usage())
diff --git a/harness/codeverse3d/tracks/zone_layout.py b/harness/codeverse3d/tracks/zone_layout.py
index 9cef1d39..949206ec 100644
--- a/harness/codeverse3d/tracks/zone_layout.py
+++ b/harness/codeverse3d/tracks/zone_layout.py
@@ -167,8 +167,6 @@ def _one_layout(zone: ZonePlan, plan: ScenePlan, model: Any, budget: Any, events
response_schema=ZoneLayout.model_json_schema(), temperature=0.3,
thinking="low", max_output_tokens=LAYOUT_MAX_TOKENS,
max_wait_s=max_wait_s, label="zone-layout"))
- if budget is not None:
- budget.add(resp.usage, stage="plan")
raw = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text or "{}")
layout = ZoneLayout.model_validate(raw)
layout.zone = zone.name # the name is an input, never model-invented
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index f39a5c5f..fefb4095 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -59,8 +59,9 @@ round and runs the optional **texture pass** on it. The record (plus git histor
7. **Cheap first.** Lint → build → deterministic gates → montaged views → VLM.
8. **Separate generator from judge.** The judge sees spec + renders +
measurements + acceptance list; never the generator's reasoning.
-9. **Cost is a budget, not a log.** Hard per-run ceilings (`BudgetGuard`), key
- pools with per-key limiters, every call yields a `Usage`.
+9. **Money is measured, time is bounded.** The wall clock is the hard per-run ceiling
+ (`BudgetGuard`), key pools have per-key limiters, and every call is one row of the
+ run's ledger, `telemetry/cost.jsonl` — the only record of money.
10. **Reproducible.** Prompt/rubric/cookbook hashes and tool versions are
recorded on every run (`record.prompt_hashes`, `record.environment`).
11. **Export as authored.** No wrapper re-centres, rescales or grounds the
@@ -161,8 +162,6 @@ codeverse3d/
MeteredAgent — one row per ChatModel.generate; one session row only for a backend
that does NOT meter itself; run_ledger nests + is context-local so bench --parallel works)
profiles.py (economy|balanced|quality; cli._common.resolve_dial is THE resolver)
- billing.py (SUBSCRIPTION_BACKENDS/bills_usd — which backends take real dollars,
- so the ledger bills real money and not list price; docs/COST.md §25)
tally.py (a block's money = its ledger rows, its lost seconds → StepTime; COST §31)
guard.py
judges/ base.py (JudgeInput; `round_input` + `plan_summary(plan, language)`, the ONE payload
@@ -184,7 +183,7 @@ codeverse3d/
docstring-only — import from the submodules
orchestrator.py the round loop's LIBRARY, not the loop: the round knobs (RoundPolicy —
no stop knob besides max_rounds), refine-task compilation + grouping,
- StageRunner + RunState (resume), BudgetGuard. The loop itself is
+ StageRunner + RunState (resume), BudgetGuard (the clock). The loop itself is
tracks/lifecycle.py:_round_loop → tracks/steps.py:run_round. A best-of-N
candidate IS steps._run_round(kind='candidate') in a _cand/c<k> sub-workspace
with two knobs (render=candidates.quick_render, geometry_views=False), its
@@ -308,8 +307,9 @@ runs/<slug>/
telemetry/cost.jsonl THE ledger: one row per metered call / CLI session, opened by BaseTrack.run — the
only record of money (runs before 2026-09-22 also carry a root cost_ledger.jsonl
and a telemetry/usage.jsonl symlink to it; nothing reads them)
- run_state.json status + stages; extra carries budget_snapshot and spec_fingerprint only — the round
- history is rounds/rNN.json + each round's commit alone (no copy since 2026-09-22)
+ run_state.json status + stages + the run-level step log (steps); extra carries budget_snapshot (the clock's
+ active seconds) and spec_fingerprint only — the round history is rounds/rNN.json + each
+ round's commit alone (no copy since 2026-09-22)
artifacts/ object.glb object.stl|step robot.urdf meshes/ build.json census.json
measurement.json … — the LAST round's build; graphics: frames/fNN_tT.png frames_sheet.png
preview.gif metrics.json; texturing (a pick's --texture): object_textured.glb
@@ -487,8 +487,8 @@ plateau, regression or diminishing-returns stop, no rewrite or surface-detail ro
bought at all: no judge or no renders — a round that finishes past the clock is still judged)
→ commit src/ (the round's commit) and copy its build to artifacts/rNN/
a round left without a verdict (judge outage / degraded) is re-judged once before the next is planned
- every round emits cost.round {stage → $, judge $, agent turns, wasted flag}; a round that raises mid-way
- still reports what it burned (rounds/aborted_rNN.json + record.extra["aborted_rounds"])
+ a round's cost is the ledger rows it booked (a cost.tally) and its steps are timed (RoundRecord.steps); a round
+ that raises mid-way still reports what it burned (rounds/aborted_rNN.json + record.extra["aborted_rounds"])
a round whose every task failed raises RoundFailed, TYPED (2026-09-22 — the loop never reads its message):
.quota (a backend saw the vendor's usage limit) → agent_quota; .transient (a session died of a provider
failure — AgentResult.transient, or a ModelError outage) → the SAME round re-runs once, baseline included;
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index e532c6d4..1231fb84 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -114,7 +114,7 @@ record_call(usage, *, run="", round=None, stage=None, role=None, label="", backe
# Δ everything left out is resolved from the ambient context + the call label (cost.context);
# unknown model -> $0 and price_source="unknown" (flagged, never silently dropped); never raises.
from codeverse3d.cost.instrument import (run_ledger, metered_chat_model, metered_agent,
- MeteredAgent, MeteredChatModel, metering_enabled)
+ MeteredAgent, MeteredChatModel) # always on: no off switch
with run_ledger(ws.root, run=slug): # binds the run, points record_call at <run>/telemetry/cost.jsonl
... # (its one name: no root alias since 2026-09-22)
# NESTS by holding ContextVar tokens (bound_run / bound_ledger) — context-local ONLY, no
@@ -126,7 +126,7 @@ with run_ledger(ws.root, run=slug): # binds the run, points record_call a
# agents.get_coding_agent returns a MeteredAgent (registry wrap, from the first call) → ambient
# round/stage for the session and one source='session' row (n_calls, latency, outcome,
# AgentResult.turns) — every backend is a vendor CLI whose calls the harness cannot see.
- # These two are the ONLY ledger writers; BudgetGuard writes no row.
+ # These two (+ the image model's own row) are the ONLY ledger writers; BudgetGuard keeps no money.
# The rule is the BACKEND, never "did a row get written while it ran": a CLI session with one
# in-process tool call used to be dropped entirely.
from codeverse3d.cost.context import CallContext, call_context, bound_run, context_from_label, SELF_DESCRIBING
@@ -454,17 +454,18 @@ RoundPolicy(max_rounds=4, max_refine_tasks=6, max_instructions_per_task=6, paral
# StopDecision, pick_best_round, judge_sigma, best_score, last_gain, REWRITE_KIND / DETAIL_KIND /
# KIND_FOR_STRATEGY / detail_blocked, RunState.best_* / update_best, and the RoundPolicy fields plateau_window,
# min_delta, target, pairwise_*, judge_model, regression_sigma, marginal_*, detail_*.
-from codeverse3d.orchestrator import BudgetGuard, usage_delta
-BudgetGuard(budget, start_time=None, *, soft_fraction=1.0) # no run= / ledger=
- .charge(usage, *, stage="other", enforce=True) # (Δ 2026-08-30: role/label/round_index/outcome gone)
- # THE door every dollar goes through: accumulate → bucket by stage → enforce the ceilings.
- # The per-call record (role, label, round, outcome) is the LEDGER row; the guard bucketing them a
- # second time fed nothing but itself.
- # It writes NO ledger row: cost.instrument.MeteredAgent (one source='session' row per CLI session) and
- # MeteredChatModel (one row per call) are the only writers of telemetry/cost.jsonl.
- # add(...) = charge(enforce=False) — "not enforced" never means "not seen".
- .by_stage / .stage_summary() / .mark() # what a round burned, live (by_round[i] deleted — no reader;
- # per-round money is CallCost.round in telemetry/cost.jsonl)
+from codeverse3d.orchestrator import BudgetGuard, BudgetSnapshot
+BudgetGuard(budget, start_time=None, *, soft_fraction=1.0) # THE CLOCK, nothing else (Δ 2026-09-22)
+ .check() / .ok() / .elapsed_minutes() / .timeout_s(want_s, *, floor_s, soft) / .soft_exceeded()
+ .grant_grace(minutes=) / .snapshot() -> BudgetSnapshot{active_s} / .restore(snap) / .summary() -> {elapsed_min,
+ max_minutes}
+ # Δ 2026-09-22: charge / add / mark / stage_summary / spent / billed_usd / by_stage and usage_delta are gone —
+ # the ledger (telemetry/cost.jsonl) is the only record of money; a run's total is its sum at list price
+ # (record.package_run / BaseTrack._record), a round's cost the rows booked inside it (cost.tally.tally)
+from codeverse3d.cost.tally import tally, timed, book_usage, book_time # a block's money + lost seconds
+with tally() as t: ... # t.usage = the ledger rows booked inside (nests; fan_out copies it)
+with timed("judge", steps, round_index=i): ... # appends StepTime{step, round, wall_s, lost_s} (docs/COST.md §31)
+RunRecord.minutes / RoundRecord.minutes # Σ (wall − lost) over the steps; no steps → its own clock
from codeverse3d.tracks.candidates import CandidateRecord, rank_candidates # the one in-loop choice left
from codeverse3d.tracks.candidates import run_best_of_n, quick_render # N parallel baselines in <ws>/_cand/c<k>
# each candidate IS steps._run_round(kind="candidate") in its sub-workspace: render=quick_render(ctx, round_index, build,
@@ -544,8 +545,9 @@ skip_judge_reason(ctx, *, renders, ignore_budget=False) -> str # "" = judge i
run_round(ctx, *, index, kind, tasks, pipeline, ..., previous_best=None) -> RoundRecord
# steps._run_round (the same round, no aborted-round record) also takes render: RenderFn | None (swaps the
# pipeline's render; candidates pass quick_render) and geometry_views=False (skips the clay/normals views)
- # emits cost.round {stages{}, judge_usd, total_usd, agent_turns, wasted, waste_reason}; on ANY exception it
- # records what the round burned (rounds/aborted_rNN.json, ctx.extra["aborted_rounds"]) and re-raises
+ # rec.usage = the ledger rows the round booked (its tally) + extra_usage; rec.steps = generate (per phase) /
+ # build / gates / render / judge; on ANY exception it records what the round burned and the steps it ran
+ # (rounds/aborted_rNN.json, ctx.extra["aborted_rounds"], RunState.steps) and re-raises
from codeverse3d.tracks.planner import plan, ensure_acceptance, plan_example, plan_temperature
plan(spec, model_id, plan_model, ws, *, model=None, events=None, budget=None) -> Plan # no runtime= (the contract is the catalog's)
# Δ 2026-09-22: every track, graphics included, through ONE loop that dispatches on spec.track — PLAN_TEMPLATES,
@@ -610,8 +612,8 @@ select.package(run_dir, round_index, *, texture=False, method="round", image_mod
from codeverse3d.addons.gallery.index import hero_view # (ws, rec, picked) -> (rel, label, n_views): the card image
from codeverse3d.record.record import complexity_block, round_complexity # objective complexity of what was built
# finalize_record fills record.extra["complexity"] = the LAST measured round's vector + plan_parts /
- # parts_per_plan_part / by_round; every rounds_summary row gains "complexity" (the index or None); readers
- # that show one round (gallery, dataset) read that round's own round_complexity
+ # parts_per_plan_part / by_round; readers that show one round (gallery, dataset) read that round's own
+ # round_complexity (a dataset sample's rounds_summary rows carry it)
from codeverse3d.addons.dataset.export import export_samples # (runs_dir, out_dir, *, min_score=None, only_passed=False,
# include_unbuilt=False, captions_dir=None, drop_duplicates=False) -> ExportReport{…, n_duplicates, duplicates, tiers}
from codeverse3d.addons.dataset.quality import quality_tier, prompt_hash, find_duplicates # tiers (of the exported round's verdict):
@@ -672,21 +674,20 @@ Event names: `run.start`, `stage.start/done`, `plan.done`, `workspace.materializ
`texture.start/plan/generated/applied/gate/done`, `budget.exceeded`,
`finalise.rebuild`, `stop` (reason, rounds), `run.done` (status, rounds, last_score) / `run.failed`;
after the run: `pick.pairwise`, `pick.packaged`, `texture.skipped` / `texture.failed` (addons.select).
-Cost events: **`cost.round`** (per round: `stages{stage → $}`, `judge_usd`, `total_usd`,
-`agent_turns` (= AgentResult.turns), `score`, `wasted`, `waste_reason` ∈ aborted | build_failed |
-unjudged, `run_usd`) — emitted for aborted rounds too (Δ 2026-09-22: `previous_best` and the
-regression / zero_delta reasons went with the in-run best: a lower-scored round is kept, not wasted);
-`judge.skipped` (reason), `generate.turn_cap` (label, max_turns, turns, cost_usd).
+Cost events: `judge.skipped` (reason), `generate.turn_cap` (label, max_turns, turns, cost_usd);
+`run.done` carries the run's `cost_usd` (the ledger's) and `minutes`. Δ 2026-09-22: `cost.round` is
+gone — no reader, and its per-round {stage → $} restated the ledger (`summarise(rows)` by round and stage).
Δ 2026-09-22: `best.updated`, `strategy.switch`, `round.refine_from_best`, `pairwise.done` and
`budget.overrun` are gone, and so is `RoundRecord.pairwise` (the in-run tie-break note; an old
rNN.json that carries one still loads, the key ignored). `PairwiseNote` lives in `addons.select`.
-`RunRecord` (record.json): spec, plan, workspace, status, rounds[RoundRecord],
-total_usage, environment,
+`RunRecord` (record.json): spec, plan, workspace, status, rounds[RoundRecord{…, usage, duration_s, steps}],
+total_usage, environment, steps (run-level StepTime rows; `minutes` = the property over all of them),
prompt_hashes{contract, cookbook, generate, refine}, error, extra{stop_reason,
-rubric, budget, cost_by_stage, aborted_rounds?, rounds_summary, candidates? (the
-rounds/candidates.json payload), texturing?, captions?}. `total_usage` is the BudgetGuard total whenever it
-exceeds the sum of the rounds (aborted rounds, retried sessions); a texture pass bought by a pick is in
-telemetry/cost.jsonl, not in `total_usage`. Δ 2026-09-22: no `best_round` / `baseline_score` /
+rubric, budget{elapsed_min, max_minutes} (the clock), aborted_rounds?, candidates? (the
+rounds/candidates.json payload), texturing?, captions?}. `total_usage` is the sum of telemetry/cost.jsonl at
+list price — every billed row once, aborted rounds, retried sessions and billed-but-discarded round-trips
+included; a texture pass a pick buys joins it when `select.package` re-packages the run. Δ 2026-09-22: no
+`cost_by_stage` / `rounds_summary` (written, never read), no `best_round` / `baseline_score` /
`final_score` (old records carry them: ignored — `addons.select.summarise` answers), and `status` is only
the stop reason: max_rounds | budget | agent_quota | no_change | no_refine_tasks | judge_unavailable |
failed (an old `passed` / `plateau` loads as `stopped`).
diff --git a/harness/docs/RUN_LAYOUT.md b/harness/docs/RUN_LAYOUT.md
index 21bb0cf9..669024cd 100644
--- a/harness/docs/RUN_LAYOUT.md
+++ b/harness/docs/RUN_LAYOUT.md
@@ -135,13 +135,12 @@ produces no diff.
provenance, stage, role, outcome), written by the metered models and agents while the
run happens. It is the only record of money; a run without one (recorded before
2026-08-23) has no rows — nothing is reconstructed.
-* **`cost.json`** (`CostSummary`) — the run-layout view of those rows: total vs
- budget, wall clock vs `max_minutes`, tokens, per stage (`plan / assets / env /
- zones / baseline / refine / repair / judge / pairwise / texture / …`), per
- role, per model, per round, plus `ledger_usd`, `unattributed_usd` (the
- ledger's residual row) and `post_run_usd` (priced calls outside the run total,
- e.g. the texture pass or pairwise verdict a pick bought after the run).
- `record.total_usage` stays the authority on what the run cost.
+* **`cost.json`** (`CostSummary`) — the run-layout view of those rows, computed by the
+ ledger's own `summarise`: the total (= `record.total_usage`, the ledger's sum at list
+ price — post-run work a pick buys joins it the next time the run is packaged), tokens, per
+ stage (`plan / assets / env / zones / baseline / refine / repair / judge / pairwise /
+ texture / …`), per role, per model, per round, and the run's minutes against
+ `max_minutes`.
## `record.json`
diff --git a/harness/tests/cost/test_instrument.py b/harness/tests/cost/test_instrument.py
index b677888c..0fcc2ee8 100644
--- a/harness/tests/cost/test_instrument.py
+++ b/harness/tests/cost/test_instrument.py
@@ -253,9 +253,7 @@ def test_parallel_runs_in_their_own_threads_keep_their_own_ledgers(tmp_path: Pat
# and the default ledger were ALSO process globals that the first thread to exit
# republished, so this plain main-thread call appended to a finished run's file
# under that run's name instead of going to the per-process log.
- monkeypatch.setenv("C3D_COST_LEDGER", str(tmp_path / "process.jsonl"))
- monkeypatch.setattr(ledger_mod, "_fallback", None)
- monkeypatch.setattr(ledger_mod, "_fallback_read", False)
+ monkeypatch.setattr(ledger_mod, "_fallback", ledger_mod.CostLedger(tmp_path / "process.jsonl"))
assert run_binding().run == ""
record_call(Usage(cost_usd=0.5), label="baseline")
assert [r.run for r in load_ledger(tmp_path / "process.jsonl")] == [""]
diff --git a/harness/tests/flywheel_cli/test_complexity_record.py b/harness/tests/flywheel_cli/test_complexity_record.py
index 2a7f1fbd..7cceac23 100644
--- a/harness/tests/flywheel_cli/test_complexity_record.py
+++ b/harness/tests/flywheel_cli/test_complexity_record.py
@@ -62,7 +62,7 @@ def test_record_carries_the_last_rounds_complexity_and_the_trail() -> None:
assert block["plan_parts"] == 4
assert block["parts_per_plan_part"] == pytest.approx(2.0)
assert block["by_round"] == [0.42, 0.55]
- assert [r["complexity"] for r in rec.extra["rounds_summary"]] == [0.42, 0.55]
+ assert [round_summary(r)["complexity"] for r in rec.rounds] == [0.42, 0.55]
def test_complexity_block_falls_back_to_the_last_measured_round() -> None:
diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.py
index d4cdf79f..175a461c 100644
--- a/harness/tests/flywheel_cli/test_flywheel.py
+++ b/harness/tests/flywheel_cli/test_flywheel.py
@@ -42,8 +42,8 @@ def test_finalize_and_load_record(fake_run):
assert not {"best_round", "baseline_score", "final_score"} & stored.keys() # the run names no best (2026-09-22)
assert loaded.environment["python"] and loaded.environment["codeverse3d"]
assert "three" in loaded.environment and "node" in loaded.environment
- assert len(loaded.extra["rounds_summary"]) == 2
- assert loaded.extra["rounds_summary"][1]["score"] == 0.80
+ assert "rounds_summary" not in loaded.extra, "written, never read: gone 2026-09-22"
+ assert loaded.total_usage.cost_usd == pytest.approx(0.06), "the total is the ledger's"
def test_iter_runs_and_errors(runs_dir: Path):
diff --git a/harness/tests/flywheel_cli/test_run_layout_cli.py b/harness/tests/flywheel_cli/test_run_layout_cli.py
index 2d456603..a0ae7972 100644
--- a/harness/tests/flywheel_cli/test_run_layout_cli.py
+++ b/harness/tests/flywheel_cli/test_run_layout_cli.py
@@ -7,6 +7,7 @@ import json
import shutil
from pathlib import Path
+import pytest
from typer.testing import CliRunner
from codeverse3d.addons import select
@@ -96,9 +97,9 @@ def test_package_run_builds_a_stable_independent_and_loadable_handover(fake_run)
assert first.generated_at == second.generated_at # unchanged content → no diff
assert [(f.path, f.sha256) for f in first.files] == [(f.path, f.sha256) for f in second.files]
- # Package metadata can be recovered from files even before the caller saves rec.
+ # A pick re-packages the record: its telemetry and its total (the ledger's) are on disk.
bare = load_record(ws)
- assert bare.telemetry is None
+ assert bare.telemetry is not None and bare.total_usage.cost_usd == pytest.approx(0.06)
assert load_telemetry(ws, bare) is not None and load_deliverable(ws) is not None
# Delivered assets are copies: editing a hand-over cannot mutate evidence.
@@ -180,7 +181,7 @@ def test_show_sections_and_status_share_the_packaged_run(fake_run):
assert out.index("DELIVERABLE") < out.index("QUALITY EVIDENCE") < out.index("COST & SETTINGS")
assert "deliverable/object.glb" in out and "deliverable/src/model.py" in out
assert "static_object_v1" in out # evidence: rubric
- assert "spent / budget" in out and "models per role" in out and "price table" in out
+ assert "the ledger, list price" in out and "models per role" in out and "price table" in out
assert "gemini-cli" in out and "generator" in out # settings: model id per role
# A selected section stays isolated, and invalid selectors fail cleanly.
diff --git a/harness/tests/orchestrator_tracks/fakes.py b/harness/tests/orchestrator_tracks/fakes.py
index dcaec5f9..4f557d01 100644
--- a/harness/tests/orchestrator_tracks/fakes.py
+++ b/harness/tests/orchestrator_tracks/fakes.py
@@ -234,7 +234,17 @@ class FakeJudge:
return Judgment(rubric="fake_v1", judge_backend="fake", scores={"geometry": s, "material": s}, overall=s, passed=s >= 0.8,
summary=f"round {inp.round_index} score {s}", issues=[JudgeIssue(target=self.targets[0], kind="geometry", severity="major",
detail="too thin", evidence="front")],
- improvement_plan=plan, acceptance_results=acc, usage=Usage(backend="fake", cost_usd=self.cost))
+ improvement_plan=plan, acceptance_results=acc, usage=bill_verdict(self.cost, inp.round_index))
+
+
+def bill_verdict(cost: float, round_index: int) -> Usage:
+ """What a real judge's metered model does per verdict: one ledger row (the ledger is
+ the only record of money), filed as the round's judge call."""
+ from codeverse3d.cost.ledger import record_call
+
+ usage = Usage(backend="fake", cost_usd=cost)
+ record_call(usage, label=f"judge:fake_v1:r{round_index:02d}:s0")
+ return usage
# ----------------------------------------------------------------------------- services
diff --git a/harness/tests/orchestrator_tracks/test_budget_snapshot.py b/harness/tests/orchestrator_tracks/test_budget_snapshot.py
index f51586fc..e4ab6188 100644
--- a/harness/tests/orchestrator_tracks/test_budget_snapshot.py
+++ b/harness/tests/orchestrator_tracks/test_budget_snapshot.py
@@ -1,4 +1,5 @@
-"""Budget continuity across resume and charge-before-persist ordering."""
+"""The budget clock across resume, and paid work that survives a clock stop (its money is
+on the ledger, the one record of it)."""
from __future__ import annotations
@@ -10,6 +11,8 @@ import pytest
from codeverse3d.contracts.common import Budget, Language, Usage
from codeverse3d.contracts.plan import StaticPlan
from codeverse3d.contracts.run import RunStatus
+from codeverse3d.cost.instrument import metered_chat_model, run_ledger
+from codeverse3d.cost.ledger import load_ledger
from codeverse3d.orchestrator import BudgetExceeded, BudgetGuard, BudgetSnapshot, RunState
from codeverse3d.proc import EventLog
from codeverse3d.tracks.generation import GenerationTask, generate_files
@@ -28,10 +31,8 @@ from tests.orchestrator_tracks.fakes import (
# --------------------------------------------------------------- snapshot / restore
-def test_resume_restores_money_calls_and_active_time_but_not_downtime():
+def test_resume_restores_active_time_but_not_downtime():
g1 = BudgetGuard(Budget(max_minutes=10.0))
- g1.charge(Usage(backend="gemini", cost_usd=0.6), stage="baseline")
- g1.add(Usage(backend="gemini", cost_usd=0.05), stage="judge")
g1.start_time -= 120 # two minutes of ACTIVE work in the first session
snap = g1.snapshot()
assert snap.active_s == pytest.approx(120, abs=2)
@@ -40,14 +41,10 @@ def test_resume_restores_money_calls_and_active_time_but_not_downtime():
revived = BudgetSnapshot.model_validate(json.loads(json.dumps(snap.model_dump(mode="json"))))
g2 = BudgetGuard(Budget(max_minutes=10.0))
g2.restore(revived)
- assert g2.billed_usd == pytest.approx(0.65) and g2.calls == 2
- assert g2.by_stage == {"baseline": pytest.approx(0.6), "judge": pytest.approx(0.05)}
# ACTIVE minutes carry over; the downtime between the sessions cost nothing
assert g2.elapsed_minutes() == pytest.approx(2.0, abs=0.1)
g2.start_time -= 60 # one more ACTIVE minute in THIS session accumulates on top
assert g2.elapsed_minutes() == pytest.approx(3.0, abs=0.1)
- g2.charge(Usage(backend="gemini", cost_usd=0.5), stage="refine")
- assert g2.billed_usd == pytest.approx(1.15) and g2.calls == 3
# prior ACTIVE time still constrains the wall clock immediately after a resume
g3 = BudgetGuard(Budget(max_minutes=1.0))
@@ -60,7 +57,6 @@ def test_a_raised_cap_grants_only_the_difference_and_grace_never_persists():
import time as _t
g1 = BudgetGuard(Budget(max_minutes=10.0), start_time=_t.time())
- g1.charge(Usage(backend="gemini", cost_usd=0.9), stage="baseline")
g1._active_s = 8.0 * 60 # noqa: SLF001 — 8 of its 10 min
g1.grant_grace(minutes=30.0) # per-attempt salvage headroom — must NOT survive
snap = g1.snapshot()
@@ -69,7 +65,6 @@ def test_a_raised_cap_grants_only_the_difference_and_grace_never_persists():
g2 = BudgetGuard(Budget(max_minutes=15.0)) # --max-minutes raised 10 → 15
g2.restore(snap)
assert g2.grace_minutes == 0.0
- assert g2.billed_usd == pytest.approx(0.9), "the spend is restored, not reset"
assert g2.timeout_s(3600, floor_s=0, soft=False) == pytest.approx(7 * 60, abs=6) # 15 − 8, never a fresh 15
@@ -78,20 +73,18 @@ def test_build_context_restores_the_budget_snapshot(tmp_path, settings):
ws = Workspace(tmp_path / "runs" / "r").create()
track = StaticObjectTrack(services=FakeServices(), settings=settings, runtime=FakeRuntime(Language.THREEJS))
- g = BudgetGuard(Budget(max_minutes=10.0))
- g.charge(Usage(backend="gemini", cost_usd=0.4), stage="baseline")
state = RunState()
- state.extra["budget_snapshot"] = g.snapshot().model_dump(mode="json")
+ state.extra["budget_snapshot"] = BudgetSnapshot(active_s=240.0).model_dump(mode="json")
ctx = track.build_context(spec, ws, EventLog(ws.events_path), state)
- assert ctx.budget.billed_usd == pytest.approx(0.4) and ctx.budget.calls == 1
- assert ctx.budget.by_stage["baseline"] == pytest.approx(0.4)
+ assert ctx.budget.elapsed_minutes() == pytest.approx(4.0, abs=0.1)
- # a snapshot written before by_round was retired (2026-08-30) still loads: extra keys are ignored
- legacy = dict(state.extra["budget_snapshot"], by_round={"0": {"baseline": 0.4}})
+ # a snapshot from before the guard lost its money (2026-09-22) still loads: extra keys are ignored
+ legacy = {"spent": {"cost_usd": 0.4}, "billed_usd": 0.4, "calls": 1, "by_stage": {"baseline": 0.4},
+ "by_round": {"0": {"baseline": 0.4}}, "active_s": 240.0}
state2 = RunState()
state2.extra["budget_snapshot"] = legacy
ctx2 = track.build_context(spec, ws, EventLog(ws.events_path), state2)
- assert ctx2.budget.billed_usd == pytest.approx(0.4)
+ assert ctx2.budget.elapsed_minutes() == pytest.approx(4.0, abs=0.1)
# --------------------------------------------------------------- ordering: single-shot
@@ -106,12 +99,13 @@ def test_a_paid_single_shot_response_is_persisted_when_the_budget_trips(tmp_path
guard._active_s = 120.0 # noqa: SLF001
return answer
- model = FakeChatModel(slow_call, cost=0.05) # paid work, booked before the stop
+ model = metered_chat_model(FakeChatModel(slow_call, cost=0.05)) # paid work, on the ledger before the stop
task = GenerationTask(label="baseline", prompt="p", files_hint=["src/object.js"])
- res = generate_files(ws, model=model, task=task, budget=guard) # must NOT raise
+ with run_ledger(ws.root):
+ res = generate_files(ws, model=model, task=task, budget=guard) # must NOT raise
assert res.ok and (ws.src / "object.js").is_file()
assert res.transcript_path and Path(res.transcript_path).is_file()
- assert guard.spent.cost_usd == pytest.approx(0.05) # booked, dollar for dollar
+ assert sum(r.cost_usd for r in load_ledger(ws.root)) == pytest.approx(0.05) # booked, dollar for dollar
assert not guard.ok() # the phase boundary (steps._run_phase) turns this into the stop
@@ -133,7 +127,7 @@ def test_single_shot_obeys_the_run_clock(tmp_path):
past._active_s = 120.0 # noqa: SLF001 — already past 1 min
with pytest.raises(BudgetExceeded):
generate_files(ws, model=model, task=task, budget=past)
- assert calls == [] and past.spent.cost_usd == 0.0 # refused BEFORE any model call
+ assert calls == [] # refused BEFORE any model call
live = BudgetGuard(Budget(max_minutes=10.0))
live._active_s = 8 * 60.0 # noqa: SLF001 — 2 minutes left
@@ -155,10 +149,11 @@ def test_single_shot_obeys_the_run_clock(tmp_path):
live2 = BudgetGuard(Budget(max_minutes=10.0))
tm = TruncatingModel()
- with pytest.raises(BudgetExceeded):
- generate_files(ws, model=tm, task=task, budget=live2)
+ with run_ledger(tmp_path / "trunc"), pytest.raises(BudgetExceeded):
+ generate_files(ws, model=metered_chat_model(tm), task=task, budget=live2)
assert tm.calls == 1 # the retry was never bought
- assert live2.spent.cost_usd == pytest.approx(0.01) # ...but the paid first call is booked
+ (row,) = load_ledger(tmp_path / "trunc") # ...but the paid first call is on the ledger
+ assert row.cost_usd == pytest.approx(0.01)
# --------------------------------------------------------------- ordering: fan-out siblings
@@ -168,7 +163,7 @@ def test_a_budget_tripped_candidate_still_lets_the_sibling_be_adopted(tmp_path,
def writer(job, ws_):
if ws_.root.name == "c0": # <ws>/_cand/c0 trips the ceiling mid-generation
- raise BudgetExceeded("elapsed 9.99 min exceeds max_minutes 5.00", spent_usd=9.99)
+ raise BudgetExceeded("elapsed 9.99 min exceeds max_minutes 5.00")
return {"src/object.js": f"// {job.label} in {ws_.root.name}\n"
"export function build(THREE) { return new THREE.Group(); }\n"}
@@ -184,7 +179,7 @@ def test_a_budget_tripped_candidate_still_lets_the_sibling_be_adopted(tmp_path,
# --------------------------------------------------------------- ordering: planner attempts
-def test_planner_attempt_zero_usage_reaches_the_guard_when_attempt_one_raises(tmp_path):
+def test_planner_attempt_zero_usage_reaches_the_ledger_when_attempt_one_raises(tmp_path):
guard = BudgetGuard(Budget(max_minutes=10.0))
ws = Workspace(tmp_path / "run").create()
calls = {"n": 0}
@@ -195,12 +190,12 @@ def test_planner_attempt_zero_usage_reaches_the_guard_when_attempt_one_raises(tm
return {"object_name": "Chair"} # schema-invalid → one re-ask
raise RuntimeError("model fell over mid-retry")
- with pytest.raises(RuntimeError, match="fell over"):
+ with run_ledger(ws.root), pytest.raises(RuntimeError, match="fell over"):
run_planner(make_spec(), "fake:planner", StaticPlan, ws,
- model=FakeChatModel(responder, cost=0.01), budget=guard)
+ model=metered_chat_model(FakeChatModel(responder, cost=0.01)), budget=guard)
assert calls["n"] == 2
- assert guard.spent.cost_usd == pytest.approx(0.01) # attempt 0's dollars are booked
- assert guard.by_stage.get("plan") == pytest.approx(0.01) and guard.calls == 1
+ paid, failed = load_ledger(ws.root) # attempt 0's dollars are booked; the crash is a $0 error row
+ assert (paid.stage, paid.cost_usd, failed.outcome, failed.cost_usd) == ("plan", pytest.approx(0.01), "error", 0.0)
def test_planning_error_dollars_are_booked_exactly_once(tmp_path):
@@ -210,11 +205,11 @@ def test_planning_error_dollars_are_booked_exactly_once(tmp_path):
def responder(req):
return {"object_name": "Chair"} # never validates
- with pytest.raises(PlanningError) as ei:
+ with run_ledger(ws.root), pytest.raises(PlanningError) as ei:
run_planner(make_spec(), "fake:planner", StaticPlan, ws,
- model=FakeChatModel(responder, cost=0.01), budget=guard)
+ model=metered_chat_model(FakeChatModel(responder, cost=0.01)), budget=guard)
n = 1 + MAX_VALIDATION_REASKS # every paid attempt, each booked once
- assert guard.spent.cost_usd == pytest.approx(0.01 * n)
+ assert sum(r.cost_usd for r in load_ledger(ws.root)) == pytest.approx(0.01 * n)
assert ei.value.usage.cost_usd == pytest.approx(0.01 * n) # the error still reports the total
@@ -254,7 +249,7 @@ def test_a_skeleton_only_budget_trip_still_salvages_nothing(tmp_path, chair_plan
adopted winner, so the salvage hook must keep saying no off-scene."""
def writer(job, ws_):
- raise BudgetExceeded("elapsed 11.0 min exceeds max_minutes 10.0", spent_usd=1.0)
+ raise BudgetExceeded("elapsed 11.0 min exceeds max_minutes 10.0")
ws = Workspace(tmp_path / "runs" / "bare")
track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.6,)), agent=FakeAgent(writer),
diff --git a/harness/tests/orchestrator_tracks/test_cost_accounting.py b/harness/tests/orchestrator_tracks/test_cost_accounting.py
index 8217c4df..814fafe7 100644
--- a/harness/tests/orchestrator_tracks/test_cost_accounting.py
+++ b/harness/tests/orchestrator_tracks/test_cost_accounting.py
@@ -1,9 +1,8 @@
-"""Cost replay, ledger attribution, and billed-versus-notional accounting."""
+"""Recorded runs replayed through today's accounting: every billed dollar is one ledger row."""
from __future__ import annotations
import json
-import time
from pathlib import Path
import pytest
@@ -12,7 +11,7 @@ from codeverse3d.contracts.agent import AgentJob, AgentResult
from codeverse3d.contracts.common import Budget, Usage
from codeverse3d.cost.instrument import MeteredAgent, run_ledger
from codeverse3d.cost.ledger import load_ledger, record_call
-from codeverse3d.orchestrator import BudgetExceeded, BudgetGuard, usage_delta
+from codeverse3d.orchestrator import BudgetExceeded, BudgetGuard
from codeverse3d.tracks.generation import GenerationTask, _SessionAcc
FIXTURES = json.loads((Path(__file__).parent / "data" / "offrecord_runs.json").read_text())["runs"]
@@ -38,19 +37,17 @@ class _RecordedAgent:
return AgentResult(ok=True, exit_reason=s["exit_reason"] or "completed", usage=_usage(s))
-def replay(name: str, root: Path | None, *, texture: bool = True) -> BudgetGuard:
- """Spend one recorded run's money through today's accounting path.
-
- The guard only buckets and enforces; the ledger rows come from the ONE writer —
- ``MeteredAgent`` for a session, ``record_call`` (what ``MeteredChatModel`` does
- per call) for the judge and texture money — inside ``run_ledger(root)``, as
- ``BaseTrack.run`` opens it. ``root=None`` replays with no ledger at all."""
+def replay(name: str, root: Path) -> float:
+ """Spend one recorded run's money through today's accounting path and return the ledger's
+ total: the ONE writer — ``MeteredAgent`` for a session, ``record_call`` (what
+ ``MeteredChatModel`` does per call) for the judge and texture money — inside
+ ``run_ledger(root)``, as ``BaseTrack.run`` opens it."""
fx = FIXTURES[name]
guard = BudgetGuard(Budget(max_minutes=10_000))
tasks: dict[tuple[str, int], list[dict]] = {}
for s in fx["sessions"]:
tasks.setdefault((s["job_label"] or s["label"], int(s["round"] or 0)), []).append(s)
- with run_ledger(root or Path("/nonexistent"), run=name, create=root is not None):
+ with run_ledger(root, run=name):
for (label, rnd), sessions in tasks.items():
sessions.sort(key=lambda s: s["attempt"])
task = GenerationTask(label=label, prompt="p", round=rnd,
@@ -60,112 +57,56 @@ def replay(name: str, root: Path | None, *, texture: bool = True) -> BudgetGuard
for i in range(len(sessions)): # attempt 1, then the .a2 retry — exactly as run_agent_task does
job_label = label if i == 0 else f"{label}.a{i + 1}"
acc.run(agent, AgentJob(workspace="/tmp", prompt="p", label=job_label, round=rnd))
- for j in fx["judges"]: # steps.run_round: the judge's MeteredChatModel writes, the guard buckets
+ for j in fx["judges"]: # the judge's MeteredChatModel writes one row per verdict call
u = Usage(cost_usd=j["cost_usd"], input_tokens=j["input_tokens"], output_tokens=j["output_tokens"])
record_call(u, round=j["round"], stage="judge", role="judge", label=j["rubric"])
- guard.add(u, stage="judge")
- if texture:
- for t in fx["post_hoc_texture_passes"]: # lifecycle._texture_pass
- u = Usage(cost_usd=t["cost_usd"])
- record_call(u, stage="texture", role="image", label="texture_pass")
- guard.add(u, stage="texture")
- return guard
+ for t in fx["post_hoc_texture_passes"]: # a texture pass joins the run's ledger
+ record_call(Usage(cost_usd=t["cost_usd"]), stage="texture", role="image", label="texture_pass")
+ return sum(r.cost_usd for r in load_ledger(root))
@pytest.mark.parametrize("name", sorted(FIXTURES))
-def test_every_recorded_dollar_reaches_the_guard_and_the_ledger(name, tmp_path):
+def test_every_recorded_dollar_reaches_the_ledger_once(name, tmp_path):
fx = FIXTURES[name]
- guard = replay(name, tmp_path)
+ total = replay(name, tmp_path)
sessions = sum(s["cost_usd"] for s in fx["sessions"])
judges = sum(j["cost_usd"] for j in fx["judges"])
texture = sum(t["cost_usd"] for t in fx["post_hoc_texture_passes"])
- # nothing is lost between the sessions and the guard — retries included
- assert guard.spent.cost_usd == pytest.approx(sessions + judges + texture, abs=1e-6)
- # and the ledger the run ships is the same number, one row per call
- rows = load_ledger(tmp_path) # telemetry/cost.jsonl, one writer
+ # nothing is lost between the sessions and the ledger — retries included — and nothing twice
+ assert total == pytest.approx(sessions + judges + texture, abs=1e-6)
+ rows = load_ledger(tmp_path) # telemetry/cost.jsonl, one writer, one row per call
assert len(rows) == len(fx["sessions"]) + len(fx["judges"]) + len(fx["post_hoc_texture_passes"])
- assert sum(r.cost_usd for r in rows) == pytest.approx(guard.spent.cost_usd, abs=1e-6)
- # a retried session is its OWN row, labelled .a2 — the reconstructor had to guess
- # this from a trajectory directory name, and double-counted when it guessed wrong
+ # a retried session is its OWN row, labelled .a2
retried = [s for s in fx["sessions"] if s["attempt"] > 1]
assert sum(1 for r in rows if r.label.endswith(".a2")) == len(retried)
-def test_the_round_the_budget_cut_is_still_reported(tmp_path):
+def test_the_round_the_budget_cut_is_still_in_the_total(tmp_path):
fx = FIXTURES["tool_med_hand_drill"]
assert fx["rounds_in_record"] == [0, 1]
cut = [s for s in fx["sessions"] if s["round"] == 2]
assert cut, "fixture must contain the round the budget cut"
- guard = replay("tool_med_hand_drill", tmp_path)
+ total = replay("tool_med_hand_drill", tmp_path)
burned = sum(s["cost_usd"] for s in cut)
- # per-round money lives in the ledger (CallCost.round), the one place that keeps it;
- # the guard totals and enforces, and its own by_round map was retired 2026-08-30
assert sum(r.cost_usd for r in load_ledger(tmp_path) if r.round == 2) == pytest.approx(burned, abs=1e-6)
- assert burned > 0.8 # $0.86 that record.rounds never mentioned
- assert guard.spent.cost_usd > burned # and it is inside the run total, not beside it
+ assert burned > 0.8 and total > burned # $0.86 that record.rounds never mentioned, inside the total
-def test_post_hoc_texture_passes_are_inside_the_total_now():
+def test_post_hoc_texture_passes_are_inside_the_total(tmp_path):
fx = FIXTURES["furn_hard_rolltop_desk"]
- with_tex = replay("furn_hard_rolltop_desk", None)
- without = replay("furn_hard_rolltop_desk", None, texture=False)
- off_record = with_tex.spent.cost_usd - without.spent.cost_usd
- assert off_record == pytest.approx(0.6562, abs=1e-3)
- assert without.spent.cost_usd == pytest.approx(fx["recorded_total_usd"], abs=0.02)
- assert with_tex.by_stage["texture"] == pytest.approx(0.6562, abs=1e-3)
-
-
-# ----------------------------------------------------------------------------- guard mechanics
-def test_spend_buckets_by_stage_and_only_charge_enforces():
- g0 = BudgetGuard(Budget(max_minutes=60))
- g0.charge(Usage(cost_usd=0.4), stage="baseline")
- g0.add(Usage(cost_usd=0.05), stage="judge")
- assert g0.by_stage == {"baseline": pytest.approx(0.4), "judge": pytest.approx(0.05)}
- assert list(g0.stage_summary())[0] == "baseline" # biggest bucket first
-
- g = BudgetGuard(Budget(max_minutes=1.0), start_time=time.time() - 600) # already past
- g.add(Usage(cost_usd=0.9), stage="texture") # over the ceiling, never raises
- assert not g.ok() and g.spent.cost_usd == pytest.approx(0.9)
- with pytest.raises(BudgetExceeded):
- g.charge(Usage(cost_usd=0.01), stage="refine")
- assert g.spent.cost_usd == pytest.approx(0.91) # the money is counted even when it raises
-
-
-def test_usage_delta_reports_what_a_round_burned():
- before = Usage(cost_usd=1.0, input_tokens=100, output_tokens=10)
- after = Usage(cost_usd=1.75, input_tokens=350, output_tokens=40)
- d = usage_delta(after, before)
- assert d.cost_usd == pytest.approx(0.75) and d.input_tokens == 250 and d.output_tokens == 30
- assert usage_delta(before, after).cost_usd == 0.0 # never negative
-
+ with_tex = replay("furn_hard_rolltop_desk", tmp_path / "with")
+ tex = sum(t["cost_usd"] for t in fx["post_hoc_texture_passes"])
+ assert tex == pytest.approx(0.6562, abs=1e-3)
+ assert with_tex - tex == pytest.approx(fx["recorded_total_usd"], abs=0.02)
-def test_the_guard_never_writes_a_ledger_row_itself(tmp_path, monkeypatch):
- """One writer: a charge outside any run ledger lands nowhere, not in a second file."""
- monkeypatch.setenv("C3D_COST_LEDGER", str(tmp_path / "process.jsonl"))
- from codeverse3d.cost import ledger as ledger_mod
- monkeypatch.setattr(ledger_mod, "_fallback_read", False)
- quiet = BudgetGuard(Budget(max_minutes=60))
- quiet.charge(Usage(cost_usd=0.1), stage="plan")
- assert quiet.spent.cost_usd == pytest.approx(0.1)
- assert not (tmp_path / "process.jsonl").exists() and not any(tmp_path.glob("**/*.jsonl"))
-
-
-def test_backend_billing_and_time_guard():
+# ----------------------------------------------------------------------------- the clock
+def test_the_guard_is_a_clock_with_a_soft_share_for_the_baseline():
g = BudgetGuard(Budget(max_minutes=60.0), soft_fraction=0.55)
- g.add(Usage(backend="codex", model="gpt-5.6-sol", cost_usd=7.712))
- assert g.billed_usd == 0.0
assert g.soft_exceeded() == "" and g.ok()
- assert g.spent.cost_usd == pytest.approx(7.712)
- assert g.summary()["notional_usd"] == pytest.approx(7.712)
- assert g.summary()["spent_usd"] == 0.0
-
- g.add(Usage(backend="gemini", cost_usd=5.0))
- assert g.billed_usd == pytest.approx(5.0)
- assert g.ok() and not g.soft_exceeded()
-
- g.add(Usage(backend="some-new-provider", cost_usd=4.0))
- assert g.billed_usd == pytest.approx(9.0)
+ g._active_s = 34 * 60.0 # noqa: SLF001 — past the baseline's 55 %, inside the ceiling
+ assert g.soft_exceeded() and g.ok()
g._active_s = 3600.0 # noqa: SLF001
with pytest.raises(BudgetExceeded):
g.check()
+ assert set(g.summary()) == {"elapsed_min", "max_minutes"}, "the guard keeps no money"
diff --git a/harness/tests/orchestrator_tracks/test_cost_latency.py b/harness/tests/orchestrator_tracks/test_cost_latency.py
index c2e0ee4e..a447e125 100644
--- a/harness/tests/orchestrator_tracks/test_cost_latency.py
+++ b/harness/tests/orchestrator_tracks/test_cost_latency.py
@@ -66,7 +66,6 @@ def test_soft_budget_degrades_before_the_hard_cap_and_grace_reopens_it():
assert g.ok() and g.hard_minutes == pytest.approx(15.0)
g.grant_grace(minutes=1.0) # never shrinks
assert g.hard_minutes == pytest.approx(15.0)
- assert g.summary()["soft_fraction"] == 0.55
def test_timeout_is_clipped_to_the_wall_clock_left():
diff --git a/harness/tests/orchestrator_tracks/test_fix_batch2.py b/harness/tests/orchestrator_tracks/test_fix_batch2.py
index 63172157..05c2f840 100644
--- a/harness/tests/orchestrator_tracks/test_fix_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_fix_batch2.py
@@ -7,7 +7,7 @@ import json
import pytest
from codeverse3d.contracts.artifacts import BuildResult, GateFinding, GateReport, Judgment, Severity
-from codeverse3d.contracts.common import Language, Track, Usage
+from codeverse3d.contracts.common import Language, Track
from codeverse3d.contracts.plan import BBox, PartPlan, ScenePlan, StaticPlan
from codeverse3d.contracts.run import RoundRecord, RunStatus
from codeverse3d.orchestrator import RoundPolicy, RunState
@@ -24,6 +24,7 @@ from tests.orchestrator_tracks.fakes import (
FakeRuntime,
FakeServices,
_planner,
+ bill_verdict,
)
@@ -31,9 +32,9 @@ def _writer(job, ws):
return {"src/object.js": f"// {job.label} r{job.round}\nexport function build(THREE) {{ return new THREE.Group(); }}\n"}
-def degraded_verdict(cost: float = 0.003) -> Judgment:
+def degraded_verdict(cost: float = 0.003, round_index: int = 0) -> Judgment:
return Judgment(rubric="fake_v1", judge_backend="fake", scores={}, overall=0.0, passed=False,
- summary="judge_error: 503 UNAVAILABLE", usage=Usage(backend="fake", cost_usd=cost),
+ summary="judge_error: 503 UNAVAILABLE", usage=bill_verdict(cost, round_index), # paid all the same
raw=json.dumps({"status": "degraded"}))
@@ -49,7 +50,7 @@ class ScriptedJudge(FakeJudge):
i = min(len(self.all_calls), len(self.script) - 1)
self.all_calls.append(inp)
if self.script[i] == "degraded":
- return degraded_verdict()
+ return degraded_verdict(round_index=inp.round_index)
self.calls = self.all_calls[: i] # FakeJudge indexes by len(calls)
out = super().judge(inp)
out = out.model_copy(update={"overall": float(self.script[i]), "passed": float(self.script[i]) >= 0.8,
@@ -72,8 +73,8 @@ def test_judged_round_survives_budget_ceiling_crossed_by_judge(tmp_path, chair_p
assert rec.status is RunStatus.BUDGET and rec.extra["stop_reason"] == "budget"
assert len(rec.rounds) == 1 and rec.rounds[0].score == pytest.approx(0.55)
assert (ws.root / "rounds" / "r00.json").is_file() and ws.judge_path(0).is_file()
- # the judge cost is still in the totals
- assert rec.total_usage.cost_usd > 0.05
+ # the judge cost is still in the totals, and in the round's
+ assert rec.total_usage.cost_usd > 0.05 and rec.rounds[0].usage.cost_usd > 0.05
# --------------------------------------------------------------------- finding: finalise with a dirty tree at the last commit
@@ -93,12 +94,10 @@ def test_finalise_restores_the_last_round_when_an_aborted_round_dirtied_src(tmp_
text = (ws.src / "object.js").read_text()
assert "baseline r0" in text and "refine" not in text # restored, not the aborted round's edits
assert ws.changed_files() == [] # tree clean at the delivered commit
- # finding lifecycle.py:256 — the aborted round's charges survive into run_state for resume
- state = RunState.load(ws)
- spent = state.extra["budget_snapshot"]["spent"]
- assert spent["cost_usd"] == pytest.approx(0.025, abs=1e-6)
- snap = state.extra["budget_snapshot"]
- assert snap["billed_usd"] == pytest.approx(0.025, abs=1e-6) and snap["calls"] > 0
+ # finding lifecycle.py:256 — the aborted round's money survives for resume: it is on the
+ # ledger (planner 0.002 + baseline agent 0.01 + judge 0.003 + the aborted refine agent 0.01)
+ assert rec.total_usage.cost_usd == pytest.approx(0.025, abs=1e-6)
+ assert rec.extra["aborted_rounds"][0]["cost_usd"] == pytest.approx(0.01, abs=1e-6)
# --------------------------------------------------------------------- finding: finalise rebuild fails after invalidation
@@ -163,11 +162,9 @@ def test_spent_usage_saved_when_a_round_crashes(tmp_path, chair_plan, settings):
runtime=FakeRuntime(Language.THREEJS))
with pytest.raises(RuntimeError, match="gate exploded"):
track.run(spec, ws)
- state = RunState.load(ws)
- spent = state.extra["budget_snapshot"]["spent"]
# planner 0.002 + r0 agent 0.01 + r0 judge 0.003 + r1 agent 0.01 — the r1 charge must not vanish
- assert spent["cost_usd"] == pytest.approx(0.025, abs=1e-6)
- assert state.extra["budget_snapshot"]["billed_usd"] == pytest.approx(0.025, abs=1e-6)
+ assert json.loads(ws.record_path.read_text())["total_usage"]["cost_usd"] == pytest.approx(0.025, abs=1e-6)
+ assert json.loads((ws.root / "rounds" / "aborted_r01.json").read_text())["usage"]["cost_usd"] == pytest.approx(0.01)
# --------------------------------------------------------------------- finding: degraded judge verdicts are glitches, not scores
@@ -187,16 +184,13 @@ def test_degraded_verdict_never_scores_and_run_stops_as_judge_unavailable(tmp_pa
# the degraded summary never reaches a refine prompt
assert len(judge.all_calls) == 3
# audit f1/f3: both degraded verdicts were PAID and the glitch note survives the
- # round's notes join — money and note reach the persisted record and cost.round
+ # round's notes join — money and note reach the persisted record
r1 = rec.rounds[1]
assert "judge degraded" in r1.notes
assert r1.usage.cost_usd == pytest.approx(0.016, abs=1e-6) # agent 0.01 + 2 x degraded 0.003
saved = json.loads((ws.root / "rounds" / "r01.json").read_text())
assert "judge degraded" in saved["notes"]
assert saved["usage"]["cost_usd"] == pytest.approx(0.016, abs=1e-6)
- r1_cost = next(e for e in EventLog(ws.events_path).read()
- if e["event"] == "cost.round" and e["round"] == 1)
- assert r1_cost["judge_usd"] == pytest.approx(0.003) and r1_cost["waste_reason"] == "unjudged"
def test_degraded_verdict_recovers_via_rejudge_of_same_commit(tmp_path, chair_plan, settings):
@@ -214,15 +208,7 @@ def test_degraded_verdict_recovers_via_rejudge_of_same_commit(tmp_path, chair_pl
assert [j.round for j in agent.jobs] == [0, 1, 2, 3]
saved = RoundRecord.model_validate(json.loads((ws.root / "rounds" / "r01.json").read_text()))
assert saved.judgment is not None and saved.judgment.overall == pytest.approx(0.85)
- # audit f2: the recovery re-emits a CORRECTED cost.round so the audit stream stops
- # counting r01 as wasted=unjudged; f1: every paid verdict is on the round's usage
- r1_costs = [e for e in EventLog(ws.events_path).read()
- if e["event"] == "cost.round" and e["round"] == 1]
- assert len(r1_costs) == 2
- first, corrected = r1_costs
- assert first["wasted"] is True and first["waste_reason"] == "unjudged" and first["corrected"] is False
- assert corrected["corrected"] is True and corrected["wasted"] is False
- assert corrected["score"] == pytest.approx(0.85) and corrected["judge_usd"] == pytest.approx(0.003)
+ # audit f1: every paid verdict is on the round's usage
assert rec.rounds[1].usage.cost_usd == pytest.approx(0.016, abs=1e-6) # agent + degraded + rejudge
assert "judge degraded" in rec.rounds[1].notes and "judge degraded" in saved.notes
diff --git a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
index 7ae2199f..c4a77dcc 100644
--- a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
+++ b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
@@ -8,6 +8,8 @@ import pytest
from codeverse3d.contracts.common import Language, Track
from codeverse3d.contracts.plan import ScenePlan, StaticPlan
+from codeverse3d.cost.instrument import metered_chat_model, run_ledger
+from codeverse3d.cost.ledger import load_ledger
from codeverse3d.orchestrator import BudgetGuard, RoundPolicy, RunState
from codeverse3d.proc import EventLog
from codeverse3d.tracks.common import RunContext
@@ -105,7 +107,7 @@ def test_generate_single_shot_writes_files_and_charges(tmp_ws):
task = GenerationTask(label="baseline", prompt="build it", system="sys", files_hint=["src/model.py"])
res = generate(tmp_ws, agent_id="single-shot:gemini:x", task=task, model=model, budget=budget, events=events)
assert res.ok and [c.path for c in res.files_changed] == ["src/model.py"] and (tmp_ws.src / "model.py").read_text() == "import bpy\n"
- assert budget.spent.cost_usd == pytest.approx(0.002)
+ assert res.usage.cost_usd == pytest.approx(0.002)
# the label carries the round so the cost ledger can attribute a single-shot call
assert "=== FILE:" in model.requests[0].system and model.requests[0].label == "baseline:r00"
assert (tmp_ws.trajectories / "baseline_r00" / "response.md").is_file()
@@ -138,10 +140,11 @@ def _valid_plan_dict() -> dict:
def test_planner_validates_retries_and_writes(tmp_ws):
spec = make_spec()
answers = [{"object_name": "X"}, _valid_plan_dict()]
- model = FakeChatModel(lambda req: answers.pop(0))
+ model = metered_chat_model(FakeChatModel(lambda req: answers.pop(0)))
events = EventLog(tmp_ws.events_path)
budget = BudgetGuard(spec.budget)
- p = plan(spec, "fake:planner", StaticPlan, tmp_ws, model=model, events=events, budget=budget)
+ with run_ledger(tmp_ws.root):
+ p = plan(spec, "fake:planner", StaticPlan, tmp_ws, model=model, events=events, budget=budget)
assert isinstance(p, StaticPlan) and p.object_name == "DiningChair" and tmp_ws.plan_path.is_file()
assert len(model.requests) == 2 and "failed validation" in model.requests[1].messages[-1].text
assert model.requests[0].response_schema is not None and "PascalCase" in model.requests[0].system
@@ -151,7 +154,7 @@ def test_planner_validates_retries_and_writes(tmp_ws):
# deterministic acceptance items from constraints were appended
texts = " ".join(a.text for a in p.acceptance)
assert "height = 0.820" in texts and "Includes: armrests" in texts
- assert budget.spent.cost_usd == pytest.approx(0.004)
+ assert sum(r.cost_usd for r in load_ledger(tmp_ws.root)) == pytest.approx(0.004) # both calls, paid
kinds = [e["event"] for e in events.read()]
assert "plan.invalid" in kinds and "plan.done" in kinds
@@ -252,7 +255,7 @@ def test_build_with_repair_single_shot_sends_file_contents_and_stops_at_max(tmp_
assert not out.ok and len(out.attempts) == 2 # max_repair_attempts=2 in make_spec
assert "--- src/model.py ---" in seen[0].messages[0].text and "import bpy" in seen[0].messages[0].text
assert seen[1].temperature > seen[0].temperature # escalation on identical signature
- assert ctx.budget.spent.cost_usd == pytest.approx(0.004)
+ assert out.usage.cost_usd == pytest.approx(0.004)
def test_format_error_report_and_cookbook_section():
diff --git a/harness/tests/orchestrator_tracks/test_orchestrator.py b/harness/tests/orchestrator_tracks/test_orchestrator.py
index 0c9866ac..172d3c7b 100644
--- a/harness/tests/orchestrator_tracks/test_orchestrator.py
+++ b/harness/tests/orchestrator_tracks/test_orchestrator.py
@@ -15,7 +15,7 @@ from codeverse3d.contracts.artifacts import (
Judgment,
Severity,
)
-from codeverse3d.contracts.common import Budget, Usage
+from codeverse3d.contracts.common import Budget
from codeverse3d.contracts.plan import AcceptanceItem
from codeverse3d.contracts.run import RunStatus
from codeverse3d.orchestrator import (
@@ -32,15 +32,13 @@ from codeverse3d.proc import EventLog, fan_out
# ----------------------------------------------------------------------------- budget
-def test_budget_guard_charges_and_raises():
+def test_budget_guard_raises_past_its_ceiling():
g = BudgetGuard(Budget(max_minutes=10.0))
- g.charge(Usage(cost_usd=0.02))
- g.charge(Usage(cost_usd=0.02))
assert g.ok()
g._active_s = 11 * 60 # noqa: SLF001 — past the ceiling
with pytest.raises(BudgetExceeded) as ei:
- g.charge(Usage(cost_usd=0.02))
- assert "max_minutes" in ei.value.reason and g.spent.cost_usd == pytest.approx(0.06)
+ g.check()
+ assert "max_minutes" in ei.value.reason and not g.ok()
def test_budget_guard_time_ceiling():
diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
index 96a6827c..12ffef84 100644
--- a/harness/tests/orchestrator_tracks/test_refactor_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
@@ -98,19 +98,21 @@ def test_judge_context_needs_no_run_context(tmp_path):
assert ScenePipeline().judge_context(ws, None, 0, build, []) == ""
-def test_graphics_planner_hooks_charge_budget_on_planning_error(tmp_ws):
+def test_graphics_planner_hooks_book_a_planning_error_on_the_ledger(tmp_ws):
import pytest
from codeverse3d.contracts.plan import GraphicsPlan
from codeverse3d.contracts.spec import Budget
+ from codeverse3d.cost.instrument import metered_chat_model, run_ledger
+ from codeverse3d.cost.ledger import load_ledger
from codeverse3d.orchestrator import BudgetGuard
from codeverse3d.tracks.planner import PlanningError
from codeverse3d.tracks.planner import plan as run_planner
spec = make_spec(Track.GRAPHICS, Language.GLSL_SHADER)
budget = BudgetGuard(Budget(max_minutes=10))
- always_bad = FakeChatModel(lambda req: {"title": "x"})
- with pytest.raises(PlanningError):
+ always_bad = metered_chat_model(FakeChatModel(lambda req: {"title": "x"}))
+ with run_ledger(tmp_ws.root), pytest.raises(PlanningError):
run_planner(spec, "fake:planner", GraphicsPlan, tmp_ws, model=always_bad, budget=budget)
- assert budget.spent.cost_usd > 0, "a failed re-ask is still paid for"
+ assert sum(r.cost_usd for r in load_ledger(tmp_ws.root)) > 0, "a failed re-ask is still paid for"
diff --git a/harness/tests/orchestrator_tracks/test_resume_reconcile.py b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
index 3a516fd1..d41b28da 100644
--- a/harness/tests/orchestrator_tracks/test_resume_reconcile.py
+++ b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
@@ -225,47 +225,22 @@ def test_notes_survive_a_judge_crash(tmp_path, chair_plan, settings):
assert "judge failed: RuntimeError: vlm 500" in saved["notes"]
-def test_resume_charges_for_spend_the_snapshot_missed(tmp_path, chair_plan, settings):
- """The ledger is appended per CALL, the snapshot saved at boundaries: a crash between
- a round's spend and its save handed the resumed run that money back — silently
- under-counting is how a resumed run walks past its ceiling."""
+def test_a_resumed_runs_total_is_its_ledger_whatever_a_session_saved(tmp_path):
+ """The ledger is appended per CALL; a session's own bookkeeping is saved at boundaries.
+ A crash between a call and a save used to hand the resumed run that money back. The
+ record's total is read off the ledger when it is written, so nothing a crash kept out
+ of a snapshot can go missing — and a subscription backend's row counts at list price
+ like any other (the billed / notional split went with the guard's money, 2026-09-22)."""
+ from codeverse3d.contracts.run import RunRecord
from codeverse3d.cost.ledger import open_run_ledger
from codeverse3d.cost.types import CallCost
- from codeverse3d.orchestrator import BudgetGuard, BudgetSnapshot
- from codeverse3d.tracks.lifecycle import _reconcile_billed_from_ledger
+ from codeverse3d.record.record import package_run
ws = Workspace(tmp_path / "runs" / "r")
ws.create()
led = open_run_ledger(ws.root)
- for cost in (0.30, 0.12): # what the provider actually billed
- led.append(CallCost(run=ws.root.name, model="gemini:flash", label="planner", cost_usd=cost))
- guard = BudgetGuard(make_spec().budget)
- guard.restore(BudgetSnapshot(spent=guard.spent, billed_usd=0.10, # the boundary save missed 0.32
- calls=1, by_stage={}, active_s=0.0))
- _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))
- assert guard.billed_usd == pytest.approx(0.42), "resume must charge for every billed call"
- guard.billed_usd = 5.0 # a snapshot AHEAD of the ledger wins
- _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))
- assert guard.billed_usd == pytest.approx(5.0), "reconcile never lowers what was already billed"
-
-
-def test_resume_reconcile_keeps_subscription_spend_notional(tmp_path, settings):
- """A codex/claude/agy ledger row is priced at list rates but bills $0 (bills_usd):
- resuming a subscription-backend run offline must not flip billed from $0 to the
- notional sum — reconcile shares the exact predicate the live spend path uses."""
- from codeverse3d.cost.ledger import open_run_ledger
- from codeverse3d.cost.types import CallCost
- from codeverse3d.orchestrator import BudgetGuard
- from codeverse3d.tracks.lifecycle import _reconcile_billed_from_ledger
-
- ws = Workspace(tmp_path / "runs" / "sub")
- ws.create()
- led = open_run_ledger(ws.root)
- led.append(CallCost(run=ws.root.name, backend="codex", model="gpt-5.6-sol", label="generator", cost_usd=7.7))
- led.append(CallCost(run=ws.root.name, backend="claude-code", model="sonnet", cost_usd=1.1))
- guard = BudgetGuard(make_spec().budget)
- _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))
- assert guard.billed_usd == 0.0, "subscription cost is notional; resume must keep billed at $0"
- led.append(CallCost(run=ws.root.name, backend="gemini", model="gemini-3.7-flash", cost_usd=0.25))
- _reconcile_billed_from_ledger(guard, ws, EventLog(ws.events_path))
- assert guard.billed_usd == pytest.approx(0.25), "the API-billed row still counts in full"
+ for backend, cost in (("gemini", 0.30), ("gemini", 0.12), ("codex", 7.7)): # session 1, the crash, session 2
+ led.append(CallCost(run=ws.root.name, backend=backend, model="m", label="planner", cost_usd=cost))
+ rec = RunRecord(spec=make_spec(), workspace=str(ws.root))
+ package_run(ws, rec)
+ assert rec.total_usage.cost_usd == pytest.approx(8.12)
diff --git a/harness/tests/orchestrator_tracks/test_round_economics.py b/harness/tests/orchestrator_tracks/test_round_economics.py
index fd7ee4fc..ccf4f286 100644
--- a/harness/tests/orchestrator_tracks/test_round_economics.py
+++ b/harness/tests/orchestrator_tracks/test_round_economics.py
@@ -10,6 +10,8 @@ from codeverse3d.contracts.agent import AgentJob, AgentResult
from codeverse3d.contracts.artifacts import BuildResult, GateFinding, GateReport, Severity
from codeverse3d.contracts.common import Language, Usage
from codeverse3d.contracts.run import RoundRecord, RunStatus
+from codeverse3d.cost.instrument import MeteredAgent, run_ledger
+from codeverse3d.cost.ledger import load_ledger
from codeverse3d.orchestrator import RoundPolicy
from codeverse3d.proc import EventLog
from codeverse3d.tracks.generation import (
@@ -90,17 +92,18 @@ def test_hitting_the_cap_asks_for_a_landing_instead_of_killing_the_session(tmp_w
monkeypatch.setenv("C3D_AGENT_MAX_TURNS", "28")
events = EventLog(tmp_ws.events_path)
agent = _TurnAgent(writes_on=2) # session 1 burns its turns and writes nothing
- guard = _guard()
- res = run_agent_task(tmp_ws, agent=agent, task=GenerationTask(label="refine", prompt="do the work", round=1,
- kind="refine"),
- budget=guard, events=events)
+ with run_ledger(tmp_ws.root):
+ res = run_agent_task(tmp_ws, agent=MeteredAgent(agent),
+ task=GenerationTask(label="refine", prompt="do the work", round=1, kind="refine"),
+ budget=_guard(), events=events)
assert len(agent.jobs) == 2 and turn_capped(_result_of(agent, 0)) and agent.jobs[0].max_turns == 28
wrap = agent.jobs[1]
assert wrap.max_turns == DEFAULT_WRAPUP_TURNS and "LAST session" in wrap.prompt and "do the work" in wrap.prompt
assert res.ok and res.turn_capped and res.sessions == 2
- # both sessions are paid for and both are in the guard, under their own labels
- assert res.usage.cost_usd == pytest.approx(0.4) and guard.spent.cost_usd == pytest.approx(0.4)
- assert guard.by_stage["refine"] == pytest.approx(0.4)
+ # both sessions are paid for and each is a ledger row of its own
+ assert res.usage.cost_usd == pytest.approx(0.4)
+ rows = load_ledger(tmp_ws.root)
+ assert len(rows) == 2 and all(r.stage == "refine" and r.cost_usd == pytest.approx(0.2) for r in rows)
kinds = [e["event"] for e in events.read()]
assert "generate.turn_cap" in kinds
done = next(e for e in events.read() if e["event"] == "generate.done")
@@ -115,13 +118,13 @@ def test_a_crashing_second_session_never_erases_what_the_first_one_spent(tmp_ws)
raise RuntimeError("503 storm")
return super().run(job)
- guard = _guard()
agent = _Flaky(writes_on=99)
- res = run_agent_task(tmp_ws, agent=agent, task=GenerationTask(label="refine", prompt="p", round=2, kind="refine"),
- budget=guard)
- assert len(agent.jobs) == 2 and not res.ok
- assert guard.spent.cost_usd == pytest.approx(0.2) and res.usage.cost_usd == pytest.approx(0.2)
- assert "503 storm" in res.notes and guard.by_stage["refine"] == pytest.approx(0.2)
+ with run_ledger(tmp_ws.root):
+ res = run_agent_task(tmp_ws, agent=MeteredAgent(agent),
+ task=GenerationTask(label="refine", prompt="p", round=2, kind="refine"), budget=_guard())
+ assert len(agent.jobs) == 2 and not res.ok and "503 storm" in res.notes
+ (row,) = load_ledger(tmp_ws.root) # the first session's row stands
+ assert res.usage.cost_usd == pytest.approx(0.2) and row.cost_usd == pytest.approx(0.2) and row.round == 2
def _result_of(agent, i):
@@ -135,11 +138,8 @@ def _guard():
return BudgetGuard(Budget(max_minutes=100))
-def test_task_stage_names_the_cost_bucket_a_task_spends_in():
- from codeverse3d.tracks.generation import task_stage
-
- def t(kind: str, label: str = "x") -> str:
- return task_stage(GenerationTask(label=label, prompt="p", kind=kind))
+def test_a_task_kind_names_the_stage_its_session_row_is_filed_under():
+ from codeverse3d.cost.types import stage_for_label as t # what MeteredAgent files a session by
assert t("baseline") == "baseline" and t("refine") == "refine" and t("repair") == "repair"
assert t("rebuild") == "repair" # regenerating after a failed build IS repair money
@@ -147,9 +147,7 @@ def test_task_stage_names_the_cost_bucket_a_task_spends_in():
assert t("env") == "env"
# the static track's surface-detail round (gone 2026-09-22) was refine money, not "other":
# the recorded runs that have one still file it there
- assert t("detail") == "refine" and t("detail", "detail_seat") == "refine"
- # an unknown kind falls back to the label, which record_call maps by prefix
- assert t("something_new", "asset_koi") == "asset_koi"
+ assert t("detail") == "refine" and t("detail_seat") == "refine"
# ----------------------------------------------------------------------------- judge skipping
@@ -250,20 +248,16 @@ def test_a_regression_changes_nothing_about_the_loop_and_every_round_reports_its
events = EventLog(ws.events_path).read()
assert not [e for e in events if e["event"] in ("strategy.switch", "round.refine_from_best")]
assert not [j for j in agent.jobs if "made the artifact WORSE" in j.prompt]
- # every round reports what it burned
- costs = [e for e in events if e["event"] == "cost.round"]
- assert len(costs) == 4 and all("generate" in c["stages"] for c in costs)
- assert costs[0]["agent_turns"] == 0 # the fake agent keeps no transcript
- assert [c["wasted"] for c in costs] == [False] * 4 and "previous_best" not in costs[1]
- assert costs[2]["judge_usd"] > 0 and costs[2]["run_usd"] >= costs[1]["run_usd"]
- assert rec.extra["cost_by_stage"]["judge"] > 0 and rec.extra["cost_by_stage"]["refine"] > 0
- # ... and the run opens its own priced ledger (telemetry/cost.jsonl) even outside the CLI;
- # the ONE writer is the metered agent/model, so the injected fakes (a bare FakeAgent, a
- # FakeJudge with no chat model) leave it empty — see test_cost_accounting for the rows a
- # metered session writes
- from codeverse3d.cost.ledger import load_ledger
-
- assert load_ledger(ws.root) == [] and not (ws.root / "cost_ledger.jsonl").exists()
+ # every round's cost is the ledger rows it booked — the injected agent and planner are
+ # metered like the ones services hand out, and the fake judge bills a row per verdict as a
+ # real one's model does — and the run's total is its ledger's, the planner included, with
+ # no per-stage copy on the record
+ assert all(r.usage.cost_usd >= 0.02 + 0.003 for r in rec.rounds)
+ rows = load_ledger(ws.root)
+ assert {r.stage for r in rows} == {"plan", "baseline", "refine", "judge"}
+ assert rec.total_usage.cost_usd == pytest.approx(sum(r.cost_usd for r in rows))
+ assert rec.total_usage.cost_usd > sum(r.usage.cost_usd for r in rec.rounds)
+ assert "cost_by_stage" not in rec.extra and not [e for e in events if e["event"] == "cost.round"]
def test_a_lint_stuck_run_keeps_every_score_instead_of_deferring_the_verdict(tmp_path, chair_plan, settings):
@@ -306,9 +300,9 @@ def test_a_round_that_raises_still_reports_what_it_burned(tmp_path, chair_plan,
assert rec.extra["aborted_rounds"][0]["index"] == 1
# the aborted round is NOT resumable state, and its money is in the total
assert [r.index for r in rec.rounds] == [0]
- assert rec.total_usage.cost_usd >= sum(r.usage.cost_usd for r in rec.rounds)
- cut = next(e for e in EventLog(ws.events_path).read() if e["event"] == "cost.round" and e["round"] == 1)
- assert cut["wasted"] is True and cut["waste_reason"] == "aborted" and cut["total_usd"] > 0
+ assert rec.extra["aborted_rounds"][0]["cost_usd"] == pytest.approx(aborted["usage"]["cost_usd"])
+ assert rec.total_usage.cost_usd == pytest.approx(
+ sum(r.usage.cost_usd for r in rec.rounds) + aborted["usage"]["cost_usd"] + 0.002) # + the planner
def test_fan_out_workers_inherit_the_callers_context():
diff --git a/harness/tests/scene_gates/test_scene_textures_stage.py b/harness/tests/scene_gates/test_scene_textures_stage.py
index 6e71fc80..6dde03fe 100644
--- a/harness/tests/scene_gates/test_scene_textures_stage.py
+++ b/harness/tests/scene_gates/test_scene_textures_stage.py
@@ -145,10 +145,10 @@ def test_the_pack_has_the_last_word_over_the_recipes(template: str):
assert out.index("RECIPE_MARKER") < out.index("TEXTURE_BLOCK_MARKER")
-def test_the_stage_puts_the_pack_on_the_run_budget(monkeypatch, tmp_path):
- """The pack is an image-model call the run pays for; every other paid stage goes
- through the guard (`BudgetGuard.add`, post-hoc, like the object texture pass), so a
- ceiling can be crossed by a paid stage the run never saw."""
+def test_the_stage_writes_the_pack_and_books_nothing_itself(monkeypatch, tmp_path):
+ """The pack is paid for through the metered image and chat models: its money is on the
+ run's ledger (the only record of it), and the stage has no second book to keep — a
+ context with no budget at all is enough."""
from types import SimpleNamespace
import codeverse3d.reference as reference
@@ -161,14 +161,11 @@ def test_the_stage_puts_the_pack_on_the_run_budget(monkeypatch, tmp_path):
monkeypatch.setattr(plan_mod, "scene_texture_pack", lambda *a, **k: pack)
monkeypatch.setattr(reference, "_image_model", lambda _m: None)
- added: list[tuple[Usage, str]] = []
ctx = SimpleNamespace(
plan=None,
ws=SimpleNamespace(public=tmp_path / "public", commit=lambda *_a, **_k: None),
spec=SimpleNamespace(backends=SimpleNamespace(planner=None)),
settings=SimpleNamespace(cache_dir=tmp_path / "cache"),
events=SimpleNamespace(emit=lambda *a, **k: None),
- budget=SimpleNamespace(add=lambda u, *, stage: added.append((u, stage))),
)
assert S.SceneTrack()._textures_stage(ctx) == MANIFEST # type: ignore[arg-type]
- assert added == [(usage, "texture")]