Skip to content

Official Codebase of 3DCodeVerse

Python

budget: enforce max_usd on money that is billed, not on list price

gaoypeng9045d8a0ba56
diff --git a/harness/codeverse/cost/billing.py b/harness/codeverse/cost/billing.py
new file mode 100644
index 00000000..bc02dc0f
--- /dev/null
+++ b/harness/codeverse/cost/billing.py
@@ -0,0 +1,41 @@
+"""Which backends actually take dollars out of an account.
+
+``Usage.cost_usd`` answers "what would these tokens cost at list price?". That is the
+right number for a report, a $/complexity-point, or a flywheel record: it is comparable
+across backends and it does not depend on who is paying. It is the WRONG number for a
+spend guard, because a backend on a local subscription bills no dollars at all.
+
+Conflating the two degraded real runs. Measured 2026-08-25 on
+``tsr_scn_temple_night`` (``codex:gpt-5.6-sol``, ``--profile quality``): two Blender hero
+sessions priced at OpenAI list rates put the run at $7.712 against the profile's $4.40
+soft cap in 6.8 minutes, so the asset judge was skipped for both heroes and every later
+stage ran degraded — over a bill of exactly $0.00. The run's own cost ledger agreed it
+was $0.00; only the guard disagreed.
+
+So the split is: the LEDGER and the reports keep pricing everything, and the BUDGET
+enforces only what is billed. See ``orchestrator/budget.py`` for the enforcement side
+and ``docs/COST.md`` §25.
+
+What a subscription backend is still bounded by: ``max_minutes``, which
+``BudgetMeter.timeout_s`` also clips individual sessions against. Wall clock, not money,
+is the scarce resource when the money is flat-rate — a runaway session is still stopped,
+just by the ceiling that actually applies to it.
+"""
+
+from __future__ import annotations
+
+#: Backends that run on a flat-rate local subscription on this machine, so their
+#: ``cost_usd`` is notional (see ``CLAUDE.md`` "Environment": codex / claude / agy run on
+#: local subscriptions). ``gemini-cli`` is deliberately NOT here: it authenticates with
+#: an API key, so its tokens draw on a real per-token quota even when that quota is free.
+SUBSCRIPTION_BACKENDS = frozenset({"codex", "claude-code", "agy", "antigravity"})
+
+
+def bills_usd(backend: str | None) -> bool:
+ """True when this backend's ``cost_usd`` is money someone is actually charged.
+
+ Unknown backends bill: a new API provider that nobody remembered to classify must be
+ enforced, not exempted. Getting this wrong in the safe direction costs a degraded
+ run; getting it wrong in the other direction spends real money with no ceiling.
+ """
+ return (backend or "").strip().lower() not in SUBSCRIPTION_BACKENDS
diff --git a/harness/codeverse/orchestrator/budget.py b/harness/codeverse/orchestrator/budget.py
index 90733ef5..cc48956e 100644
--- a/harness/codeverse/orchestrator/budget.py
+++ b/harness/codeverse/orchestrator/budget.py
@@ -33,6 +33,7 @@ from pathlib import Path
from typing import Any
from codeverse.contracts.common import Budget, Usage
+from codeverse.cost.billing import bills_usd
log = logging.getLogger(__name__)
@@ -57,11 +58,23 @@ class BudgetGuard:
``BudgetExceeded`` when ``max_usd`` or ``max_minutes`` is exceeded.
"""
- def __init__(self, budget: Budget, start_time: float | None = None, *, soft_fraction: float = 1.0,
- run: str = "", ledger: str | Path | Any | None = None):
+ def __init__(
+ self,
+ budget: Budget,
+ start_time: float | None = None,
+ *,
+ soft_fraction: float = 1.0,
+ run: str = "",
+ ledger: str | Path | Any | None = None,
+ ):
self.budget = budget
self.start_time = time.time() if start_time is None else start_time
self.spent = Usage()
+ #: the share of ``spent.cost_usd`` that is money someone is actually charged.
+ #: The ceilings enforce THIS, not ``spent.cost_usd`` — a backend on a flat-rate
+ #: subscription prices its tokens notionally and must not consume a spend guard.
+ #: See ``cost/billing.py`` for why, and what still bounds a subscription run.
+ self.billed_usd = 0.0
self._lock = threading.Lock()
self.calls = 0
#: fraction of the hard ceilings the *baseline* may use (1.0 = no soft cap)
@@ -98,13 +111,17 @@ class BudgetGuard:
return
with self._lock:
self.spent = self.spent + usage
+ if bills_usd(getattr(usage, "backend", None)):
+ self.billed_usd += float(usage.cost_usd)
self.calls += 1
key = stage or OTHER_STAGE
self.by_stage[key] = self.by_stage.get(key, 0.0) + float(usage.cost_usd)
if round_index is not None:
per = self.by_round.setdefault(int(round_index), {})
per[key] = per.get(key, 0.0) + float(usage.cost_usd)
- self._ledger_row(usage, stage=key, role=role, label=label, round_index=round_index, outcome=outcome)
+ self._ledger_row(
+ usage, stage=key, role=role, label=label, round_index=round_index, outcome=outcome
+ )
if enforce:
self.check()
@@ -125,8 +142,16 @@ class BudgetGuard:
kw["enforce"] = False
self.spend(usage, **kw)
- def _ledger_row(self, usage: Usage, *, stage: str, role: str | None, label: str,
- round_index: int | None, outcome: str) -> None:
+ def _ledger_row(
+ self,
+ usage: Usage,
+ *,
+ stage: str,
+ role: str | None,
+ label: str,
+ round_index: int | None,
+ outcome: str,
+ ) -> None:
if self.ledger is None:
return
try:
@@ -137,8 +162,18 @@ class BudgetGuard:
# tokens, cache hits and latency — one aggregate row on top would double count.
if per_call_metering():
return
- record_call(usage, run=self.run, stage=stage, role=role, round=round_index, label=label,
- backend=usage.backend, model=usage.model, outcome=outcome, ledger=self.ledger)
+ record_call(
+ usage,
+ run=self.run,
+ stage=stage,
+ role=role,
+ round=round_index,
+ label=label,
+ backend=usage.backend,
+ model=usage.model,
+ outcome=outcome,
+ ledger=self.ledger,
+ )
except Exception as e: # noqa: BLE001 — accounting must never fail a run
log.debug("cost ledger row failed (%s): %s", self.ledger, e)
@@ -173,28 +208,33 @@ class BudgetGuard:
def check(self) -> None:
"""Raise ``BudgetExceeded`` if any hard ceiling has been crossed."""
- spent = self.spent.cost_usd
+ spent = self.billed_usd
elapsed = self.elapsed_minutes()
if spent > self.hard_usd:
raise BudgetExceeded(
f"cost ${spent:.3f} exceeds max_usd ${self.hard_usd:.2f}",
- spent_usd=spent, elapsed_min=elapsed,
+ spent_usd=spent,
+ elapsed_min=elapsed,
)
if elapsed > self.hard_minutes:
raise BudgetExceeded(
f"elapsed {elapsed:.1f} min exceeds max_minutes {self.hard_minutes:.1f}",
- spent_usd=spent, elapsed_min=elapsed,
+ spent_usd=spent,
+ elapsed_min=elapsed,
)
# ----------------------------------------------------------------- soft cap
def soft_limits(self) -> tuple[float, float]:
"""(usd, minutes) the soft sub-budget allows (grace is hard-only)."""
- return self.budget.max_usd * self.soft_fraction, self.budget.max_minutes * self.soft_fraction
+ return (
+ self.budget.max_usd * self.soft_fraction,
+ self.budget.max_minutes * self.soft_fraction,
+ )
def soft_exceeded(self) -> str:
"""Reason string when the soft sub-budget is used up, else ``""``."""
usd, minutes = self.soft_limits()
- spent, elapsed = self.spent.cost_usd, self.elapsed_minutes()
+ spent, elapsed = self.billed_usd, self.elapsed_minutes()
if spent > usd:
return f"cost ${spent:.3f} exceeds soft cap ${usd:.2f} ({self.soft_fraction:.0%} of ${self.budget.max_usd:.2f})"
if elapsed > minutes:
@@ -208,7 +248,10 @@ class BudgetGuard:
def soft_remaining(self) -> dict[str, float]:
"""Headroom left inside the soft sub-budget (never negative)."""
usd, minutes = self.soft_limits()
- return {"usd": max(0.0, usd - self.spent.cost_usd), "minutes": max(0.0, minutes - self.elapsed_minutes())}
+ return {
+ "usd": max(0.0, usd - self.billed_usd),
+ "minutes": max(0.0, minutes - self.elapsed_minutes()),
+ }
def ok(self) -> bool:
"""True when no ceiling is crossed (non-raising variant of ``check``)."""
@@ -220,7 +263,7 @@ class BudgetGuard:
def remaining(self) -> dict[str, float]:
"""Remaining headroom: ``{"usd": ..., "minutes": ..., "fraction": ...}``."""
- usd = max(0.0, self.hard_usd - self.spent.cost_usd)
+ usd = max(0.0, self.hard_usd - self.billed_usd)
minutes = max(0.0, self.hard_minutes - self.elapsed_minutes())
frac_usd = usd / self.hard_usd if self.hard_usd > 0 else 0.0
frac_min = minutes / self.hard_minutes if self.hard_minutes > 0 else 0.0
@@ -237,7 +280,10 @@ class BudgetGuard:
def summary(self) -> dict[str, float | int]:
return {
- "spent_usd": round(self.spent.cost_usd, 4),
+ "spent_usd": round(self.billed_usd, 4),
+ # what it WOULD have cost at list price; equal to spent_usd unless a
+ # subscription backend ran (cost/billing.py)
+ "notional_usd": round(self.spent.cost_usd, 4),
"elapsed_min": round(self.elapsed_minutes(), 2),
"soft_fraction": round(self.soft_fraction, 3),
"grace_usd": round(self.grace_usd, 4),
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index dfee7c3c..7f63725b 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -124,6 +124,8 @@ codeverse/
that does NOT meter itself; run_ledger nests + is context-local so bench --parallel works)
profiles.py (economy|balanced|quality; cli._common.resolve_dial is THE resolver)
caching.py (Block/order_blocks/session_cache — measurement only, docs/COST.md §13)
+ billing.py (SUBSCRIPTION_BACKENDS/bills_usd — which backends take real dollars,
+ so max_usd guards money and not list price; docs/COST.md §25)
guard.py routing.py reconstruct.py (old runs) audit.py report.py
judges/ rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,
prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.py
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 9e241d4d..95c25d25 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1142,3 +1142,49 @@ What did work, in production, on those two lost cells: they were recorded
`infra_failed` with `score=None` rather than a hard 0.0 (§7 of `docs/EVAL.md`), and the
retry deadline (§22) stopped each at ~15 min instead of the 56-87 min the same cells
burned earlier the same morning.
+
+## 25. `max_usd` guards money, and a subscription costs none
+
+`Usage.cost_usd` answers *"what would these tokens cost at list price?"*. That is the
+right number for a report, a $/complexity point, or a flywheel record — it is comparable
+across backends and independent of who is paying. It is the wrong number to hand a
+spend guard, because a backend on a flat-rate local subscription bills no dollars.
+
+The harness had exactly one number and used it for both.
+
+**Measured 2026-08-25**, `tsr_scn_temple_night`, `codex:gpt-5.6-sol`, `--profile
+quality`. The two Blender hero-asset sessions were priced at OpenAI list rates
+(`agents/codex.py` → `estimate_cost_safe("openai", …)`) and the run crossed the profile's
+soft cap 6.8 minutes in:
+
+```
+budget.degraded cost $7.712 exceeds soft cap $4.40 (55% of $8.00)
+asset.judge_skipped BronzeCenser reason=soft_budget
+asset.judge_skipped StoneLantern reason=soft_budget
+```
+
+Both heroes went unjudged and every later stage ran degraded — over a bill of **$0.00**.
+The run's own cost ledger said $0.00 the whole time; `PROVIDER_PRICED_BACKENDS` and
+`meters_own_calls()` had already got the ledger side right. Only the guard disagreed
+with it, and the guard is the half that changes what the run does.
+
+**The split.** The ledger and the reports keep pricing everything; the budget enforces
+only what is billed. `cost/billing.py` names the flat-rate backends
+(`SUBSCRIPTION_BACKENDS` = codex, claude-code, agy, antigravity — `CLAUDE.md`
+"Environment" is the source of that list), and `BudgetGuard` accumulates a second
+counter, `billed_usd`, which every ceiling now reads instead of `spent.cost_usd`.
+`summary()` reports both: `spent_usd` (billed, what the ceilings saw) and `notional_usd`
+(list price, what the reports want).
+
+`gemini-cli` is deliberately **not** exempt: it authenticates with an API key, so its
+tokens draw on a real per-token quota even when that quota is free. Unknown backends
+bill by default — a new provider nobody classified must be enforced, not exempted.
+Wrong in that direction costs a degraded run; wrong in the other spends real money with
+no ceiling.
+
+**What still bounds a subscription run:** `max_minutes`, which `BudgetGuard.timeout_s()`
+also clips individual sessions against. When the money is flat-rate, wall clock is the
+scarce resource — the runaway is still stopped, by the ceiling that actually applies to
+it. Note the corollary for benchmarking: an arm on a subscription backend and an arm on
+an API backend are not being held to the same ceiling, so compare them on
+`notional_usd`, never on `spent_usd`.
diff --git a/harness/tests/orchestrator_tracks/test_cost_accounting.py b/harness/tests/orchestrator_tracks/test_cost_accounting.py
index c969369a..7bcf8182 100644
--- a/harness/tests/orchestrator_tracks/test_cost_accounting.py
+++ b/harness/tests/orchestrator_tracks/test_cost_accounting.py
@@ -144,3 +144,46 @@ def test_the_ledger_is_optional_and_never_breaks_a_run(tmp_path):
quiet = BudgetGuard(Budget(max_usd=1.0, max_minutes=60)) # no ledger configured
quiet.charge(Usage(cost_usd=0.1))
assert quiet.spent.cost_usd == pytest.approx(0.1)
+
+
+# ------------------------------------------------- subscription backends vs the guard
+def test_a_subscription_backend_does_not_consume_the_spend_guard():
+ """``max_usd`` guards money, and a flat-rate CLI costs none.
+
+ Measured 2026-08-25, ``tsr_scn_temple_night`` (``codex:gpt-5.6-sol``,
+ ``--profile quality``): two Blender hero sessions priced at OpenAI list rates put the
+ run at $7.712 against the $4.40 soft cap in 6.8 minutes, so the asset judge was
+ skipped for BOTH heroes and every later stage ran degraded — over a bill of $0.00.
+ The run's own cost ledger said $0.00; only the guard disagreed.
+ """
+ from codeverse.contracts.common import Budget, Usage
+ from codeverse.orchestrator.budget import BudgetGuard
+
+ g = BudgetGuard(Budget(max_usd=8.0, max_minutes=60.0), soft_fraction=0.55)
+ g.spend(Usage(backend="codex", model="gpt-5.6-sol", cost_usd=7.712), enforce=False)
+
+ assert g.billed_usd == 0.0
+ assert g.soft_exceeded() == "" # the degradation that actually happened
+ assert g.ok()
+ # the notional price is NOT discarded — reports and $/complexity still want it
+ assert g.spent.cost_usd == pytest.approx(7.712)
+ assert g.summary()["notional_usd"] == pytest.approx(7.712)
+ assert g.summary()["spent_usd"] == 0.0
+
+
+def test_the_guard_still_enforces_backends_that_really_bill():
+ """The exemption must not become a hole: API backends keep both ceilings, and an
+ unclassified backend is enforced rather than exempted."""
+ from codeverse.contracts.common import Budget, Usage
+ from codeverse.orchestrator.budget import BudgetExceeded, BudgetGuard
+
+ g = BudgetGuard(Budget(max_usd=8.0, max_minutes=60.0), soft_fraction=0.55)
+ g.spend(Usage(backend="codex", cost_usd=99.0), enforce=False) # free, ignored
+ g.spend(Usage(backend="gemini", cost_usd=5.0), enforce=False)
+ assert g.soft_exceeded() # soft cap fires on the real $5
+ assert g.ok() # but the hard ceiling has not
+
+ g.spend(Usage(backend="some-new-provider", cost_usd=4.0), enforce=False)
+ assert g.billed_usd == pytest.approx(9.0), "an unknown backend must bill, not be exempt"
+ with pytest.raises(BudgetExceeded):
+ g.check()