Official Codebase of 3DCodeVerse
Python
budget: enforce max_usd on money that is billed, not on list price
gaoypeng9045d8a0ba56
diff --git a/harness/codeverse/cost/billing.py b/harness/codeverse/cost/billing.pynew file mode 100644index 00000000..bc02dc0f--- /dev/null+++ b/harness/codeverse/cost/billing.py@@ -0,0 +1,41 @@+"""Which backends actually take dollars out of an account.++``Usage.cost_usd`` answers "what would these tokens cost at list price?". That is the+right number for a report, a $/complexity-point, or a flywheel record: it is comparable+across backends and it does not depend on who is paying. It is the WRONG number for a+spend guard, because a backend on a local subscription bills no dollars at all.++Conflating the two degraded real runs. Measured 2026-08-25 on+``tsr_scn_temple_night`` (``codex:gpt-5.6-sol``, ``--profile quality``): two Blender hero+sessions priced at OpenAI list rates put the run at $7.712 against the profile's $4.40+soft cap in 6.8 minutes, so the asset judge was skipped for both heroes and every later+stage ran degraded — over a bill of exactly $0.00. The run's own cost ledger agreed it+was $0.00; only the guard disagreed.++So the split is: the LEDGER and the reports keep pricing everything, and the BUDGET+enforces only what is billed. See ``orchestrator/budget.py`` for the enforcement side+and ``docs/COST.md`` §25.++What a subscription backend is still bounded by: ``max_minutes``, which+``BudgetMeter.timeout_s`` also clips individual sessions against. Wall clock, not money,+is the scarce resource when the money is flat-rate — a runaway session is still stopped,+just by the ceiling that actually applies to it.+"""++from __future__ import annotations++#: Backends that run on a flat-rate local subscription on this machine, so their+#: ``cost_usd`` is notional (see ``CLAUDE.md`` "Environment": codex / claude / agy run on+#: local subscriptions). ``gemini-cli`` is deliberately NOT here: it authenticates with+#: an API key, so its tokens draw on a real per-token quota even when that quota is free.+SUBSCRIPTION_BACKENDS = frozenset({"codex", "claude-code", "agy", "antigravity"})+++def bills_usd(backend: str | None) -> bool:+ """True when this backend's ``cost_usd`` is money someone is actually charged.++ Unknown backends bill: a new API provider that nobody remembered to classify must be+ enforced, not exempted. Getting this wrong in the safe direction costs a degraded+ run; getting it wrong in the other direction spends real money with no ceiling.+ """+ return (backend or "").strip().lower() not in SUBSCRIPTION_BACKENDSdiff --git a/harness/codeverse/orchestrator/budget.py b/harness/codeverse/orchestrator/budget.pyindex 90733ef5..cc48956e 100644--- a/harness/codeverse/orchestrator/budget.py+++ b/harness/codeverse/orchestrator/budget.py@@ -33,6 +33,7 @@ from pathlib import Pathfrom typing import Anyfrom codeverse.contracts.common import Budget, Usage+from codeverse.cost.billing import bills_usdlog = logging.getLogger(__name__)@@ -57,11 +58,23 @@ class BudgetGuard:``BudgetExceeded`` when ``max_usd`` or ``max_minutes`` is exceeded."""- def __init__(self, budget: Budget, start_time: float | None = None, *, soft_fraction: float = 1.0,- run: str = "", ledger: str | Path | Any | None = None):+ def __init__(+ self,+ budget: Budget,+ start_time: float | None = None,+ *,+ soft_fraction: float = 1.0,+ run: str = "",+ ledger: str | Path | Any | None = None,+ ):self.budget = budgetself.start_time = time.time() if start_time is None else start_timeself.spent = Usage()+ #: the share of ``spent.cost_usd`` that is money someone is actually charged.+ #: The ceilings enforce THIS, not ``spent.cost_usd`` — a backend on a flat-rate+ #: subscription prices its tokens notionally and must not consume a spend guard.+ #: See ``cost/billing.py`` for why, and what still bounds a subscription run.+ self.billed_usd = 0.0self._lock = threading.Lock()self.calls = 0#: fraction of the hard ceilings the *baseline* may use (1.0 = no soft cap)@@ -98,13 +111,17 @@ class BudgetGuard:returnwith self._lock:self.spent = self.spent + usage+ if bills_usd(getattr(usage, "backend", None)):+ self.billed_usd += float(usage.cost_usd)self.calls += 1key = stage or OTHER_STAGEself.by_stage[key] = self.by_stage.get(key, 0.0) + float(usage.cost_usd)if round_index is not None:per = self.by_round.setdefault(int(round_index), {})per[key] = per.get(key, 0.0) + float(usage.cost_usd)- self._ledger_row(usage, stage=key, role=role, label=label, round_index=round_index, outcome=outcome)+ self._ledger_row(+ usage, stage=key, role=role, label=label, round_index=round_index, outcome=outcome+ )if enforce:self.check()@@ -125,8 +142,16 @@ class BudgetGuard:kw["enforce"] = Falseself.spend(usage, **kw)- def _ledger_row(self, usage: Usage, *, stage: str, role: str | None, label: str,- round_index: int | None, outcome: str) -> None:+ def _ledger_row(+ self,+ usage: Usage,+ *,+ stage: str,+ role: str | None,+ label: str,+ round_index: int | None,+ outcome: str,+ ) -> None:if self.ledger is None:returntry:@@ -137,8 +162,18 @@ class BudgetGuard:# tokens, cache hits and latency — one aggregate row on top would double count.if per_call_metering():return- record_call(usage, run=self.run, stage=stage, role=role, round=round_index, label=label,- backend=usage.backend, model=usage.model, outcome=outcome, ledger=self.ledger)+ record_call(+ usage,+ run=self.run,+ stage=stage,+ role=role,+ round=round_index,+ label=label,+ backend=usage.backend,+ model=usage.model,+ outcome=outcome,+ ledger=self.ledger,+ )except Exception as e: # noqa: BLE001 — accounting must never fail a runlog.debug("cost ledger row failed (%s): %s", self.ledger, e)@@ -173,28 +208,33 @@ class BudgetGuard:def check(self) -> None:"""Raise ``BudgetExceeded`` if any hard ceiling has been crossed."""- spent = self.spent.cost_usd+ spent = self.billed_usdelapsed = self.elapsed_minutes()if spent > self.hard_usd:raise BudgetExceeded(f"cost ${spent:.3f} exceeds max_usd ${self.hard_usd:.2f}",- spent_usd=spent, elapsed_min=elapsed,+ spent_usd=spent,+ elapsed_min=elapsed,)if elapsed > self.hard_minutes:raise BudgetExceeded(f"elapsed {elapsed:.1f} min exceeds max_minutes {self.hard_minutes:.1f}",- spent_usd=spent, elapsed_min=elapsed,+ spent_usd=spent,+ elapsed_min=elapsed,)# ----------------------------------------------------------------- soft capdef soft_limits(self) -> tuple[float, float]:"""(usd, minutes) the soft sub-budget allows (grace is hard-only)."""- return self.budget.max_usd * self.soft_fraction, self.budget.max_minutes * self.soft_fraction+ return (+ self.budget.max_usd * self.soft_fraction,+ self.budget.max_minutes * self.soft_fraction,+ )def soft_exceeded(self) -> str:"""Reason string when the soft sub-budget is used up, else ``""``."""usd, minutes = self.soft_limits()- spent, elapsed = self.spent.cost_usd, self.elapsed_minutes()+ spent, elapsed = self.billed_usd, self.elapsed_minutes()if spent > usd:return f"cost ${spent:.3f} exceeds soft cap ${usd:.2f} ({self.soft_fraction:.0%} of ${self.budget.max_usd:.2f})"if elapsed > minutes:@@ -208,7 +248,10 @@ class BudgetGuard:def soft_remaining(self) -> dict[str, float]:"""Headroom left inside the soft sub-budget (never negative)."""usd, minutes = self.soft_limits()- return {"usd": max(0.0, usd - self.spent.cost_usd), "minutes": max(0.0, minutes - self.elapsed_minutes())}+ return {+ "usd": max(0.0, usd - self.billed_usd),+ "minutes": max(0.0, minutes - self.elapsed_minutes()),+ }def ok(self) -> bool:"""True when no ceiling is crossed (non-raising variant of ``check``)."""@@ -220,7 +263,7 @@ class BudgetGuard:def remaining(self) -> dict[str, float]:"""Remaining headroom: ``{"usd": ..., "minutes": ..., "fraction": ...}``."""- usd = max(0.0, self.hard_usd - self.spent.cost_usd)+ usd = max(0.0, self.hard_usd - self.billed_usd)minutes = max(0.0, self.hard_minutes - self.elapsed_minutes())frac_usd = usd / self.hard_usd if self.hard_usd > 0 else 0.0frac_min = minutes / self.hard_minutes if self.hard_minutes > 0 else 0.0@@ -237,7 +280,10 @@ class BudgetGuard:def summary(self) -> dict[str, float | int]:return {- "spent_usd": round(self.spent.cost_usd, 4),+ "spent_usd": round(self.billed_usd, 4),+ # what it WOULD have cost at list price; equal to spent_usd unless a+ # subscription backend ran (cost/billing.py)+ "notional_usd": round(self.spent.cost_usd, 4),"elapsed_min": round(self.elapsed_minutes(), 2),"soft_fraction": round(self.soft_fraction, 3),"grace_usd": round(self.grace_usd, 4),diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex dfee7c3c..7f63725b 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -124,6 +124,8 @@ codeverse/that does NOT meter itself; run_ledger nests + is context-local so bench --parallel works)profiles.py (economy|balanced|quality; cli._common.resolve_dial is THE resolver)caching.py (Block/order_blocks/session_cache — measurement only, docs/COST.md §13)+ billing.py (SUBSCRIPTION_BACKENDS/bills_usd — which backends take real dollars,+ so max_usd guards money and not list price; docs/COST.md §25)guard.py routing.py reconstruct.py (old runs) audit.py report.pyjudges/ rubrics.py + rubrics/*.yaml (defect checklists), vlm_judge.py, montage.py,prompt_builder.py, output_schema.py, scoring.py, caps.py, images.py, pairwise.pydiff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 9e241d4d..95c25d25 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1142,3 +1142,49 @@ What did work, in production, on those two lost cells: they were recorded`infra_failed` with `score=None` rather than a hard 0.0 (§7 of `docs/EVAL.md`), and theretry deadline (§22) stopped each at ~15 min instead of the 56-87 min the same cellsburned earlier the same morning.++## 25. `max_usd` guards money, and a subscription costs none++`Usage.cost_usd` answers *"what would these tokens cost at list price?"*. That is the+right number for a report, a $/complexity point, or a flywheel record — it is comparable+across backends and independent of who is paying. It is the wrong number to hand a+spend guard, because a backend on a flat-rate local subscription bills no dollars.++The harness had exactly one number and used it for both.++**Measured 2026-08-25**, `tsr_scn_temple_night`, `codex:gpt-5.6-sol`, `--profile+quality`. The two Blender hero-asset sessions were priced at OpenAI list rates+(`agents/codex.py` → `estimate_cost_safe("openai", …)`) and the run crossed the profile's+soft cap 6.8 minutes in:++```+budget.degraded cost $7.712 exceeds soft cap $4.40 (55% of $8.00)+asset.judge_skipped BronzeCenser reason=soft_budget+asset.judge_skipped StoneLantern reason=soft_budget+```++Both heroes went unjudged and every later stage ran degraded — over a bill of **$0.00**.+The run's own cost ledger said $0.00 the whole time; `PROVIDER_PRICED_BACKENDS` and+`meters_own_calls()` had already got the ledger side right. Only the guard disagreed+with it, and the guard is the half that changes what the run does.++**The split.** The ledger and the reports keep pricing everything; the budget enforces+only what is billed. `cost/billing.py` names the flat-rate backends+(`SUBSCRIPTION_BACKENDS` = codex, claude-code, agy, antigravity — `CLAUDE.md`+"Environment" is the source of that list), and `BudgetGuard` accumulates a second+counter, `billed_usd`, which every ceiling now reads instead of `spent.cost_usd`.+`summary()` reports both: `spent_usd` (billed, what the ceilings saw) and `notional_usd`+(list price, what the reports want).++`gemini-cli` is deliberately **not** exempt: it authenticates with an API key, so its+tokens draw on a real per-token quota even when that quota is free. Unknown backends+bill by default — a new provider nobody classified must be enforced, not exempted.+Wrong in that direction costs a degraded run; wrong in the other spends real money with+no ceiling.++**What still bounds a subscription run:** `max_minutes`, which `BudgetGuard.timeout_s()`+also clips individual sessions against. When the money is flat-rate, wall clock is the+scarce resource — the runaway is still stopped, by the ceiling that actually applies to+it. Note the corollary for benchmarking: an arm on a subscription backend and an arm on+an API backend are not being held to the same ceiling, so compare them on+`notional_usd`, never on `spent_usd`.diff --git a/harness/tests/orchestrator_tracks/test_cost_accounting.py b/harness/tests/orchestrator_tracks/test_cost_accounting.pyindex c969369a..7bcf8182 100644--- a/harness/tests/orchestrator_tracks/test_cost_accounting.py+++ b/harness/tests/orchestrator_tracks/test_cost_accounting.py@@ -144,3 +144,46 @@ def test_the_ledger_is_optional_and_never_breaks_a_run(tmp_path):quiet = BudgetGuard(Budget(max_usd=1.0, max_minutes=60)) # no ledger configuredquiet.charge(Usage(cost_usd=0.1))assert quiet.spent.cost_usd == pytest.approx(0.1)+++# ------------------------------------------------- subscription backends vs the guard+def test_a_subscription_backend_does_not_consume_the_spend_guard():+ """``max_usd`` guards money, and a flat-rate CLI costs none.++ Measured 2026-08-25, ``tsr_scn_temple_night`` (``codex:gpt-5.6-sol``,+ ``--profile quality``): two Blender hero sessions priced at OpenAI list rates put the+ run at $7.712 against the $4.40 soft cap in 6.8 minutes, so the asset judge was+ skipped for BOTH heroes and every later stage ran degraded — over a bill of $0.00.+ The run's own cost ledger said $0.00; only the guard disagreed.+ """+ from codeverse.contracts.common import Budget, Usage+ from codeverse.orchestrator.budget import BudgetGuard++ g = BudgetGuard(Budget(max_usd=8.0, max_minutes=60.0), soft_fraction=0.55)+ g.spend(Usage(backend="codex", model="gpt-5.6-sol", cost_usd=7.712), enforce=False)++ assert g.billed_usd == 0.0+ assert g.soft_exceeded() == "" # the degradation that actually happened+ assert g.ok()+ # the notional price is NOT discarded — reports and $/complexity still want it+ assert g.spent.cost_usd == pytest.approx(7.712)+ assert g.summary()["notional_usd"] == pytest.approx(7.712)+ assert g.summary()["spent_usd"] == 0.0+++def test_the_guard_still_enforces_backends_that_really_bill():+ """The exemption must not become a hole: API backends keep both ceilings, and an+ unclassified backend is enforced rather than exempted."""+ from codeverse.contracts.common import Budget, Usage+ from codeverse.orchestrator.budget import BudgetExceeded, BudgetGuard++ g = BudgetGuard(Budget(max_usd=8.0, max_minutes=60.0), soft_fraction=0.55)+ g.spend(Usage(backend="codex", cost_usd=99.0), enforce=False) # free, ignored+ g.spend(Usage(backend="gemini", cost_usd=5.0), enforce=False)+ assert g.soft_exceeded() # soft cap fires on the real $5+ assert g.ok() # but the hard ceiling has not++ g.spend(Usage(backend="some-new-provider", cost_usd=4.0), enforce=False)+ assert g.billed_usd == pytest.approx(9.0), "an unknown backend must bill, not be exempt"+ with pytest.raises(BudgetExceeded):+ g.check()