Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/cost): mark estimated-basis budget records and add an enforcement policy (#2713) (#2725)
Parideboy01df2452529a
diff --git a/docs/content/docs/metrics.mdx b/docs/content/docs/metrics.mdxindex 1439c6861..6c8b1412f 100644--- a/docs/content/docs/metrics.mdx+++ b/docs/content/docs/metrics.mdx@@ -252,6 +252,44 @@ headroom proxy --budget 10.00When the budget is exceeded, requests return a budget exceeded error, the `/stats` endpoint shows budget status, and logs indicate the budget state.+### Measured vs Estimated Spend++Every cost record carries a *basis* — where its input-token count came from. When a provider response includes a usage breakdown, the basis is `measured`. When it doesn't, Headroom substitutes its own `tokens_sent` count so input cost isn't dropped from the budget, and the record's basis is `estimated`. Headroom logs one warning per model the first time this happens.++`/stats` keeps the two separable under `cost.budget_basis`:++```json+{+ "cost": {+ "budget_limit_usd": 10.0,+ "budget_period": "daily",+ "budget_estimated_basis": "count",+ "budget_basis": {+ "total_usd": 3.1400,+ "measured_usd": 2.9000,+ "estimated_usd": 0.2400,+ "estimated_pct": 7.6,+ "records": 412,+ "estimated_records": 31+ }+ }+}+```++An estimate can drift in either direction, so you choose what it does to the hard limit:++```bash+headroom proxy --budget 10.00 --budget-estimated-basis count # default+```++| Value | Effect |+|-------|--------|+| `count` | Estimated spend consumes the budget like measured spend. The default; matches historical behavior. |+| `ignore` | Estimated spend is still booked and reported, but only provider-reported spend consumes the budget. |+| `block` | Refuse requests once the period holds any estimated spend, rather than enforcing a hard limit against a guess. |++Env: `HEADROOM_BUDGET_ESTIMATED_BASIS`. `headroom doctor` reports the estimated share alongside the budget check.+## Key Metrics to Monitor| Metric | What It Tells You | Target |diff --git a/headroom/cli/doctor.py b/headroom/cli/doctor.pyindex 5a0a032b4..f1f06b380 100644--- a/headroom/cli/doctor.py+++ b/headroom/cli/doctor.py@@ -426,7 +426,36 @@ def check_budget(stats: dict[str, Any] | None) -> CheckResult:hint="set one: headroom proxy --budget 10 (env: HEADROOM_BUDGET)",)period = cost.get("budget_period", "daily")- return CheckResult(name=name, status=PASS, summary=f"${limit}/{period} budget enforced")+ summary = f"${limit}/{period} budget enforced"+ return CheckResult(name=name, status=PASS, summary=summary + _estimated_basis_note(cost))+++def _estimated_basis_note(cost: dict[str, Any]) -> str:+ """Describe how much of the period's spend was booked from a token estimate.++ Informational, never a WARN: a provider that simply never reports a usage+ breakdown would otherwise sit at a permanent warning. Every read is+ defensive so `doctor` still works against a proxy predating these fields.+ """+ note = ""++ basis = cost.get("budget_basis")+ if isinstance(basis, dict):+ estimated_usd = basis.get("estimated_usd")+ estimated_pct = basis.get("estimated_pct")+ if isinstance(estimated_usd, (int, float)) and estimated_usd > 0:+ pct = f"{estimated_pct:.0f}% " if isinstance(estimated_pct, (int, float)) else ""+ note += (+ f" — {pct}of period spend (${estimated_usd:.4f}) "+ "booked from Headroom token estimates"+ )++ # Reported independently of the breakdown: a non-default policy changes how+ # the budget is enforced and should surface even if the split is missing.+ policy = cost.get("budget_estimated_basis")+ if isinstance(policy, str) and policy and policy != "count":+ note += f" — estimated-basis policy: {policy}"+ return notedef check_deployments(manifests: list[Any], probe: Any = probe_json) -> CheckResult | None:diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 1a4dbe7ee..30292a4e1 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -534,6 +534,20 @@ def dashboard(port: int, no_open: bool) -> None:"Env: HEADROOM_BUDGET_PERIOD."),)+ "--budget-estimated-basis",+ type=click.Choice(["count", "ignore", "block"]),+ default="count",+ envvar="HEADROOM_BUDGET_ESTIMATED_BASIS",+ help=(+ "What to do with spend booked from Headroom's own token estimate, which "+ "happens when a provider response carries no input-token breakdown. "+ "count: it consumes the budget like measured spend (default). "+ "ignore: only provider-reported spend consumes the budget. "+ "block: refuse requests rather than enforce a hard limit on an estimate. "+ "Env: HEADROOM_BUDGET_ESTIMATED_BASIS."+ ),+)# Code-aware compression (AST-based, requires `pip install headroom-ai[code]`).# Pair of flags so users can override the env-var default in either direction.# We resolve HEADROOM_CODE_AWARE_ENABLED in the body (not via Click's envvar=),@@ -941,6 +955,7 @@ def proxy(codex_wire_debug_dir: str | None,budget: float | None,budget_period: str,+ budget_estimated_basis: str,code_aware_flag: bool | None,disable_kompress: bool,disable_kompress_fallback: bool,@@ -1235,6 +1250,7 @@ def proxy(or os.environ.get("HEADROOM_LOG_MESSAGES", "").lower() in ("true", "1", "yes", "on"),budget_limit_usd=budget,budget_period=cast(Literal["hourly", "daily", "monthly"], budget_period),+ budget_estimated_basis=cast(Literal["count", "ignore", "block"], budget_estimated_basis),# Code-aware compression resolution:# 1. Explicit --code-aware / --no-code-aware always wins.# 2. Otherwise read HEADROOM_CODE_AWARE_ENABLED (truthy = on).diff --git a/headroom/proxy/budget_basis_policy.py b/headroom/proxy/budget_basis_policy.pynew file mode 100644index 000000000..1226755ec--- /dev/null+++ b/headroom/proxy/budget_basis_policy.py@@ -0,0 +1,81 @@+"""Policy for how estimate-based cost records feed budget enforcement.++Every cost record the proxy books carries a *basis*: where the input-token+count came from. ``measured`` means the provider reported a usage breakdown;+``estimated`` means the response carried none and Headroom substituted its own+``tokens_sent`` count so input cost wasn't silently dropped from the budget+(#2713).++The substitution is the right default — dropping input cost entirely would+under-enforce far worse — but a budget is a hard control, and an estimate can+drift in either direction. This module names the two bases and the three+policies an operator can pick for what estimated spend does to the budget:++* ``count`` — estimated spend counts toward the limit (the historical+ behavior, and still the default; nothing changes except that the record is+ now marked, logged, and separable in ``/stats``).+* ``ignore`` — the record is still booked to the ledger and reported, but only+ provider-measured spend consumes the budget. Fails open on the estimate.+* ``block`` — fail closed: once the period holds any estimated spend, refuse+ rather than enforce a hard limit against a guess.+"""++from __future__ import annotations++import logging+from collections.abc import Mapping++logger = logging.getLogger("headroom.proxy")++# Provenance of the input-token count behind a booked cost record.+COST_BASIS_MEASURED = "measured"+COST_BASIS_ESTIMATED = "estimated"++# What estimated spend does to budget enforcement.+BUDGET_BASIS_COUNT = "count"+BUDGET_BASIS_IGNORE = "ignore"+BUDGET_BASIS_BLOCK = "block"++VALID_POLICIES = (BUDGET_BASIS_COUNT, BUDGET_BASIS_IGNORE, BUDGET_BASIS_BLOCK)+DEFAULT_POLICY = BUDGET_BASIS_COUNT++ENV_VAR = "HEADROOM_BUDGET_ESTIMATED_BASIS"++# A typo'd knob must not fail proxy startup, but it also must not warn on every+# resolve. Bounded by the number of distinct bad values seen (realistically 1).+_warned_invalid_policies: set[str] = set()+++def resolve_estimated_basis_policy(+ configured: str | None = None,+ env: Mapping[str, str] | None = None,+) -> str:+ """Resolve the estimated-basis budget policy.++ Precedence: explicit ``configured`` value, then ``env[ENV_VAR]``, then+ :data:`DEFAULT_POLICY`. An unrecognized value falls back to the default and+ warns once — a misconfigured knob should never stop the proxy from booting,+ but it must not silently change enforcement either.+ """+ raw = configured+ if raw is None and env is not None:+ raw = env.get(ENV_VAR)+ if raw is None:+ return DEFAULT_POLICY++ candidate = raw.strip().lower()+ if candidate in VALID_POLICIES:+ return candidate+ if not candidate:+ return DEFAULT_POLICY++ if candidate not in _warned_invalid_policies:+ _warned_invalid_policies.add(candidate)+ logger.warning(+ "Unknown %s=%r — falling back to %r (valid: %s)",+ ENV_VAR,+ raw,+ DEFAULT_POLICY,+ ", ".join(VALID_POLICIES),+ )+ return DEFAULT_POLICYdiff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex f2142023f..067af7dcd 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -13,8 +13,17 @@ import loggingimport mathfrom collections import dequefrom datetime import datetime, timedelta-from typing import TYPE_CHECKING, Any-+from typing import TYPE_CHECKING, Any, NamedTuple++from headroom.proxy.budget_basis_policy import (+ BUDGET_BASIS_BLOCK,+ BUDGET_BASIS_IGNORE,+ COST_BASIS_ESTIMATED,+ COST_BASIS_MEASURED,+ DEFAULT_POLICY,+ ENV_VAR,+ resolve_estimated_basis_policy,+)from headroom.proxy.modes import PROXY_MODE_CACHEif TYPE_CHECKING:@@ -60,6 +69,40 @@ def _warn_pricing_once(model: str, message: str) -> None:logger.warning(message)+# A route whose responses never carry a usage breakdown hits the estimated-basis+# fallback on *every* request, so the warning is deduped per model for the same+# reason pricing warnings are (#2504): one line per model per process, not one+# per request. The distinction stays permanently visible in /stats regardless.+_warned_estimated_basis_models: set[str] = set()+++def _warn_estimated_basis_once(model: str) -> None:+ """Warn the first time ``model`` books a cost against Headroom's estimate."""+ if model in _warned_estimated_basis_models:+ return+ _warned_estimated_basis_models.add(model)+ logger.warning(+ "budget basis estimated: no usage breakdown from provider for %s — "+ "input cost booked from Headroom's own token count",+ model,+ )+++class CostEntry(NamedTuple):+ """One booked cost, with the provenance of the input count behind it.++ ``basis`` is :data:`~headroom.proxy.budget_basis_policy.COST_BASIS_MEASURED`+ when the provider reported a usage breakdown, and ``COST_BASIS_ESTIMATED``+ when it didn't and Headroom's own ``tokens_sent`` stood in for the input+ count. Budget enforcement is a hard control, so the two must stay separable+ in the ledger rather than collapsing into an undifferentiated dollar figure.+ """++ timestamp: datetime+ cost_usd: float+ basis: str++# Provider-specific cache discount multipliers (what fraction of input price)# Used to calculate dollar savings from prefix caching_CACHE_ECONOMICS = {@@ -644,12 +687,21 @@ class CostTracker:# get_period_cost() undercounts and check_budget() silently under-enforces.COST_RETENTION_HOURS = 744 # 31 days- def __init__(self, budget_limit_usd: float | None = None, budget_period: str = "daily"):+ def __init__(+ self,+ budget_limit_usd: float | None = None,+ budget_period: str = "daily",+ estimated_basis_policy: str = DEFAULT_POLICY,+ ):self.budget_limit_usd = budget_limit_usdself.budget_period = budget_period+ # What estimated-basis spend does to enforcement. Normalized here so a+ # bad value degrades to the default instead of quietly disabling the+ # budget. See headroom.proxy.budget_basis_policy.+ self.estimated_basis_policy = resolve_estimated_basis_policy(estimated_basis_policy)# Cost tracking - using deque for efficient left-side removal- self._costs: deque[tuple[datetime, float]] = deque(maxlen=self.MAX_COST_ENTRIES)+ self._costs: deque[CostEntry] = deque(maxlen=self.MAX_COST_ENTRIES)self._last_prune_time: datetime = datetime.now()# Token savings per model (exact, no dollar estimation)@@ -743,7 +795,7 @@ class CostTracker:cutoff = now - timedelta(hours=self.COST_RETENTION_HOURS)# Remove entries from the left (oldest) while they're older than cutoff- while self._costs and self._costs[0][0] < cutoff:+ while self._costs and self._costs[0].timestamp < cutoff:self._costs.popleft()def record_tokens(@@ -807,9 +859,17 @@ class CostTracker:# When the call site had no API usage breakdown (all cache/uncached# fields are 0), fall back to tokens_sent so input cost isn't# silently dropped from the budget.+ #+ # That fallback is a guess, and check_budget() is a hard control, so the+ # record is stamped ``estimated`` and warned about once per model (#2713).+ # The fallback behaviour itself is unchanged — the estimate is now+ # labelled rather than indistinguishable from provider-reported usage.+ basis = COST_BASIS_MEASUREDinput_tokens = uncached_tokensif not (uncached_tokens or cache_read_tokens or cache_write_tokens):input_tokens = tokens_sent+ basis = COST_BASIS_ESTIMATED+ _warn_estimated_basis_once(model)cost = self.estimate_cost(model=model,input_tokens=input_tokens,@@ -818,31 +878,121 @@ class CostTracker:cache_write_tokens=cache_write_tokens,)if cost is not None:- self._costs.append((datetime.now(), cost))+ self._costs.append(CostEntry(datetime.now(), cost, basis))self._prune_old_costs()- def get_period_cost(self) -> float:- """Get cost for current budget period."""+ def _period_cutoff(self) -> datetime:+ """Start of the current budget period."""now = datetime.now()if self.budget_period == "hourly":- cutoff = now - timedelta(hours=1)- elif self.budget_period == "daily":- cutoff = now.replace(hour=0, minute=0, second=0, microsecond=0)- else: # monthly- cutoff = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)+ return now - timedelta(hours=1)+ if self.budget_period == "daily":+ return now.replace(hour=0, minute=0, second=0, microsecond=0)+ # monthly+ return now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)++ def get_period_cost(self, basis: str | None = None) -> float:+ """Get cost for current budget period.++ With no argument this is the total spend booked in the period,+ regardless of how each record's input count was derived. Pass a basis+ (``"measured"`` / ``"estimated"``) to get just that slice.+ """+ cutoff = self._period_cutoff()+ return sum(+ entry.cost_usd+ for entry in self._costs+ if entry.timestamp >= cutoff and (basis is None or entry.basis == basis)+ )- return sum(cost for ts, cost in self._costs if ts >= cutoff)+ def period_cost_breakdown(self) -> dict[str, Any]:+ """Split the period's booked spend by how its input count was derived.++ ``estimated_usd`` is spend whose input token count came from Headroom's+ own estimate because the provider returned no usage breakdown. Keeping+ it separable is the point: a budget refusal driven by a guess should be+ distinguishable from one driven by provider-reported usage (#2713).+ """+ cutoff = self._period_cutoff()+ measured_usd = 0.0+ estimated_usd = 0.0+ records = 0+ estimated_records = 0+ for entry in self._costs:+ if entry.timestamp < cutoff:+ continue+ records += 1+ if entry.basis == COST_BASIS_ESTIMATED:+ estimated_usd += entry.cost_usd+ estimated_records += 1+ else:+ measured_usd += entry.cost_usd++ total_usd = measured_usd + estimated_usd+ return {+ "period": self.budget_period,+ "policy": self.estimated_basis_policy,+ "total_usd": total_usd,+ "measured_usd": measured_usd,+ "estimated_usd": estimated_usd,+ "estimated_pct": round(estimated_usd / total_usd * 100, 1) if total_usd > 0 else 0.0,+ "records": records,+ "estimated_records": estimated_records,+ }def check_budget(self) -> tuple[bool, float]:- """Check if within budget. Returns (allowed, remaining)."""+ """Check if within budget. Returns (allowed, remaining).++ How estimated-basis spend participates is governed by+ ``estimated_basis_policy``: ``count`` (default) enforces against total+ spend exactly as before, ``ignore`` enforces against provider-measured+ spend only, and ``block`` refuses outright once the period holds any+ estimated spend rather than enforcing a hard limit against a guess.+ """if self.budget_limit_usd is None:return True, float("inf")- period_cost = self.get_period_cost()+ breakdown = self.period_cost_breakdown()++ if self.estimated_basis_policy == BUDGET_BASIS_BLOCK and breakdown["estimated_usd"] > 0:+ return False, 0.0++ if self.estimated_basis_policy == BUDGET_BASIS_IGNORE:+ period_cost = breakdown["measured_usd"]+ else:+ period_cost = breakdown["total_usd"]+remaining = self.budget_limit_usd - period_costreturn remaining > 0, max(0, remaining)+ def budget_denial_detail(self) -> str:+ """Human-readable reason a request was refused on budget grounds.++ Built here rather than in the handler so the message can name what the+ ledger actually knows — specifically how much of the period's spend was+ booked from Headroom's own token estimate.+ """+ breakdown = self.period_cost_breakdown()+ estimated_usd = breakdown["estimated_usd"]++ if self.estimated_basis_policy == BUDGET_BASIS_BLOCK and estimated_usd > 0:+ return (+ f"Budget enforcement blocked for {self.budget_period} period: "+ f"${estimated_usd:.4f} of ${breakdown['total_usd']:.4f} was booked from "+ "Headroom's own token estimate because the provider returned no usage "+ f"breakdown, and {ENV_VAR}=block refuses to enforce a budget on an "+ "estimate. Set it to 'count' or 'ignore' to serve these requests."+ )++ detail = f"Budget exceeded for {self.budget_period} period"+ if estimated_usd > 0:+ detail += (+ f" (${estimated_usd:.4f} of ${breakdown['total_usd']:.4f} booked from "+ f"Headroom token estimates, not provider-reported usage)"+ )+ return detail+def _get_list_price(self, model: str) -> float | None:"""Get list input price per 1M tokens for a model."""litellm = _get_litellm_module()@@ -956,4 +1106,8 @@ class CostTracker:# `headroom doctor` can report whether a budget is set."budget_limit_usd": self.budget_limit_usd,"budget_period": self.budget_period,+ "budget_estimated_basis": self.estimated_basis_policy,+ # Period spend split by input-count provenance, so estimate-derived+ # spend stays separable from provider-reported spend (#2713).+ "budget_basis": self.period_cost_breakdown(),}diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex e255ab886..cafd72edd 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -868,7 +868,7 @@ class AnthropicHandlerMixin:await _finalize_pre_upstream()raise HTTPException(status_code=429,- detail=f"Budget exceeded for {self.config.budget_period} period",+ detail=self.cost_tracker.budget_denial_detail(),)# Memory: Get user ID when memory is enabled (fallback to "default" for simple DevEx).diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 8a5651fe9..069867e2b 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -319,6 +319,9 @@ class ProxyConfig:cost_tracking_enabled: bool = Truebudget_limit_usd: float | None = Nonebudget_period: Literal["hourly", "daily", "monthly"] = "daily"+ # What spend booked from Headroom's own token estimate (provider returned no+ # usage breakdown) does to budget enforcement. See budget_basis_policy.+ budget_estimated_basis: Literal["count", "ignore", "block"] = "count"# Logginglog_requests: bool = Truediff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex a6dff74f3..04ea3cb5e 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -113,6 +113,7 @@ from headroom.proxy import runtime_envfrom headroom.proxy.audit import is_auditable_path, record_admin_actionfrom headroom.proxy.auth_mode import should_stamp_codex_clientfrom headroom.proxy.background_compression import BackgroundCompressor+from headroom.proxy.budget_basis_policy import resolve_estimated_basis_policy# =============================================================================# Extracted modules (re-exported for backward compatibility)@@ -797,6 +798,7 @@ class HeadroomProxy(CostTracker(budget_limit_usd=config.budget_limit_usd,budget_period=config.budget_period,+ estimated_basis_policy=config.budget_estimated_basis,)if config.cost_tracking_enabledelse None@@ -5472,6 +5474,17 @@ if __name__ == "__main__":# Costparser.add_argument("--budget", type=float, help="Budget limit in USD")parser.add_argument("--budget-period", choices=["hourly", "daily", "monthly"], default="daily")+ parser.add_argument(+ "--budget-estimated-basis",+ choices=["count", "ignore", "block"],+ default=None,+ help=(+ "What spend booked from Headroom's own token estimate (provider returned no "+ "usage breakdown) does to the budget: count it (default), ignore it, or block "+ "requests rather than enforce the limit on an estimate. "+ "Env: HEADROOM_BUDGET_ESTIMATED_BASIS."+ ),+ )# Loggingparser.add_argument("--log-file", help="Log file path")@@ -5574,6 +5587,10 @@ if __name__ == "__main__":rate_limit_tokens_per_minute=_get_env_int("HEADROOM_TPM", args.tpm),budget_limit_usd=args.budget,budget_period=args.budget_period,+ budget_estimated_basis=cast(+ Literal["count", "ignore", "block"],+ resolve_estimated_basis_policy(args.budget_estimated_basis, os.environ),+ ),log_file=_get_env_str("HEADROOM_LOG_FILE", args.log_file)if args.log_fileelse os.environ.get("HEADROOM_LOG_FILE"),diff --git a/headroom/settings_store.py b/headroom/settings_store.pyindex 6a8dff200..c085c181f 100644--- a/headroom/settings_store.py+++ b/headroom/settings_store.py@@ -204,6 +204,22 @@ SETTINGS: tuple[SettingField, ...] = (help="Period the budget applies to.",tier="basic",),+ SettingField(+ "HEADROOM_BUDGET_ESTIMATED_BASIS",+ "budget_estimated_basis",+ "Estimated-basis spend",+ "Budget",+ "enum",+ default="count",+ choices=("count", "ignore", "block"),+ help=(+ "What spend booked from Headroom's own token estimate does to the budget "+ "when a provider response carries no input-token breakdown. count: it "+ "consumes the budget. ignore: only provider-reported spend does. block: "+ "refuse requests rather than enforce a hard limit on an estimate."+ ),+ tier="advanced",+ ),# --- Networking (baked into the install manifest on supervised deploys) ---SettingField("HEADROOM_HOST",diff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.pyindex 3595fc429..777ee7c6a 100644--- a/tests/test_anthropic_pre_upstream_backpressure.py+++ b/tests/test_anthropic_pre_upstream_backpressure.py@@ -852,6 +852,9 @@ class _CostTrackerBlock:def check_budget(self):return False, 0+ def budget_denial_detail(self):+ return "Budget exceeded for daily period"+def record_tokens(self, *a, **k):return Nonediff --git a/tests/test_cli_doctor.py b/tests/test_cli_doctor.pyindex a8cebba3f..6bce283e8 100644--- a/tests/test_cli_doctor.py+++ b/tests/test_cli_doctor.py@@ -444,6 +444,62 @@ class TestBudget:assert result.status == PASSassert "$10.0/daily" in result.summary+ def test_estimated_basis_share_is_reported_without_warning(self):+ """#2713: spend booked from a token estimate is surfaced, not warned on.++ A provider that never reports a usage breakdown would otherwise sit at a+ permanent WARN, so this stays informational.+ """+ result = check_budget(+ {+ "cost": {+ "budget_limit_usd": 10.0,+ "budget_period": "daily",+ "budget_estimated_basis": "count",+ "budget_basis": {"estimated_usd": 1.24, "estimated_pct": 62.3},+ }+ }+ )+ assert result.status == PASS+ assert "62% of period spend ($1.2400)" in result.summary+ assert "Headroom token estimates" in result.summary++ def test_all_measured_spend_adds_no_note(self):+ result = check_budget(+ {+ "cost": {+ "budget_limit_usd": 10.0,+ "budget_period": "daily",+ "budget_estimated_basis": "count",+ "budget_basis": {"estimated_usd": 0.0, "estimated_pct": 0.0},+ }+ }+ )+ assert result.summary == "$10.0/daily budget enforced"++ def test_non_default_basis_policy_is_named(self):+ result = check_budget(+ {+ "cost": {+ "budget_limit_usd": 10.0,+ "budget_period": "daily",+ "budget_estimated_basis": "block",+ }+ }+ )+ assert "estimated-basis policy: block" in result.summary++ def test_missing_basis_fields_degrade_quietly(self):+ """`doctor` must still work against a proxy predating these fields."""+ result = check_budget({"cost": {"budget_limit_usd": 10.0, "budget_period": "daily"}})+ assert result.status == PASS+ assert result.summary == "$10.0/daily budget enforced"++ malformed = check_budget(+ {"cost": {"budget_limit_usd": 10.0, "budget_period": "daily", "budget_basis": "nope"}}+ )+ assert malformed.status == PASS+@dataclassclass _FakeManifest:diff --git a/tests/test_cost_budget_basis.py b/tests/test_cost_budget_basis.pynew file mode 100644index 000000000..cdb7ad1d6--- /dev/null+++ b/tests/test_cost_budget_basis.py@@ -0,0 +1,306 @@+"""Budget records must say whether their input count was measured or estimated.++#2713: when a provider response carries no input-token breakdown,+``record_tokens`` substitutes Headroom's own ``tokens_sent`` for the input+count. The fallback is right — dropping input cost would under-enforce far+worse — but the resulting record used to be indistinguishable from a+provider-measured one, so ``check_budget`` (a hard control) could refuse or+allow on the strength of a guess with nothing saying so.++These tests pin the marking, the deduped warning, the separable ledger, and+the three operator policies.+"""++from __future__ import annotations++import logging++import pytest++from tests._dotenv import (+ autouse_apply_env,+ importorskip_no_env_leak,+ load_env_overrides,+)++_env_overrides = load_env_overrides()+apply_dotenv = autouse_apply_env(_env_overrides)++importorskip_no_env_leak("litellm")++MODEL = "claude-sonnet-4-20250514"++[email protected](autouse=True)+def _reset_warning_dedup():+ """The per-model warn-once set is module-global; keep tests order-independent."""+ import headroom.proxy.cost as cost_mod++ cost_mod._warned_estimated_basis_models.clear()+ yield+ cost_mod._warned_estimated_basis_models.clear()+++def _tracker(**kwargs):+ from headroom.proxy.server import CostTracker++ return CostTracker(**kwargs)+++# ── Basis marking ────────────────────────────────────────────────────+++def test_missing_usage_breakdown_books_estimated_basis():+ """No breakdown → the record is marked estimated and reported as such."""+ ct = _tracker(budget_limit_usd=100.0)+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=50_000, output_tokens=1_000)++ basis = ct.stats()["budget_basis"]+ assert basis["estimated_usd"] > 0+ assert basis["measured_usd"] == 0+ assert basis["estimated_records"] == 1+ assert basis["estimated_pct"] == 100.0+++def test_provider_breakdown_books_measured_basis():+ """A reported breakdown → measured; nothing lands in the estimated bucket."""+ ct = _tracker(budget_limit_usd=100.0)+ ct.record_tokens(+ MODEL,+ tokens_saved=0,+ tokens_sent=50_000,+ uncached_tokens=30_000,+ output_tokens=1_000,+ )++ basis = ct.stats()["budget_basis"]+ assert basis["measured_usd"] > 0+ assert basis["estimated_usd"] == 0+ assert basis["estimated_records"] == 0+ assert basis["estimated_pct"] == 0.0+++def test_cache_read_only_response_counts_as_measured():+ """A fully cache-read turn reports usage, so it is not an estimate."""+ ct = _tracker(budget_limit_usd=100.0)+ ct.record_tokens(+ MODEL,+ tokens_saved=0,+ tokens_sent=50_000,+ cache_read_tokens=40_000,+ output_tokens=1_000,+ )++ assert ct.stats()["budget_basis"]["estimated_usd"] == 0+++def test_mixed_records_stay_separable_and_sum_to_total():+ """Default policy is unchanged: the budget still sees every booked dollar."""+ ct = _tracker(budget_limit_usd=100.0)+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=50_000, output_tokens=1_000)+ ct.record_tokens(+ MODEL,+ tokens_saved=0,+ tokens_sent=50_000,+ uncached_tokens=30_000,+ output_tokens=1_000,+ )++ basis = ct.stats()["budget_basis"]+ assert basis["records"] == 2+ assert basis["estimated_records"] == 1+ assert basis["measured_usd"] > 0+ assert basis["estimated_usd"] > 0+ assert basis["total_usd"] == pytest.approx(basis["measured_usd"] + basis["estimated_usd"])+ # Regression guard: `count` (the default) enforces against total spend+ # exactly as it did before this change.+ assert ct.get_period_cost() == pytest.approx(basis["total_usd"])+++def test_get_period_cost_can_filter_by_basis():+ ct = _tracker(budget_limit_usd=100.0)+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=50_000, output_tokens=1_000)+ ct.record_tokens(+ MODEL, tokens_saved=0, tokens_sent=50_000, uncached_tokens=30_000, output_tokens=1_000+ )++ measured = ct.get_period_cost("measured")+ estimated = ct.get_period_cost("estimated")+ assert measured > 0+ assert estimated > 0+ assert ct.get_period_cost() == pytest.approx(measured + estimated)+++# ── Warning ──────────────────────────────────────────────────────────+++def test_estimated_basis_warns_once_per_model(caplog):+ """A route that never reports usage must not flood proxy.log (cf. #2504)."""+ ct = _tracker(budget_limit_usd=100.0)++ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):+ for _ in range(5):+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=10_000, output_tokens=100)++ hits = [r for r in caplog.records if "budget basis estimated" in r.getMessage()]+ assert len(hits) == 1+ assert MODEL in hits[0].getMessage()+++def test_distinct_models_each_warn_once(caplog):+ ct = _tracker(budget_limit_usd=100.0)+ other = "claude-haiku-4-5-20251001"++ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):+ for model in (MODEL, MODEL, other, other):+ ct.record_tokens(model, tokens_saved=0, tokens_sent=10_000, output_tokens=100)++ msgs = [r.getMessage() for r in caplog.records if "budget basis estimated" in r.getMessage()]+ assert sum(MODEL in m for m in msgs) == 1+ assert sum(other in m for m in msgs) == 1+++def test_measured_records_do_not_warn(caplog):+ ct = _tracker(budget_limit_usd=100.0)++ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):+ ct.record_tokens(+ MODEL, tokens_saved=0, tokens_sent=10_000, uncached_tokens=9_000, output_tokens=100+ )++ assert not [r for r in caplog.records if "budget basis estimated" in r.getMessage()]+++# ── Enforcement policies ─────────────────────────────────────────────+++def test_count_policy_lets_estimated_spend_exhaust_the_budget():+ """Default: an estimate consumes the budget, as it always has."""+ ct = _tracker(budget_limit_usd=0.0001, estimated_basis_policy="count")+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=500_000, output_tokens=10_000)++ allowed, remaining = ct.check_budget()+ assert not allowed+ assert remaining == 0+++def test_ignore_policy_keeps_estimated_spend_out_of_enforcement():+ """`ignore`: the record is still booked and reported, but doesn't enforce."""+ ct = _tracker(budget_limit_usd=0.0001, estimated_basis_policy="ignore")+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=500_000, output_tokens=10_000)++ allowed, _remaining = ct.check_budget()+ assert allowed+ # Still visible in the ledger — ignored for enforcement, not dropped.+ assert ct.stats()["budget_basis"]["estimated_usd"] > 0+++def test_ignore_policy_still_enforces_measured_spend():+ ct = _tracker(budget_limit_usd=0.0001, estimated_basis_policy="ignore")+ ct.record_tokens(+ MODEL,+ tokens_saved=0,+ tokens_sent=500_000,+ uncached_tokens=500_000,+ output_tokens=10_000,+ )++ allowed, _remaining = ct.check_budget()+ assert not allowed+++def test_block_policy_refuses_once_any_estimated_spend_exists():+ """`block`: fail closed rather than enforce a hard limit on a guess."""+ ct = _tracker(budget_limit_usd=1_000_000.0, estimated_basis_policy="block")+ allowed, _remaining = ct.check_budget()+ assert allowed # nothing booked yet++ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=1_000, output_tokens=10)++ allowed, remaining = ct.check_budget()+ assert not allowed+ assert remaining == 0.0+++def test_block_policy_is_inert_without_a_budget_limit():+ """No limit configured means no hard control to protect."""+ ct = _tracker(budget_limit_usd=None, estimated_basis_policy="block")+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=1_000, output_tokens=10)++ allowed, remaining = ct.check_budget()+ assert allowed+ assert remaining == float("inf")+++def test_block_policy_allows_purely_measured_traffic():+ ct = _tracker(budget_limit_usd=1_000_000.0, estimated_basis_policy="block")+ ct.record_tokens(+ MODEL, tokens_saved=0, tokens_sent=1_000, uncached_tokens=900, output_tokens=10+ )++ allowed, _remaining = ct.check_budget()+ assert allowed+++def test_invalid_policy_falls_back_to_count():+ ct = _tracker(budget_limit_usd=0.0001, estimated_basis_policy="nonsense")+ assert ct.estimated_basis_policy == "count"++ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=500_000, output_tokens=10_000)+ allowed, _remaining = ct.check_budget()+ assert not allowed+++# ── Denial message ───────────────────────────────────────────────────+++def test_denial_detail_names_the_estimated_share():+ ct = _tracker(budget_limit_usd=0.0001)+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=500_000, output_tokens=10_000)++ detail = ct.budget_denial_detail()+ assert "Budget exceeded for daily period" in detail+ assert "Headroom token estimates" in detail+++def test_denial_detail_unchanged_for_purely_measured_spend():+ ct = _tracker(budget_limit_usd=0.0001)+ ct.record_tokens(+ MODEL,+ tokens_saved=0,+ tokens_sent=500_000,+ uncached_tokens=500_000,+ output_tokens=10_000,+ )++ assert ct.budget_denial_detail() == "Budget exceeded for daily period"+++def test_block_denial_is_distinguishable_from_overspend():+ ct = _tracker(budget_limit_usd=1_000_000.0, estimated_basis_policy="block")+ ct.record_tokens(MODEL, tokens_saved=0, tokens_sent=1_000, output_tokens=10)++ detail = ct.budget_denial_detail()+ assert "Budget enforcement blocked" in detail+ assert "HEADROOM_BUDGET_ESTIMATED_BASIS=block" in detail+++# ── Policy resolver ──────────────────────────────────────────────────+++def test_resolver_precedence_and_fallback():+ from headroom.proxy.budget_basis_policy import resolve_estimated_basis_policy++ env = {"HEADROOM_BUDGET_ESTIMATED_BASIS": "ignore"}+ assert resolve_estimated_basis_policy("block", env) == "block" # explicit wins+ assert resolve_estimated_basis_policy(None, env) == "ignore" # env next+ assert resolve_estimated_basis_policy(None, {}) == "count" # default+ assert resolve_estimated_basis_policy(None, None) == "count"+ assert resolve_estimated_basis_policy("BLOCK", None) == "block" # normalized+ assert resolve_estimated_basis_policy("nope", None) == "count" # fallback+++def test_stats_reports_the_active_policy():+ ct = _tracker(budget_limit_usd=10.0, estimated_basis_policy="block")+ assert ct.stats()["budget_estimated_basis"] == "block"+ assert ct.stats()["budget_basis"]["policy"] == "block"