Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(perf): make budget checks constant-time (#3368)
C0sm0catsed08069ba00d
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex 672b8f152..81025ebca 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -863,6 +863,15 @@ class CostTracker:# Cost tracking - using deque for efficient left-side removalself._costs: deque[CostEntry] = deque(maxlen=self.MAX_COST_ENTRIES)self._last_prune_time: datetime = datetime.now()+ # Current budget-window ledger and O(1) aggregates. ``_costs`` retains+ # the reporting history, while this deque evicts entries as soon as the+ # configured hourly/daily/monthly window advances. Each entry is added+ # and removed once, making expiry amortized O(1) instead of rescanning+ # up to MAX_COST_ENTRIES before every request (#3367).+ self._budget_costs: deque[CostEntry] = deque()+ self._budget_measured_usd = 0.0+ self._budget_estimated_usd = 0.0+ self._budget_estimated_records = 0# Token savings per model (exact, no dollar estimation)self._tokens_saved_by_model: dict[str, int] = {}@@ -914,6 +923,10 @@ class CostTracker:"""Reset in-memory cost/token counters for local test/debug use."""self._costs.clear()self._last_prune_time = datetime.now()+ self._budget_costs.clear()+ self._budget_measured_usd = 0.0+ self._budget_estimated_usd = 0.0+ self._budget_estimated_records = 0self._tokens_saved_by_model.clear()self._saved_write_5m_by_tier.clear()self._saved_write_1h_by_tier.clear()@@ -1188,12 +1201,14 @@ class CostTracker:cache_write_tokens=effective_cache_write,)if cost is not None:- self._costs.append(CostEntry(datetime.now(), cost, basis))+ entry = CostEntry(datetime.now(), cost, basis)+ self._costs.append(entry)+ self._record_budget_cost(entry)self._prune_old_costs()- def _period_cutoff(self) -> datetime:+ def _period_cutoff(self, now: datetime | None = None) -> datetime:"""Start of the current budget period."""- now = datetime.now()+ now = now or datetime.now()if self.budget_period == "hourly":return now - timedelta(hours=1)@@ -1202,6 +1217,37 @@ class CostTracker:# monthlyreturn now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)+ def _subtract_budget_cost(self, entry: CostEntry) -> None:+ if entry.basis == COST_BASIS_ESTIMATED:+ self._budget_estimated_usd -= entry.cost_usd+ self._budget_estimated_records -= 1+ else:+ self._budget_measured_usd -= entry.cost_usd++ def _refresh_budget_window(self, now: datetime | None = None) -> None:+ """Evict expired current-period entries, each exactly once."""+ cutoff = self._period_cutoff(now)+ while self._budget_costs and self._budget_costs[0].timestamp < cutoff:+ self._subtract_budget_cost(self._budget_costs.popleft())++ # Repeated floating-point additions/subtractions can leave a tiny+ # negative residue after a complete window rollover.+ self._budget_measured_usd = max(0.0, self._budget_measured_usd)+ self._budget_estimated_usd = max(0.0, self._budget_estimated_usd)+ self._budget_estimated_records = max(0, self._budget_estimated_records)++ def _record_budget_cost(self, entry: CostEntry) -> None:+ """Add one entry to the current-period aggregate in amortized O(1)."""+ self._refresh_budget_window(entry.timestamp)+ if len(self._budget_costs) >= self.MAX_COST_ENTRIES:+ self._subtract_budget_cost(self._budget_costs.popleft())+ self._budget_costs.append(entry)+ if entry.basis == COST_BASIS_ESTIMATED:+ self._budget_estimated_usd += entry.cost_usd+ self._budget_estimated_records += 1+ else:+ self._budget_measured_usd += entry.cost_usd+def get_period_cost(self, basis: str | None = None) -> float:"""Get cost for current budget period.@@ -1209,12 +1255,14 @@ class CostTracker:regardless of how each record's input count was derived. Pass a basis(``"measured"`` / ``"estimated"``) to get just that slice."""- cutoff = self._period_cutoff()- return sum(- entry.cost_usd- for entry in self._costs- if entry.timestamp >= cutoff and (basis is None or entry.basis == basis)- )+ breakdown = self.period_cost_breakdown()+ if basis == COST_BASIS_MEASURED:+ return float(breakdown["measured_usd"])+ if basis == COST_BASIS_ESTIMATED:+ return float(breakdown["estimated_usd"])+ if basis is not None:+ return 0.0+ return float(breakdown["total_usd"])def period_cost_breakdown(self) -> dict[str, Any]:"""Split the period's booked spend by how its input count was derived.@@ -1224,20 +1272,11 @@ class CostTracker:it separable is the point: a budget refusal driven by a guess should bedistinguishable from one driven by provider-reported usage (#2713)."""- cutoff = self._period_cutoff()- measured_usd = 0.0- estimated_usd = 0.0- records = 0- estimated_records = 0- for entry in self._costs:- if entry.timestamp < cutoff:- continue- records += 1- if entry.basis == COST_BASIS_ESTIMATED:- estimated_usd += entry.cost_usd- estimated_records += 1- else:- measured_usd += entry.cost_usd+ self._refresh_budget_window()+ measured_usd = self._budget_measured_usd+ estimated_usd = self._budget_estimated_usd+ records = len(self._budget_costs)+ estimated_records = self._budget_estimated_recordstotal_usd = measured_usd + estimated_usdreturn {diff --git a/tests/test_cost_budget_performance.py b/tests/test_cost_budget_performance.pynew file mode 100644index 000000000..4e8050a0d--- /dev/null+++ b/tests/test_cost_budget_performance.py@@ -0,0 +1,102 @@+"""Incremental budget aggregation regression coverage (#3367)."""++from __future__ import annotations++from datetime import datetime++import pytest++from headroom.proxy.budget_basis_policy import COST_BASIS_ESTIMATED, COST_BASIS_MEASURED+from headroom.proxy.cost import CostEntry, CostTracker+++def test_budget_breakdown_does_not_scan_reporting_history() -> None:+ tracker = CostTracker(budget_limit_usd=100.0)+ now = datetime.now()+ tracker._costs.extend(+ CostEntry(now, 1.0, COST_BASIS_MEASURED) for _ in range(tracker.MAX_COST_ENTRIES)+ )+ tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_MEASURED))++ class NonIterableHistory:+ def __iter__(self):+ raise AssertionError("budget check rescanned reporting history")++ tracker._costs = NonIterableHistory() # type: ignore[assignment]++ assert tracker.check_budget() == (True, 98.0)+++def test_budget_aggregate_preserves_basis_totals_and_policy() -> None:+ tracker = CostTracker(+ budget_limit_usd=10.0,+ estimated_basis_policy="ignore",+ )+ now = datetime.now()+ tracker._record_budget_cost(CostEntry(now, 3.0, COST_BASIS_MEASURED))+ tracker._record_budget_cost(CostEntry(now, 20.0, COST_BASIS_ESTIMATED))++ assert tracker.period_cost_breakdown() == {+ "period": "daily",+ "policy": "ignore",+ "total_usd": 23.0,+ "measured_usd": 3.0,+ "estimated_usd": 20.0,+ "estimated_pct": pytest.approx(87.0),+ "records": 2,+ "estimated_records": 1,+ }+ assert tracker.check_budget() == (True, 7.0)+++ ("period", "now", "expired_at", "current_at"),+ [+ (+ "hourly",+ datetime(2026, 8, 31, 12),+ datetime(2026, 8, 31, 10),+ datetime(2026, 8, 31, 11, 30),+ ),+ (+ "daily",+ datetime(2026, 8, 31, 12),+ datetime(2026, 8, 30, 23, 59),+ datetime(2026, 8, 31, 1),+ ),+ (+ "monthly",+ datetime(2026, 8, 31, 12),+ datetime(2026, 7, 31, 23, 59),+ datetime(2026, 8, 1),+ ),+ ],+)+def test_budget_window_evicts_expired_entries_once(period, now, expired_at, current_at) -> None:+ tracker = CostTracker(budget_limit_usd=100.0, budget_period=period)+ tracker._record_budget_cost(CostEntry(expired_at, 40.0, COST_BASIS_MEASURED))+ tracker._record_budget_cost(CostEntry(current_at, 5.0, COST_BASIS_ESTIMATED))++ tracker._refresh_budget_window(now)+ assert tracker._budget_measured_usd == 0.0+ assert tracker._budget_estimated_usd == 5.0+ assert tracker._budget_estimated_records == 1+ assert len(tracker._budget_costs) == 1++ # Re-reading the same window is constant-time and must not subtract twice.+ tracker._refresh_budget_window(now)+ assert tracker._budget_estimated_usd == 5.0+ assert len(tracker._budget_costs) == 1+++def test_budget_aggregate_keeps_existing_100k_record_cap() -> None:+ tracker = CostTracker(budget_limit_usd=1_000_000.0)+ tracker.MAX_COST_ENTRIES = 2+ now = datetime.now()+ tracker._record_budget_cost(CostEntry(now, 1.0, COST_BASIS_MEASURED))+ tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_ESTIMATED))+ tracker._record_budget_cost(CostEntry(now, 4.0, COST_BASIS_MEASURED))++ assert tracker.period_cost_breakdown()["total_usd"] == 6.0+ assert tracker.period_cost_breakdown()["records"] == 2+ assert tracker.period_cost_breakdown()["estimated_records"] == 1