Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(perf): make budget checks constant-time (#3368)

C0sm0catsed08069ba00d
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py
index 672b8f152..81025ebca 100644
--- a/headroom/proxy/cost.py
+++ b/headroom/proxy/cost.py
@@ -863,6 +863,15 @@ class CostTracker:
# Cost tracking - using deque for efficient left-side removal
self._costs: deque[CostEntry] = deque(maxlen=self.MAX_COST_ENTRIES)
self._last_prune_time: datetime = datetime.now()
+ # Current budget-window ledger and O(1) aggregates. ``_costs`` retains
+ # the reporting history, while this deque evicts entries as soon as the
+ # configured hourly/daily/monthly window advances. Each entry is added
+ # and removed once, making expiry amortized O(1) instead of rescanning
+ # up to MAX_COST_ENTRIES before every request (#3367).
+ self._budget_costs: deque[CostEntry] = deque()
+ self._budget_measured_usd = 0.0
+ self._budget_estimated_usd = 0.0
+ self._budget_estimated_records = 0
# Token savings per model (exact, no dollar estimation)
self._tokens_saved_by_model: dict[str, int] = {}
@@ -914,6 +923,10 @@ class CostTracker:
"""Reset in-memory cost/token counters for local test/debug use."""
self._costs.clear()
self._last_prune_time = datetime.now()
+ self._budget_costs.clear()
+ self._budget_measured_usd = 0.0
+ self._budget_estimated_usd = 0.0
+ self._budget_estimated_records = 0
self._tokens_saved_by_model.clear()
self._saved_write_5m_by_tier.clear()
self._saved_write_1h_by_tier.clear()
@@ -1188,12 +1201,14 @@ class CostTracker:
cache_write_tokens=effective_cache_write,
)
if cost is not None:
- self._costs.append(CostEntry(datetime.now(), cost, basis))
+ entry = CostEntry(datetime.now(), cost, basis)
+ self._costs.append(entry)
+ self._record_budget_cost(entry)
self._prune_old_costs()
- def _period_cutoff(self) -> datetime:
+ def _period_cutoff(self, now: datetime | None = None) -> datetime:
"""Start of the current budget period."""
- now = datetime.now()
+ now = now or datetime.now()
if self.budget_period == "hourly":
return now - timedelta(hours=1)
@@ -1202,6 +1217,37 @@ class CostTracker:
# monthly
return now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ def _subtract_budget_cost(self, entry: CostEntry) -> None:
+ if entry.basis == COST_BASIS_ESTIMATED:
+ self._budget_estimated_usd -= entry.cost_usd
+ self._budget_estimated_records -= 1
+ else:
+ self._budget_measured_usd -= entry.cost_usd
+
+ def _refresh_budget_window(self, now: datetime | None = None) -> None:
+ """Evict expired current-period entries, each exactly once."""
+ cutoff = self._period_cutoff(now)
+ while self._budget_costs and self._budget_costs[0].timestamp < cutoff:
+ self._subtract_budget_cost(self._budget_costs.popleft())
+
+ # Repeated floating-point additions/subtractions can leave a tiny
+ # negative residue after a complete window rollover.
+ self._budget_measured_usd = max(0.0, self._budget_measured_usd)
+ self._budget_estimated_usd = max(0.0, self._budget_estimated_usd)
+ self._budget_estimated_records = max(0, self._budget_estimated_records)
+
+ def _record_budget_cost(self, entry: CostEntry) -> None:
+ """Add one entry to the current-period aggregate in amortized O(1)."""
+ self._refresh_budget_window(entry.timestamp)
+ if len(self._budget_costs) >= self.MAX_COST_ENTRIES:
+ self._subtract_budget_cost(self._budget_costs.popleft())
+ self._budget_costs.append(entry)
+ if entry.basis == COST_BASIS_ESTIMATED:
+ self._budget_estimated_usd += entry.cost_usd
+ self._budget_estimated_records += 1
+ else:
+ self._budget_measured_usd += entry.cost_usd
+
def get_period_cost(self, basis: str | None = None) -> float:
"""Get cost for current budget period.
@@ -1209,12 +1255,14 @@ class CostTracker:
regardless of how each record's input count was derived. Pass a basis
(``"measured"`` / ``"estimated"``) to get just that slice.
"""
- cutoff = self._period_cutoff()
- return sum(
- entry.cost_usd
- for entry in self._costs
- if entry.timestamp >= cutoff and (basis is None or entry.basis == basis)
- )
+ breakdown = self.period_cost_breakdown()
+ if basis == COST_BASIS_MEASURED:
+ return float(breakdown["measured_usd"])
+ if basis == COST_BASIS_ESTIMATED:
+ return float(breakdown["estimated_usd"])
+ if basis is not None:
+ return 0.0
+ return float(breakdown["total_usd"])
def period_cost_breakdown(self) -> dict[str, Any]:
"""Split the period's booked spend by how its input count was derived.
@@ -1224,20 +1272,11 @@ class CostTracker:
it separable is the point: a budget refusal driven by a guess should be
distinguishable from one driven by provider-reported usage (#2713).
"""
- cutoff = self._period_cutoff()
- measured_usd = 0.0
- estimated_usd = 0.0
- records = 0
- estimated_records = 0
- for entry in self._costs:
- if entry.timestamp < cutoff:
- continue
- records += 1
- if entry.basis == COST_BASIS_ESTIMATED:
- estimated_usd += entry.cost_usd
- estimated_records += 1
- else:
- measured_usd += entry.cost_usd
+ self._refresh_budget_window()
+ measured_usd = self._budget_measured_usd
+ estimated_usd = self._budget_estimated_usd
+ records = len(self._budget_costs)
+ estimated_records = self._budget_estimated_records
total_usd = measured_usd + estimated_usd
return {
diff --git a/tests/test_cost_budget_performance.py b/tests/test_cost_budget_performance.py
new file mode 100644
index 000000000..4e8050a0d
--- /dev/null
+++ b/tests/test_cost_budget_performance.py
@@ -0,0 +1,102 @@
+"""Incremental budget aggregation regression coverage (#3367)."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+import pytest
+
+from headroom.proxy.budget_basis_policy import COST_BASIS_ESTIMATED, COST_BASIS_MEASURED
+from headroom.proxy.cost import CostEntry, CostTracker
+
+
+def test_budget_breakdown_does_not_scan_reporting_history() -> None:
+ tracker = CostTracker(budget_limit_usd=100.0)
+ now = datetime.now()
+ tracker._costs.extend(
+ CostEntry(now, 1.0, COST_BASIS_MEASURED) for _ in range(tracker.MAX_COST_ENTRIES)
+ )
+ tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_MEASURED))
+
+ class NonIterableHistory:
+ def __iter__(self):
+ raise AssertionError("budget check rescanned reporting history")
+
+ tracker._costs = NonIterableHistory() # type: ignore[assignment]
+
+ assert tracker.check_budget() == (True, 98.0)
+
+
+def test_budget_aggregate_preserves_basis_totals_and_policy() -> None:
+ tracker = CostTracker(
+ budget_limit_usd=10.0,
+ estimated_basis_policy="ignore",
+ )
+ now = datetime.now()
+ tracker._record_budget_cost(CostEntry(now, 3.0, COST_BASIS_MEASURED))
+ tracker._record_budget_cost(CostEntry(now, 20.0, COST_BASIS_ESTIMATED))
+
+ assert tracker.period_cost_breakdown() == {
+ "period": "daily",
+ "policy": "ignore",
+ "total_usd": 23.0,
+ "measured_usd": 3.0,
+ "estimated_usd": 20.0,
+ "estimated_pct": pytest.approx(87.0),
+ "records": 2,
+ "estimated_records": 1,
+ }
+ assert tracker.check_budget() == (True, 7.0)
+
+
+ ("period", "now", "expired_at", "current_at"),
+ [
+ (
+ "hourly",
+ datetime(2026, 8, 31, 12),
+ datetime(2026, 8, 31, 10),
+ datetime(2026, 8, 31, 11, 30),
+ ),
+ (
+ "daily",
+ datetime(2026, 8, 31, 12),
+ datetime(2026, 8, 30, 23, 59),
+ datetime(2026, 8, 31, 1),
+ ),
+ (
+ "monthly",
+ datetime(2026, 8, 31, 12),
+ datetime(2026, 7, 31, 23, 59),
+ datetime(2026, 8, 1),
+ ),
+ ],
+)
+def test_budget_window_evicts_expired_entries_once(period, now, expired_at, current_at) -> None:
+ tracker = CostTracker(budget_limit_usd=100.0, budget_period=period)
+ tracker._record_budget_cost(CostEntry(expired_at, 40.0, COST_BASIS_MEASURED))
+ tracker._record_budget_cost(CostEntry(current_at, 5.0, COST_BASIS_ESTIMATED))
+
+ tracker._refresh_budget_window(now)
+ assert tracker._budget_measured_usd == 0.0
+ assert tracker._budget_estimated_usd == 5.0
+ assert tracker._budget_estimated_records == 1
+ assert len(tracker._budget_costs) == 1
+
+ # Re-reading the same window is constant-time and must not subtract twice.
+ tracker._refresh_budget_window(now)
+ assert tracker._budget_estimated_usd == 5.0
+ assert len(tracker._budget_costs) == 1
+
+
+def test_budget_aggregate_keeps_existing_100k_record_cap() -> None:
+ tracker = CostTracker(budget_limit_usd=1_000_000.0)
+ tracker.MAX_COST_ENTRIES = 2
+ now = datetime.now()
+ tracker._record_budget_cost(CostEntry(now, 1.0, COST_BASIS_MEASURED))
+ tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_ESTIMATED))
+ tracker._record_budget_cost(CostEntry(now, 4.0, COST_BASIS_MEASURED))
+
+ assert tracker.period_cost_breakdown()["total_usd"] == 6.0
+ assert tracker.period_cost_breakdown()["records"] == 2
+ assert tracker.period_cost_breakdown()["estimated_records"] == 1