Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): keep in-period costs in the budget ledger when the entry cap is reached (#4448)

danieldutu41c15b311cd5
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.py
index 5ba62dc55..fb317408b 100644
--- a/headroom/proxy/cost.py
+++ b/headroom/proxy/cost.py
@@ -873,7 +873,9 @@ class CostTracker:
# the reporting history, while this deque evicts entries as soon as the
# configured hourly/daily/monthly window advances. Each entry is added
# and removed once, making expiry amortized O(1) instead of rescanning
- # up to MAX_COST_ENTRIES before every request (#3367).
+ # up to MAX_COST_ENTRIES before every request (#3367). It is bounded by
+ # the period, not by MAX_COST_ENTRIES: evicting an in-period entry would
+ # drop spend that still counts against the budget (#4416).
self._budget_costs: deque[CostEntry] = deque()
self._budget_measured_usd = 0.0
self._budget_estimated_usd = 0.0
@@ -1266,8 +1268,6 @@ class CostTracker:
def _record_budget_cost(self, entry: CostEntry) -> None:
"""Add one entry to the current-period aggregate in amortized O(1)."""
self._refresh_budget_window(entry.timestamp)
- if len(self._budget_costs) >= self.MAX_COST_ENTRIES:
- self._subtract_budget_cost(self._budget_costs.popleft())
self._budget_costs.append(entry)
if entry.basis == COST_BASIS_ESTIMATED:
self._budget_estimated_usd += entry.cost_usd
diff --git a/tests/test_cost_budget_performance.py b/tests/test_cost_budget_performance.py
index 4e8050a0d..cf1a1c82f 100644
--- a/tests/test_cost_budget_performance.py
+++ b/tests/test_cost_budget_performance.py
@@ -89,7 +89,7 @@ def test_budget_window_evicts_expired_entries_once(period, now, expired_at, curr
assert len(tracker._budget_costs) == 1
-def test_budget_aggregate_keeps_existing_100k_record_cap() -> None:
+def test_budget_ledger_keeps_in_period_entries_past_record_cap() -> None:
tracker = CostTracker(budget_limit_usd=1_000_000.0)
tracker.MAX_COST_ENTRIES = 2
now = datetime.now()
@@ -97,6 +97,6 @@ def test_budget_aggregate_keeps_existing_100k_record_cap() -> None:
tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_ESTIMATED))
tracker._record_budget_cost(CostEntry(now, 4.0, COST_BASIS_MEASURED))
- assert tracker.period_cost_breakdown()["total_usd"] == 6.0
- assert tracker.period_cost_breakdown()["records"] == 2
+ assert tracker.period_cost_breakdown()["total_usd"] == 7.0
+ assert tracker.period_cost_breakdown()["records"] == 3
assert tracker.period_cost_breakdown()["estimated_records"] == 1