Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): keep in-period costs in the budget ledger when the entry cap is reached (#4448)
danieldutu41c15b311cd5
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex 5ba62dc55..fb317408b 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -873,7 +873,9 @@ class CostTracker:# the reporting history, while this deque evicts entries as soon as the# configured hourly/daily/monthly window advances. Each entry is added# and removed once, making expiry amortized O(1) instead of rescanning- # up to MAX_COST_ENTRIES before every request (#3367).+ # up to MAX_COST_ENTRIES before every request (#3367). It is bounded by+ # the period, not by MAX_COST_ENTRIES: evicting an in-period entry would+ # drop spend that still counts against the budget (#4416).self._budget_costs: deque[CostEntry] = deque()self._budget_measured_usd = 0.0self._budget_estimated_usd = 0.0@@ -1266,8 +1268,6 @@ class CostTracker:def _record_budget_cost(self, entry: CostEntry) -> None:"""Add one entry to the current-period aggregate in amortized O(1)."""self._refresh_budget_window(entry.timestamp)- if len(self._budget_costs) >= self.MAX_COST_ENTRIES:- self._subtract_budget_cost(self._budget_costs.popleft())self._budget_costs.append(entry)if entry.basis == COST_BASIS_ESTIMATED:self._budget_estimated_usd += entry.cost_usddiff --git a/tests/test_cost_budget_performance.py b/tests/test_cost_budget_performance.pyindex 4e8050a0d..cf1a1c82f 100644--- a/tests/test_cost_budget_performance.py+++ b/tests/test_cost_budget_performance.py@@ -89,7 +89,7 @@ def test_budget_window_evicts_expired_entries_once(period, now, expired_at, currassert len(tracker._budget_costs) == 1-def test_budget_aggregate_keeps_existing_100k_record_cap() -> None:+def test_budget_ledger_keeps_in_period_entries_past_record_cap() -> None:tracker = CostTracker(budget_limit_usd=1_000_000.0)tracker.MAX_COST_ENTRIES = 2now = datetime.now()@@ -97,6 +97,6 @@ def test_budget_aggregate_keeps_existing_100k_record_cap() -> None:tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_ESTIMATED))tracker._record_budget_cost(CostEntry(now, 4.0, COST_BASIS_MEASURED))- assert tracker.period_cost_breakdown()["total_usd"] == 6.0- assert tracker.period_cost_breakdown()["records"] == 2+ assert tracker.period_cost_breakdown()["total_usd"] == 7.0+ assert tracker.period_cost_breakdown()["records"] == 3assert tracker.period_cost_breakdown()["estimated_records"] == 1