Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): make budget enforcement actually work (#885)
Ashisha14ab45cf0e6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex f86f1feb1..154528daf 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -191,6 +191,11 @@ jobs:cp "${SITE}/headroom/"_core*.so headroom/python -c "from headroom._core import DiffCompressor; print('headroom._core OK')"+ # Coverage upload: without this, codecov only receives reports from+ # the two native-e2e workflows (3 CLI test files total), so head+ # coverage reads ~6% and codecov/patch fails for ANY diff not+ # exercised by those files — a false negative on every PR. The main+ # suite runs here; its coverage must be what codecov sees.- name: Run test shard ${{ matrix.shard }}/4run: |pytest tests scripts/tests \diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 2221e184a..5c9b6e87b 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -329,8 +329,19 @@ def _selected_context_tool() -> str:default=None,envvar="HEADROOM_BUDGET",help=(- "Daily budget limit in USD. Requests are rejected with 429 once the limit is reached. "- "Resets at midnight UTC. Env: HEADROOM_BUDGET."+ "Budget limit in USD per --budget-period. Requests are rejected with 429 "+ "once the limit is reached. Env: HEADROOM_BUDGET."+ ),+)+ "--budget-period",+ type=click.Choice(["hourly", "daily", "monthly"]),+ default="daily",+ envvar="HEADROOM_BUDGET_PERIOD",+ help=(+ "Period the --budget limit applies to. Hourly resets on a rolling hour, "+ "daily at local midnight, monthly on the 1st. Default: daily. "+ "Env: HEADROOM_BUDGET_PERIOD."),)# Code-aware compression (AST-based, requires `pip install headroom-ai[code]`).@@ -621,6 +632,7 @@ def proxy(codex_wire_debug: bool,codex_wire_debug_dir: str | None,budget: float | None,+ budget_period: str,code_aware_flag: bool | None,disable_kompress: bool,code_graph: bool,@@ -827,6 +839,7 @@ def proxy(log_full_messages=log_messagesor os.environ.get("HEADROOM_LOG_MESSAGES", "").lower() in ("true", "1", "yes", "on"),budget_limit_usd=budget,+ budget_period=cast(Literal["hourly", "daily", "monthly"], budget_period),# Code-aware compression resolution:# 1. Explicit --code-aware / --no-code-aware always wins.# 2. Otherwise read HEADROOM_CODE_AWARE_ENABLED (truthy = on).diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex b180e181d..938d2217f 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -556,7 +556,10 @@ class CostTracker:"""MAX_COST_ENTRIES = 100_000- COST_RETENTION_HOURS = 24+ # Used by _prune_old_costs(), called from record_tokens() on every request.+ # Must be >= the longest budget_period (monthly = up to 31 days), otherwise+ # get_period_cost() undercounts and check_budget() silently under-enforces.+ COST_RETENTION_HOURS = 744 # 31 daysdef __init__(self, budget_limit_usd: float | None = None, budget_period: str = "daily"):self.budget_limit_usd = budget_limit_usd@@ -667,8 +670,9 @@ class CostTracker:cache_write_5m_tokens: int = 0,cache_write_1h_tokens: int = 0,uncached_tokens: int = 0,+ output_tokens: int = 0,):- """Record token counts per model.+ """Record token counts per model and accumulate request cost for budget enforcement.Args:model: Model name.@@ -677,6 +681,7 @@ class CostTracker:cache_read_tokens: Cache read tokens from API response usage.cache_write_tokens: Cache write tokens from API response usage.uncached_tokens: Non-cached input tokens from API response usage.+ output_tokens: Output tokens from API response usage."""self._tokens_saved_by_model[model] = (self._tokens_saved_by_model.get(model, 0) + tokens_saved@@ -699,6 +704,24 @@ class CostTracker:self._api_uncached_by_model.get(model, 0) + uncached_tokens)+ # Populate _costs so check_budget() has real data to enforce against.+ # When the call site had no API usage breakdown (all cache/uncached+ # fields are 0), fall back to tokens_sent so input cost isn't+ # silently dropped from the budget.+ input_tokens = uncached_tokens+ if not (uncached_tokens or cache_read_tokens or cache_write_tokens):+ input_tokens = tokens_sent+ cost = self.estimate_cost(+ model=model,+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ cache_read_tokens=cache_read_tokens,+ cache_write_tokens=cache_write_tokens,+ )+ if cost is not None:+ self._costs.append((datetime.now(), cost))+ self._prune_old_costs()+def get_period_cost(self) -> float:"""Get cost for current budget period."""now = datetime.now()diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex 3e5d0fc5d..a23ed2e79 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -371,6 +371,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:cache_write_5m_tokens=outcome.cache_write_5m_tokens,cache_write_1h_tokens=outcome.cache_write_1h_tokens,uncached_tokens=outcome.uncached_input_tokens,+ output_tokens=outcome.output_tokens,)# 3. Per-request log (optional). The ``client`` outcome field isdiff --git a/tests/test_cli_proxy_env.py b/tests/test_cli_proxy_env.pyindex 12781dfb1..013768159 100644--- a/tests/test_cli_proxy_env.py+++ b/tests/test_cli_proxy_env.py@@ -222,6 +222,34 @@ class TestCLIProxyEnvVars:assert result.exit_code == 0, result.outputassert captured_config["config"].budget_limit_usd == 100.5+ def test_budget_period_flag_and_env(self, runner):+ """--budget-period and HEADROOM_BUDGET_PERIOD should reach ProxyConfig."""+ captured_config = {}++ def mock_run_server(config, **kwargs):+ captured_config["config"] = config++ with patch("headroom.proxy.server.run_server", mock_run_server):+ result = runner.invoke(+ main,+ ["proxy", "--budget", "50", "--budget-period", "monthly"],+ catch_exceptions=False,+ )++ assert result.exit_code == 0, result.output+ assert captured_config["config"].budget_period == "monthly"++ with patch("headroom.proxy.server.run_server", mock_run_server):+ result = runner.invoke(+ main,+ ["proxy"],+ env={"HEADROOM_BUDGET_PERIOD": "hourly"},+ catch_exceptions=False,+ )++ assert result.exit_code == 0, result.output+ assert captured_config["config"].budget_period == "hourly"+def test_code_aware_enabled_from_env(self, runner):"""HEADROOM_CODE_AWARE_ENABLED env var should be passed to ProxyConfig."""captured_config = {}diff --git a/tests/test_cost_tracker_counterfactual.py b/tests/test_cost_tracker_counterfactual.pyindex 4dde0f678..f49c974db 100644--- a/tests/test_cost_tracker_counterfactual.py+++ b/tests/test_cost_tracker_counterfactual.py@@ -108,3 +108,47 @@ def test_no_cost_without_headroom_field():stats = ct.stats()assert "cost_without_headroom_usd" not in stats+++def test_budget_enforced_after_recording_costs():+ """record_tokens must populate cost history so check_budget enforces the limit.++ Regression test: _costs was never written, so check_budget always+ returned (True, budget_limit) and budgets were silently unenforced.+ """+ from headroom.proxy.server import CostTracker++ ct = CostTracker(budget_limit_usd=0.0001, budget_period="daily")+ allowed, remaining = ct.check_budget()+ assert allowed # nothing spent yet++ # ~$1.50+ of Sonnet input at list price — far over the budget+ ct.record_tokens(+ "claude-sonnet-4-20250514",+ tokens_saved=0,+ tokens_sent=500_000,+ uncached_tokens=500_000,+ output_tokens=10_000,+ )++ assert ct.get_period_cost() > 0+ allowed, remaining = ct.check_budget()+ assert not allowed+ assert remaining == 0+++def test_budget_input_cost_counted_without_usage_breakdown():+ """When the call site has no API usage breakdown (cache/uncached all 0),+ tokens_sent must be used as the input count — input cost must not be+ silently dropped from the budget."""+ from headroom.proxy.server import CostTracker++ ct = CostTracker(budget_limit_usd=100.0)+ ct.record_tokens(+ "claude-sonnet-4-20250514",+ tokens_saved=0,+ tokens_sent=500_000,+ )++ # 500k input tokens at Sonnet list price is ~$1.50 — must be > output-only+ assert ct.get_period_cost() > 0.5diff --git a/tests/test_request_outcome.py b/tests/test_request_outcome.pyindex 35ec7285c..5f898b714 100644--- a/tests/test_request_outcome.py+++ b/tests/test_request_outcome.py@@ -293,6 +293,7 @@ async def test_funnel_passes_canonical_record_tokens_shape() -> None:"cache_write_5m_tokens": 80,"cache_write_1h_tokens": 20,"uncached_tokens": 0,+ "output_tokens": 50,}