Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): enforce configured TPM limits (#3350)
C0sm0cats138736c96f24
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex b1f0a8869..c757ec9b7 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -1432,6 +1432,19 @@ class AnthropicHandlerMixin:# conversation is CPU-bound — on-loop it froze the server (#1701).tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)+ if self.rate_limiter:+ allowed, wait_seconds = await self.rate_limiter.check_tokens(+ rate_key, original_tokens+ )+ if not allowed:+ await self.metrics.record_rate_limited(provider=provider_name)+ await _finalize_pre_upstream()+ raise HTTPException(+ status_code=429,+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",+ headers={"Retry-After": str(int(wait_seconds) + 1)},+ )+# Enterprise Security: scan request before compression_security_ctx = Noneif self.security:diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex 0d8e1bc36..1219648c2 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -532,6 +532,15 @@ class GeminiHandlerMixin:# Token counting (offloaded off the event loop — GH #1701)tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)+ if self.rate_limiter:+ allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)+ if not allowed:+ await self.metrics.record_rate_limited(provider=provider_name)+ raise HTTPException(+ status_code=429,+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",+ )+# Optimizationtransforms_applied: list[str] = []waste_signals_dict: dict[str, int] | None = Nonediff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 6fc152f75..a4628367f 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3689,6 +3689,15 @@ class OpenAIHandlerMixin:# Token counting (offloaded off the event loop — GH #1701)tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)+ if self.rate_limiter:+ allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)+ if not allowed:+ await self.metrics.record_rate_limited(provider=openai_chat_outcome_provider)+ raise HTTPException(+ status_code=429,+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",+ )+# Hook: pre_compress_hook_biases = None# Hard per-message veto. Separate from ``_hook_biases`` because a bias is@@ -5735,6 +5744,15 @@ class OpenAIHandlerMixin:# shift when it does.message_input_tokens = original_tokens+ if self.rate_limiter:+ allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)+ if not allowed:+ await self.metrics.record_rate_limited(provider="openai")+ raise HTTPException(+ status_code=429,+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",+ )+# Defaults below feed downstream telemetry and memory injection.# If optimization remains enabled, the Responses payload is compressed# later through `_compress_openai_responses_payload`.diff --git a/tests/test_proxy_tpm_enforcement.py b/tests/test_proxy_tpm_enforcement.pynew file mode 100644index 000000000..f16586c94--- /dev/null+++ b/tests/test_proxy_tpm_enforcement.py@@ -0,0 +1,114 @@+"""The configured token-per-minute bucket must be consumed by handlers."""++from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock++import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.server import ProxyConfig, create_app # noqa: E402+++def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) -> None:+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=True,+ rate_limit_requests_per_minute=1_000,+ rate_limit_tokens_per_minute=1,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )++ with TestClient(create_app(config)) as client:+ proxy = client.app.state.proxy+ tokenizer = MagicMock()+ proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))+ proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))++ response = client.post(+ "/v1/messages",+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-haiku-4-5",+ "max_tokens": 64,+ "messages": [{"role": "user", "content": "large request"}],+ "stream": False,+ },+ )++ assert response.status_code == 429+ assert response.json()["detail"].startswith("Token rate limited.")+ proxy._retry_request.assert_not_awaited()+ assert len(proxy.rate_limiter._token_buckets) == 1+++ ("path", "headers", "body"),+ [+ (+ "/v1/chat/completions",+ {"authorization": "Bearer test-key"},+ {+ "model": "gpt-4o-mini",+ "messages": [{"role": "user", "content": "large request"}],+ "stream": False,+ },+ ),+ (+ "/v1/responses",+ {"authorization": "Bearer test-key"},+ {+ "model": "gpt-4o-mini",+ "input": "large request",+ "stream": False,+ },+ ),+ (+ "/v1beta/models/gemini-2.5-flash:generateContent",+ {"x-goog-api-key": "test-key"},+ {+ "contents": [{"role": "user", "parts": [{"text": "large request"}]}],+ },+ ),+ ],+)+def test_other_handlers_reject_request_over_tpm_before_upstream(+ monkeypatch, path, headers, body+) -> None:+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=True,+ rate_limit_requests_per_minute=1_000,+ rate_limit_tokens_per_minute=1,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )++ with TestClient(create_app(config)) as client:+ proxy = client.app.state.proxy+ tokenizer = MagicMock()+ proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))+ proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))++ response = client.post(path, headers=headers, json=body)++ assert response.status_code == 429+ assert response.json()["detail"].startswith("Token rate limited.")+ proxy._retry_request.assert_not_awaited()+ assert len(proxy.rate_limiter._token_buckets) == 1