Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): enforce configured TPM limits (#3350)

C0sm0cats138736c96f24
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index b1f0a8869..c757ec9b7 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -1432,6 +1432,19 @@ class AnthropicHandlerMixin:
# conversation is CPU-bound — on-loop it froze the server (#1701).
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
+ if self.rate_limiter:
+ allowed, wait_seconds = await self.rate_limiter.check_tokens(
+ rate_key, original_tokens
+ )
+ if not allowed:
+ await self.metrics.record_rate_limited(provider=provider_name)
+ await _finalize_pre_upstream()
+ raise HTTPException(
+ status_code=429,
+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
+ headers={"Retry-After": str(int(wait_seconds) + 1)},
+ )
+
# Enterprise Security: scan request before compression
_security_ctx = None
if self.security:
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.py
index 0d8e1bc36..1219648c2 100644
--- a/headroom/proxy/handlers/gemini.py
+++ b/headroom/proxy/handlers/gemini.py
@@ -532,6 +532,15 @@ class GeminiHandlerMixin:
# Token counting (offloaded off the event loop — GH #1701)
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
+ if self.rate_limiter:
+ allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
+ if not allowed:
+ await self.metrics.record_rate_limited(provider=provider_name)
+ raise HTTPException(
+ status_code=429,
+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
# Optimization
transforms_applied: list[str] = []
waste_signals_dict: dict[str, int] | None = None
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 6fc152f75..a4628367f 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3689,6 +3689,15 @@ class OpenAIHandlerMixin:
# Token counting (offloaded off the event loop — GH #1701)
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
+ if self.rate_limiter:
+ allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
+ if not allowed:
+ await self.metrics.record_rate_limited(provider=openai_chat_outcome_provider)
+ raise HTTPException(
+ status_code=429,
+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
# Hook: pre_compress
_hook_biases = None
# Hard per-message veto. Separate from ``_hook_biases`` because a bias is
@@ -5735,6 +5744,15 @@ class OpenAIHandlerMixin:
# shift when it does.
message_input_tokens = original_tokens
+ if self.rate_limiter:
+ allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
+ if not allowed:
+ await self.metrics.record_rate_limited(provider="openai")
+ raise HTTPException(
+ status_code=429,
+ detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
# Defaults below feed downstream telemetry and memory injection.
# If optimization remains enabled, the Responses payload is compressed
# later through `_compress_openai_responses_payload`.
diff --git a/tests/test_proxy_tpm_enforcement.py b/tests/test_proxy_tpm_enforcement.py
new file mode 100644
index 000000000..f16586c94
--- /dev/null
+++ b/tests/test_proxy_tpm_enforcement.py
@@ -0,0 +1,114 @@
+"""The configured token-per-minute bucket must be consumed by handlers."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+
+def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) -> None:
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=True,
+ rate_limit_requests_per_minute=1_000,
+ rate_limit_tokens_per_minute=1,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+ tokenizer = MagicMock()
+ proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
+ proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
+
+ response = client.post(
+ "/v1/messages",
+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-haiku-4-5",
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "large request"}],
+ "stream": False,
+ },
+ )
+
+ assert response.status_code == 429
+ assert response.json()["detail"].startswith("Token rate limited.")
+ proxy._retry_request.assert_not_awaited()
+ assert len(proxy.rate_limiter._token_buckets) == 1
+
+
+ ("path", "headers", "body"),
+ [
+ (
+ "/v1/chat/completions",
+ {"authorization": "Bearer test-key"},
+ {
+ "model": "gpt-4o-mini",
+ "messages": [{"role": "user", "content": "large request"}],
+ "stream": False,
+ },
+ ),
+ (
+ "/v1/responses",
+ {"authorization": "Bearer test-key"},
+ {
+ "model": "gpt-4o-mini",
+ "input": "large request",
+ "stream": False,
+ },
+ ),
+ (
+ "/v1beta/models/gemini-2.5-flash:generateContent",
+ {"x-goog-api-key": "test-key"},
+ {
+ "contents": [{"role": "user", "parts": [{"text": "large request"}]}],
+ },
+ ),
+ ],
+)
+def test_other_handlers_reject_request_over_tpm_before_upstream(
+ monkeypatch, path, headers, body
+) -> None:
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=True,
+ rate_limit_requests_per_minute=1_000,
+ rate_limit_tokens_per_minute=1,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+ tokenizer = MagicMock()
+ proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
+ proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
+
+ response = client.post(path, headers=headers, json=body)
+
+ assert response.status_code == 429
+ assert response.json()["detail"].startswith("Token rate limited.")
+ proxy._retry_request.assert_not_awaited()
+ assert len(proxy.rate_limiter._token_buckets) == 1