Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): scope OpenAI rate limits by api-key (#3365)
C0sm0catsb9e8462a3a84
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 466e2283d..be26a3b83 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -9,10 +9,12 @@ import asyncioimport contextlibimport copyimport hashlib+import hmacimport jsonimport loggingimport osimport re+import secretsimport threadingimport timeimport uuid@@ -121,6 +123,30 @@ _CCR_HASH_RE = re.compile(r"(?:Retrieve (?:more|original): hash=|<<ccr:)([a-fA-F0-9]{12,24})(?=[^a-fA-F0-9]|$)")_BARE_CCR_HASH_RE = re.compile(r"[a-fA-F0-9]{12,24}")+_OPENAI_RATE_KEY_SECRET = secrets.token_bytes(32)+++def _openai_rate_limit_key(headers: dict[str, str]) -> str:+ """Return the credential identity used by the OpenAI rate limiter.++ OpenAI-compatible gateways may authenticate with either a bearer token or+ an ``api-key`` header. Deriving an identity from the complete value keeps common+ prefixes distinct without retaining recoverable credential material in+ bucket keys. The secret is process-local, like the limiter state.+ Requests without either retain the existing shared fallback bucket.+ """+ authorization = headers.get("authorization")+ api_key = headers.get("api-key")+ if authorization:+ kind, credential = "authorization", authorization+ elif api_key:+ kind, credential = "api-key", api_key+ else:+ return "default"+ # API credentials are identifiers, not passwords to verify. A process-keyed+ # HMAC keeps bucket keys opaque without a password KDF on the request path.+ digest = hmac.digest(_OPENAI_RATE_KEY_SECRET, f"{kind}:{credential}".encode(), "sha256").hex()+ return f"{kind}:{digest}"def _response_ccr_hashes(messages: list[dict[str, Any]], markers: list[str]) -> list[str]:@@ -3566,7 +3592,7 @@ class OpenAIHandlerMixin:# Rate limitingif self.rate_limiter:- rate_key = headers.get("authorization", "default")[:20]+ rate_key = _openai_rate_limit_key(headers)allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)if not allowed:await self.metrics.record_rate_limited(@@ -5728,7 +5754,7 @@ class OpenAIHandlerMixin:# Rate limitingif self.rate_limiter:- rate_key = headers.get("authorization", "default")[:20]+ rate_key = _openai_rate_limit_key(headers)allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)if not allowed:await self.metrics.record_rate_limited(provider="openai", source="headroom")diff --git a/tests/test_proxy_openai_rate_limit_key.py b/tests/test_proxy_openai_rate_limit_key.pynew file mode 100644index 000000000..bd1801759--- /dev/null+++ b/tests/test_proxy_openai_rate_limit_key.py@@ -0,0 +1,97 @@+"""Regression coverage for OpenAI credential-scoped rate limiting (#3364)."""++from __future__ import annotations++from unittest.mock import AsyncMock++import httpx+import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.handlers.openai import _openai_rate_limit_key # noqa: E402+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402+++def test_openai_rate_limit_key_accepts_api_key_header() -> None:+ assert _openai_rate_limit_key({"api-key": "gateway-key-A"}) != _openai_rate_limit_key(+ {"api-key": "gateway-key-B"}+ )+++def test_openai_rate_limit_key_uses_complete_credential() -> None:+ shared_prefix = "a" * 20+ assert _openai_rate_limit_key({"api-key": f"{shared_prefix}-A"}) != _openai_rate_limit_key(+ {"api-key": f"{shared_prefix}-B"}+ )+++def test_openai_rate_limit_key_prefers_authorization() -> None:+ headers = {"authorization": "Bearer primary", "api-key": "secondary"}+ assert _openai_rate_limit_key(headers) == _openai_rate_limit_key(+ {"authorization": "Bearer primary"}+ )++[email protected]("endpoint", ["chat", "responses"])+def test_distinct_api_keys_do_not_share_openai_rate_limit_bucket(monkeypatch, endpoint) -> None:+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=True,+ rate_limit_requests_per_minute=1,+ rate_limit_tokens_per_minute=100_000,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ retry_enabled=False,+ )++ if endpoint == "chat":+ path = "/v1/chat/completions"+ body = {+ "model": "gpt-4o-mini",+ "messages": [{"role": "user", "content": "hello"}],+ "stream": False,+ }+ upstream_body = {+ "id": "chatcmpl-test",+ "object": "chat.completion",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},+ }+ else:+ path = "/v1/responses"+ body = {"model": "gpt-4o-mini", "input": "hello", "stream": False}+ upstream_body = {+ "id": "resp-test",+ "object": "response",+ "status": "completed",+ "output": [],+ "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4},+ }++ with TestClient(create_app(config)) as client:+ proxy = client.app.state.proxy+ proxy._retry_request = AsyncMock(+ side_effect=lambda *args, **kwargs: httpx.Response(200, json=upstream_body)+ )++ first = client.post(path, headers={"api-key": "gateway-key-A"}, json=body)+ second = client.post(path, headers={"api-key": "gateway-key-B"}, json=body)++ assert first.status_code == 200+ assert second.status_code == 200+ assert proxy._retry_request.await_count == 2