Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): scope OpenAI rate limits by api-key (#3365)

C0sm0catsb9e8462a3a84
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 466e2283d..be26a3b83 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -9,10 +9,12 @@ import asyncio
import contextlib
import copy
import hashlib
+import hmac
import json
import logging
import os
import re
+import secrets
import threading
import time
import uuid
@@ -121,6 +123,30 @@ _CCR_HASH_RE = re.compile(
r"(?:Retrieve (?:more|original): hash=|<<ccr:)([a-fA-F0-9]{12,24})(?=[^a-fA-F0-9]|$)"
)
_BARE_CCR_HASH_RE = re.compile(r"[a-fA-F0-9]{12,24}")
+_OPENAI_RATE_KEY_SECRET = secrets.token_bytes(32)
+
+
+def _openai_rate_limit_key(headers: dict[str, str]) -> str:
+ """Return the credential identity used by the OpenAI rate limiter.
+
+ OpenAI-compatible gateways may authenticate with either a bearer token or
+ an ``api-key`` header. Deriving an identity from the complete value keeps common
+ prefixes distinct without retaining recoverable credential material in
+ bucket keys. The secret is process-local, like the limiter state.
+ Requests without either retain the existing shared fallback bucket.
+ """
+ authorization = headers.get("authorization")
+ api_key = headers.get("api-key")
+ if authorization:
+ kind, credential = "authorization", authorization
+ elif api_key:
+ kind, credential = "api-key", api_key
+ else:
+ return "default"
+ # API credentials are identifiers, not passwords to verify. A process-keyed
+ # HMAC keeps bucket keys opaque without a password KDF on the request path.
+ digest = hmac.digest(_OPENAI_RATE_KEY_SECRET, f"{kind}:{credential}".encode(), "sha256").hex()
+ return f"{kind}:{digest}"
def _response_ccr_hashes(messages: list[dict[str, Any]], markers: list[str]) -> list[str]:
@@ -3566,7 +3592,7 @@ class OpenAIHandlerMixin:
# Rate limiting
if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
+ rate_key = _openai_rate_limit_key(headers)
allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
if not allowed:
await self.metrics.record_rate_limited(
@@ -5728,7 +5754,7 @@ class OpenAIHandlerMixin:
# Rate limiting
if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
+ rate_key = _openai_rate_limit_key(headers)
allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
if not allowed:
await self.metrics.record_rate_limited(provider="openai", source="headroom")
diff --git a/tests/test_proxy_openai_rate_limit_key.py b/tests/test_proxy_openai_rate_limit_key.py
new file mode 100644
index 000000000..bd1801759
--- /dev/null
+++ b/tests/test_proxy_openai_rate_limit_key.py
@@ -0,0 +1,97 @@
+"""Regression coverage for OpenAI credential-scoped rate limiting (#3364)."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.proxy.handlers.openai import _openai_rate_limit_key # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+
+def test_openai_rate_limit_key_accepts_api_key_header() -> None:
+ assert _openai_rate_limit_key({"api-key": "gateway-key-A"}) != _openai_rate_limit_key(
+ {"api-key": "gateway-key-B"}
+ )
+
+
+def test_openai_rate_limit_key_uses_complete_credential() -> None:
+ shared_prefix = "a" * 20
+ assert _openai_rate_limit_key({"api-key": f"{shared_prefix}-A"}) != _openai_rate_limit_key(
+ {"api-key": f"{shared_prefix}-B"}
+ )
+
+
+def test_openai_rate_limit_key_prefers_authorization() -> None:
+ headers = {"authorization": "Bearer primary", "api-key": "secondary"}
+ assert _openai_rate_limit_key(headers) == _openai_rate_limit_key(
+ {"authorization": "Bearer primary"}
+ )
+
+
[email protected]("endpoint", ["chat", "responses"])
+def test_distinct_api_keys_do_not_share_openai_rate_limit_bucket(monkeypatch, endpoint) -> None:
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=True,
+ rate_limit_requests_per_minute=1,
+ rate_limit_tokens_per_minute=100_000,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ retry_enabled=False,
+ )
+
+ if endpoint == "chat":
+ path = "/v1/chat/completions"
+ body = {
+ "model": "gpt-4o-mini",
+ "messages": [{"role": "user", "content": "hello"}],
+ "stream": False,
+ }
+ upstream_body = {
+ "id": "chatcmpl-test",
+ "object": "chat.completion",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
+ }
+ else:
+ path = "/v1/responses"
+ body = {"model": "gpt-4o-mini", "input": "hello", "stream": False}
+ upstream_body = {
+ "id": "resp-test",
+ "object": "response",
+ "status": "completed",
+ "output": [],
+ "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4},
+ }
+
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+ proxy._retry_request = AsyncMock(
+ side_effect=lambda *args, **kwargs: httpx.Response(200, json=upstream_body)
+ )
+
+ first = client.post(path, headers={"api-key": "gateway-key-A"}, json=body)
+ second = client.post(path, headers={"api-key": "gateway-key-B"}, json=body)
+
+ assert first.status_code == 200
+ assert second.status_code == 200
+ assert proxy._retry_request.await_count == 2