Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(security): stop forwarding the proxy token to upstream providers (#3891)

Satya Bharadwaj0147cf02207f
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx
index d08755278..f29891899 100644
--- a/docs/content/docs/proxy.mdx
+++ b/docs/content/docs/proxy.mdx
@@ -338,7 +338,7 @@ with a privileged job.
| Flag / env | Default | Effect |
|---|---|---|
-| `HEADROOM_PROXY_TOKEN` | none | Require a bearer token (`X-Headroom-Proxy-Token`) from non-loopback callers. |
+| `HEADROOM_PROXY_TOKEN` | none | Require a token from non-loopback callers, as `X-Headroom-Proxy-Token: <token>` or `Authorization: Bearer <token>`. The token is removed from the request before it is forwarded upstream; clients that also send a provider credential should use `X-Headroom-Proxy-Token` and keep `Authorization` for the provider. |
| `HEADROOM_COMPRESS_ALLOW_REMOTE` | `false` | Allow non-loopback callers to reach [`POST /v1/compress`](#post-v1compress) and `POST /v1/usage`. Required to run Headroom as a remote gateway/sidecar; without it remote callers get `404`. |
| `--offline` / `HEADROOM_OFFLINE` | `false` | Air-gap mode: hard-disable **all** egress (telemetry, update checks, license reporting, model downloads). |
| `HEADROOM_LICENSE` | unset | Licence token, read by core and every licensed extension. `HEADROOM_LICENSE_KEY` is a deprecated alias for one release and logs a warning. A licence on its own sends nothing to Headroom. |
diff --git a/docs/metrics-technical-guide.md b/docs/metrics-technical-guide.md
index b5ff7e152..d1dc3db6c 100644
--- a/docs/metrics-technical-guide.md
+++ b/docs/metrics-technical-guide.md
@@ -227,7 +227,7 @@ These rows *explain* the headline total — they are never added to it.
4. **A 5xx erases its own savings.** Requests that fail upstream are dropped from every savings and token counter. During a provider incident, savings rates look artificially clean while throughput falls.
-5. **`/metrics` needs auth if you set a proxy token.** With `HEADROOM_PROXY_TOKEN` set, any non-loopback scraper must send `Authorization: Bearer <token>`. Loopback is always exempt.
+5. **`/metrics` needs auth if you set a proxy token.** With `HEADROOM_PROXY_TOKEN` set, any non-loopback scraper must send `Authorization: Bearer <token>` (or `X-Headroom-Proxy-Token: <token>`). Loopback is always exempt. The proxy removes the token from every request before forwarding it, so it never reaches a model provider.
---
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 91098e718..d5b758b0e 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -2868,6 +2868,60 @@ def read_proxy_token(headers: Mapping[str, str]) -> str | None:
return None
+_PROXY_TOKEN_HEADER = b"x-headroom-proxy-token"
+
+
+def _is_proxy_token_bearer(value: bytes, token_bytes: bytes) -> bool:
+ """Whether a raw ``Authorization`` value is ``Bearer <proxy token>``.
+
+ Parses exactly as :func:`read_proxy_token` does (case-insensitive scheme,
+ stripped credential) and compares the way the gates do, so a value the gate
+ accepted as the proxy credential is always recognised here.
+ """
+ auth = value.decode("latin-1")
+ if not auth.lower().startswith("bearer "):
+ return False
+ credential = auth[7:].strip()
+ return bool(credential) and hmac.compare_digest(
+ credential.encode("utf-8", "replace"), token_bytes
+ )
+
+
+def scrub_proxy_token_headers(scope: Any, token_bytes: bytes) -> None:
+ """Remove the proxy credential from an ASGI scope before any handler reads it.
+
+ The proxy token authenticates the caller to Headroom and nothing else, but
+ the handlers build their upstream headers from the inbound request and only
+ drop ``x-headroom-*`` names. A token sent as ``Authorization: Bearer <token>``
+ (the documented scraper form, and what the TypeScript SDK sends when it has
+ no provider credential) was therefore forwarded to Anthropic/OpenAI. Doing
+ this once, on the scope, covers every handler and transport at once instead
+ of relying on each upstream call site to remember.
+
+ ``x-headroom-proxy-token`` is dropped unconditionally, so it cannot reach an
+ upstream even with ``HEADROOM_STRIP_INTERNAL_HEADERS=disabled``. An
+ ``Authorization`` header is dropped only when it carries the configured
+ token; any other value is a provider credential and is left alone.
+ """
+ headers = scope.get("headers")
+ if not headers:
+ return
+ kept = [
+ (name, value)
+ for name, value in headers
+ if not (
+ name.lower() == _PROXY_TOKEN_HEADER
+ or (
+ token_bytes
+ and name.lower() == b"authorization"
+ and _is_proxy_token_bearer(value, token_bytes)
+ )
+ )
+ ]
+ if len(kept) != len(headers):
+ scope["headers"] = kept
+
+
class WebSocketAuthMiddleware:
"""Enforce ``HEADROOM_PROXY_TOKEN`` on WebSocket handshakes.
@@ -2900,13 +2954,18 @@ class WebSocketAuthMiddleware:
self.token_bytes = proxy_token.encode("utf-8") if proxy_token else b""
async def __call__(self, scope: Any, receive: Any, send: Any) -> None:
- if scope["type"] != "websocket" or not self.proxy_token:
+ if scope["type"] != "websocket":
+ await self.app(scope, receive, send)
+ return
+ if not self.proxy_token:
+ scrub_proxy_token_headers(scope, self.token_bytes)
await self.app(scope, receive, send)
return
client = scope.get("client")
client_host = client[0] if client else None
if is_loopback_host(client_host):
+ scrub_proxy_token_headers(scope, self.token_bytes)
await self.app(scope, receive, send)
return
@@ -2921,6 +2980,7 @@ class WebSocketAuthMiddleware:
if provided is not None and hmac.compare_digest(
provided.encode("utf-8", "replace"), self.token_bytes
):
+ scrub_proxy_token_headers(scope, self.token_bytes)
await self.app(scope, receive, send)
return
@@ -3973,6 +4033,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
_apply_security_headers(rejection)
return rejection
+ # The credential has done its job; take it off the request so no handler
+ # can forward it upstream. Runs on the exempt paths too, since a loopback
+ # caller's token is no more the provider's business than a remote one's.
+ scrub_proxy_token_headers(request.scope, _proxy_token_bytes)
response = await call_next(request)
_apply_security_headers(response)
diff --git a/tests/test_proxy_hardening.py b/tests/test_proxy_hardening.py
index 9542db6cf..efe277946 100644
--- a/tests/test_proxy_hardening.py
+++ b/tests/test_proxy_hardening.py
@@ -8,8 +8,10 @@
from __future__ import annotations
+import contextlib
import logging
+import httpx
import pytest
pytest.importorskip("fastapi")
@@ -19,7 +21,12 @@ from fastapi.testclient import TestClient
from headroom.cache.compression_store import reset_compression_store
from headroom.offline import apply_offline_env, is_offline
from headroom.proxy.audit import is_auditable_path
-from headroom.proxy.server import ProxyConfig, WebSocketAuthMiddleware, create_app
+from headroom.proxy.server import (
+ ProxyConfig,
+ WebSocketAuthMiddleware,
+ create_app,
+ scrub_proxy_token_headers,
+)
NONLOOPBACK = ("203.0.113.5", 44444) # TEST-NET-3, never loopback
LOOPBACK = ("127.0.0.1", 12345)
@@ -332,6 +339,279 @@ def _record_ws_handler_reached(app, monkeypatch):
return lambda: bool(seen)
+# ─────────────── 2.1c the proxy token never reaches an upstream ───────────
+
+
+TOKEN = "s3cr3t-token"
+
+
+class _CapturingTransport(httpx.AsyncBaseTransport):
+ """Upstream stand-in that records every request the proxy sends it."""
+
+ def __init__(self) -> None:
+ self.requests: list[httpx.Request] = []
+
+ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+ self.requests.append(request)
+ if "chat/completions" in str(request.url):
+ body = {
+ "id": "chatcmpl-mock",
+ "object": "chat.completion",
+ "created": 1,
+ "model": "gpt-4o",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "hi"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ }
+ elif "/v1/messages" in str(request.url):
+ body = {
+ "id": "msg_mock",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "hi"}],
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ }
+ else:
+ body = {"object": "list", "data": []}
+ return httpx.Response(200, headers={"content-type": "application/json"}, json=body)
+
+
+def _upstream_capturing_client(*, client=NONLOOPBACK, base_url="http://testserver"):
+ reset_compression_store()
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ anthropic_api_url="https://api.anthropic.test",
+ openai_api_url="https://api.openai.test",
+ proxy_token=TOKEN,
+ )
+ app = create_app(config)
+ transport = _CapturingTransport()
+ with TestClient(app, base_url=base_url, client=client) as c:
+ # Startup builds the real upstream client, so swap it only after
+ # entering the lifespan or requests would leave the test process.
+ app.state.proxy.http_client = httpx.AsyncClient(transport=transport)
+ yield c, transport
+
+
+def _anthropic_request(c, headers):
+ return c.post(
+ "/v1/messages",
+ headers={"anthropic-version": "2023-06-01", **headers},
+ json={
+ "model": "claude-3-5-sonnet-20241022",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "max_tokens": 16,
+ },
+ )
+
+
+def _only_upstream_request(transport) -> httpx.Request:
+ assert len(transport.requests) == 1, transport.requests
+ return transport.requests[0]
+
+
+class TestProxyTokenIsNotForwardedUpstream:
+ """The proxy credential authenticates to Headroom only.
+
+ Handlers build upstream headers from the inbound request and strip only
+ ``x-headroom-*``, so a token sent as ``Authorization: Bearer`` used to be
+ relayed to the provider verbatim. These tests capture what actually leaves
+ the proxy.
+ """
+
+ def test_bearer_token_is_dropped_and_provider_key_kept(self):
+ with _upstream_capturing_client() as (c, transport):
+ resp = _anthropic_request(
+ c, {"x-api-key": "sk-ant-test", "Authorization": f"Bearer {TOKEN}"}
+ )
+ assert resp.status_code == 200, resp.text
+
+ upstream = _only_upstream_request(transport)
+ assert "authorization" not in upstream.headers
+ assert upstream.headers["x-api-key"] == "sk-ant-test"
+ assert TOKEN not in str(upstream.headers.raw)
+
+ def test_lowercase_scheme_is_dropped_too(self):
+ """The gate accepts ``bearer`` in any case, so the scrub must as well."""
+ with _upstream_capturing_client() as (c, transport):
+ resp = _anthropic_request(
+ c, {"x-api-key": "sk-ant-test", "Authorization": f"bearer {TOKEN}"}
+ )
+ assert resp.status_code == 200, resp.text
+ assert "authorization" not in _only_upstream_request(transport).headers
+
+ def test_custom_header_is_dropped_and_provider_bearer_kept(self):
+ with _upstream_capturing_client() as (c, transport):
+ resp = c.post(
+ "/v1/chat/completions",
+ headers={"X-Headroom-Proxy-Token": TOKEN, "Authorization": "Bearer sk-test"},
+ json={"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]},
+ )
+ assert resp.status_code == 200, resp.text
+
+ upstream = _only_upstream_request(transport)
+ assert upstream.headers["authorization"] == "Bearer sk-test"
+ assert "x-headroom-proxy-token" not in upstream.headers
+
+ def test_custom_header_is_dropped_even_with_internal_strip_disabled(self, monkeypatch):
+ """``HEADROOM_STRIP_INTERNAL_HEADERS=disabled`` must not leak the credential."""
+ monkeypatch.setenv("HEADROOM_STRIP_INTERNAL_HEADERS", "disabled")
+ with _upstream_capturing_client() as (c, transport):
+ resp = _anthropic_request(
+ c, {"x-api-key": "sk-ant-test", "X-Headroom-Proxy-Token": TOKEN}
+ )
+ assert resp.status_code == 200, resp.text
+ assert "x-headroom-proxy-token" not in _only_upstream_request(transport).headers
+
+ def test_loopback_caller_token_is_dropped(self):
+ """Loopback skips the check, but its token is still not the provider's."""
+ with _upstream_capturing_client(client=LOOPBACK, base_url="http://127.0.0.1") as (
+ c,
+ transport,
+ ):
+ resp = _anthropic_request(
+ c, {"x-api-key": "sk-ant-test", "Authorization": f"Bearer {TOKEN}"}
+ )
+ assert resp.status_code == 200, resp.text
+ assert "authorization" not in _only_upstream_request(transport).headers
+
+ def test_catch_all_passthrough_drops_bearer_token(self):
+ with _upstream_capturing_client() as (c, transport):
+ resp = c.get(
+ "/v1/models",
+ headers={"x-api-key": "sk-ant-test", "Authorization": f"Bearer {TOKEN}"},
+ )
+ assert resp.status_code == 200, resp.text
+ upstream = _only_upstream_request(transport)
+ assert "authorization" not in upstream.headers
+ assert upstream.headers["x-api-key"] == "sk-ant-test"
+
+
+class TestScrubProxyTokenHeaders:
+ """The scope-level scrub both gates share."""
+
+ @staticmethod
+ def _scope(*headers):
+ return {"headers": [(k.encode("latin-1"), v.encode("latin-1")) for k, v in headers]}
+
+ def test_drops_matching_bearer_keeps_other_headers(self):
+ scope = self._scope(("authorization", f"Bearer {TOKEN}"), ("x-api-key", "sk-ant"))
+ scrub_proxy_token_headers(scope, TOKEN.encode())
+ assert scope["headers"] == [(b"x-api-key", b"sk-ant")]
+
+ def test_keeps_non_matching_bearer(self):
+ scope = self._scope(("authorization", "Bearer sk-provider"))
+ scrub_proxy_token_headers(scope, TOKEN.encode())
+ assert scope["headers"] == [(b"authorization", b"Bearer sk-provider")]
+
+ def test_keeps_non_bearer_authorization(self):
+ scope = self._scope(("authorization", f"Basic {TOKEN}"))
+ scrub_proxy_token_headers(scope, TOKEN.encode())
+ assert scope["headers"] == [(b"authorization", f"Basic {TOKEN}".encode())]
+
+ def test_duplicate_authorization_drops_only_the_token(self):
+ scope = self._scope(
+ ("authorization", f"Bearer {TOKEN}"),
+ ("authorization", "Bearer sk-provider"),
+ )
+ scrub_proxy_token_headers(scope, TOKEN.encode())
+ assert scope["headers"] == [(b"authorization", b"Bearer sk-provider")]
+
+ def test_custom_header_dropped_even_without_configured_token(self):
+ scope = self._scope(("x-headroom-proxy-token", "anything"), ("x-api-key", "sk-ant"))
+ scrub_proxy_token_headers(scope, b"")
+ assert scope["headers"] == [(b"x-api-key", b"sk-ant")]
+
+ def test_no_configured_token_leaves_authorization_alone(self):
+ scope = self._scope(("authorization", f"Bearer {TOKEN}"))
+ scrub_proxy_token_headers(scope, b"")
+ assert scope["headers"] == [(b"authorization", f"Bearer {TOKEN}".encode())]
+
+ def test_non_ascii_header_value_does_not_raise(self):
+ scope = {"headers": [(b"authorization", "Bearer café".encode("latin-1"))]}
+ scrub_proxy_token_headers(scope, TOKEN.encode())
+ assert len(scope["headers"]) == 1
+
+
+class _HeaderSpyApp:
+ """Downstream ASGI app that records the headers it was handed."""
+
+ def __init__(self) -> None:
+ self.headers: list[tuple[bytes, bytes]] | None = None
+
+ async def __call__(self, scope, receive, send) -> None:
+ self.headers = list(scope["headers"])
+
+
+class TestWebSocketScrubsProxyToken:
+ @pytest.mark.parametrize("client", [NONLOOPBACK, LOOPBACK])
+ async def test_bearer_token_removed_before_the_app(self, client):
+ downstream = _HeaderSpyApp()
+ mw = WebSocketAuthMiddleware(downstream, proxy_token=TOKEN)
+
+ await _drive(
+ mw,
+ _ws_scope(
+ client=client,
+ headers=[("authorization", f"Bearer {TOKEN}"), ("x-api-key", "sk-ant")],
+ ),
+ )
+
+ assert downstream.headers == [(b"x-api-key", b"sk-ant")]
+
+ async def test_custom_header_removed_provider_bearer_kept(self):
+ downstream = _HeaderSpyApp()
+ mw = WebSocketAuthMiddleware(downstream, proxy_token=TOKEN)
+
+ await _drive(
+ mw,
+ _ws_scope(
+ headers=[
+ ("authorization", "Bearer sk-provider"),
+ ("x-headroom-proxy-token", TOKEN),
+ ]
+ ),
+ )
+
+ assert downstream.headers == [(b"authorization", b"Bearer sk-provider")]
+
+ def test_responses_route_handler_never_sees_the_token(self, monkeypatch):
+ app = _make_app(proxy_token=TOKEN)
+ seen: list[dict[str, str]] = []
+
+ async def _responses_spy(websocket):
+ seen.append(dict(websocket.headers))
+ await websocket.close(code=1000)
+
+ monkeypatch.setattr(app.state.proxy, "handle_openai_responses_ws", _responses_spy)
+
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ try:
+ with c.websocket_connect(
+ "/v1/responses", headers={"Authorization": f"Bearer {TOKEN}"}
+ ):
+ pass
+ except Exception: # noqa: BLE001 - spy closes the socket immediately
+ pass
+
+ assert len(seen) == 1
+ assert "authorization" not in seen[0]
+
+
# ───────────────────────────── 3.1 security headers ───────────────────────