Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): strip upstream transport framing headers from Gemini response replays (#4605)

Chen Yufeiyangdfcac78119b7
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.py
index aceef8c62..beacd2bfb 100644
--- a/headroom/proxy/handlers/gemini.py
+++ b/headroom/proxy/handlers/gemini.py
@@ -27,6 +27,7 @@ from headroom.proxy.helpers import (
COMPRESSION_TIMEOUT_SECONDS,
extract_tags,
invalid_request_body_message,
+ sanitize_forwarded_response_headers,
)
from headroom.proxy.identity import resolve_memory_identity
from headroom.proxy.outcome import RequestOutcome
@@ -586,9 +587,7 @@ class GeminiHandlerMixin:
client=client,
)
)
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
+ response_headers = sanitize_forwarded_response_headers(response.headers)
response_headers["x-headroom-tokens-before"] = str(total_input_tokens)
response_headers["x-headroom-tokens-after"] = str(total_input_tokens)
response_headers["x-headroom-tokens-saved"] = "0"
@@ -1090,12 +1089,9 @@ class GeminiHandlerMixin:
else:
logger.info(f"[{request_id}] Gemini {model}: {original_tokens:,} tokens")
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- # Inject Headroom compression metrics (for SaaS metering)
+ # Remove transport framing headers (the body is re-framed here)
+ # and inject Headroom compression metrics (for SaaS metering).
+ response_headers = sanitize_forwarded_response_headers(response.headers)
response_headers["x-headroom-tokens-before"] = str(original_tokens)
response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
@@ -1118,9 +1114,7 @@ class GeminiHandlerMixin:
)
except _GeminiContinuationError as e:
await self.metrics.record_failed(provider=provider_name)
- response_headers = dict(e.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
+ response_headers = sanitize_forwarded_response_headers(e.headers)
return Response(content=e.content, status_code=e.status_code, headers=response_headers)
except Exception as e:
await self.metrics.record_failed(provider=provider_name)
@@ -1434,9 +1428,7 @@ class GeminiHandlerMixin:
url += f"?key={query_params['key']}"
response = await self._retry_request("POST", url, headers, body)
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
+ response_headers = sanitize_forwarded_response_headers(response.headers)
return Response(
content=response.content,
status_code=response.status_code,
@@ -1585,10 +1577,8 @@ class GeminiHandlerMixin:
f"[{request_id}] Gemini countTokens {model}: {compressed_tokens:,} tokens"
)
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
+ # Remove transport framing headers (the body is re-framed here).
+ response_headers = sanitize_forwarded_response_headers(response.headers)
return Response(
content=response.content,
diff --git a/tests/test_proxy/test_gemini_response_framing_headers.py b/tests/test_proxy/test_gemini_response_framing_headers.py
new file mode 100644
index 000000000..94cb8f736
--- /dev/null
+++ b/tests/test_proxy/test_gemini_response_framing_headers.py
@@ -0,0 +1,107 @@
+"""Gemini response replay must strip upstream transport framing headers.
+
+The Gemini generateContent / countTokens replay paths copied upstream headers
+by hand and popped only content-encoding / content-length. They therefore
+replayed headers the shared sanitize_forwarded_response_headers exists to drop:
+transfer-encoding from a chunked upstream, connection-local connection /
+keep-alive, and the upstream server banner.
+
+A stale transfer-encoding on an already-decoded, re-framed body is the #3019
+class of bug: RFC 9112 makes Transfer-Encoding override Content-Length, so the
+client parses plain JSON as chunked frames.
+"""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient
+
+from headroom.proxy.server import ProxyConfig, create_app
+
+_UPSTREAM_HEADERS = {
+ "content-type": "application/json",
+ "server": "upstream-marker/9.9",
+ "keep-alive": "timeout=5, max=100",
+ "connection": "keep-alive",
+ "transfer-encoding": "chunked",
+}
+
+_FRAMING_HEADERS = ("server", "keep-alive", "connection", "transfer-encoding")
+
+_GENERATE_BODY = json.dumps(
+ {
+ "candidates": [
+ {
+ "content": {"parts": [{"text": "hi"}], "role": "model"},
+ "finishReason": "STOP",
+ }
+ ],
+ "usageMetadata": {
+ "promptTokenCount": 4,
+ "candidatesTokenCount": 1,
+ "totalTokenCount": 5,
+ },
+ }
+).encode()
+
+_COUNT_TOKENS_BODY = json.dumps({"totalTokens": 4}).encode()
+
+
+def _client(body: bytes) -> TestClient:
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ image_optimize=False,
+ )
+ app = create_app(config)
+ proxy = app.state.proxy
+
+ async def _fake_retry(*_args: object, **_kwargs: object) -> httpx.Response:
+ return httpx.Response(200, headers=_UPSTREAM_HEADERS, content=body)
+
+ proxy._retry_request = _fake_retry # type: ignore[method-assign]
+ return TestClient(app)
+
+
+def _assert_upstream_framing_is_stripped(response: httpx.Response) -> None:
+ lower = {name.lower() for name in response.headers}
+ for header in _FRAMING_HEADERS:
+ assert header not in lower, (
+ f"upstream {header!r} reached the client: {dict(response.headers)}"
+ )
+ assert response.headers["content-type"].startswith("application/json")
+
+
+def test_generate_content_replay_strips_upstream_framing_headers() -> None:
+ client = _client(_GENERATE_BODY)
+
+ response = client.post(
+ "/v1beta/models/gemini-pro:generateContent?key=test-key",
+ json={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},
+ )
+
+ assert response.status_code == 200
+ assert response.json()["candidates"][0]["content"]["parts"][0]["text"] == "hi"
+ _assert_upstream_framing_is_stripped(response)
+
+
+def test_count_tokens_replay_strips_upstream_framing_headers() -> None:
+ client = _client(_COUNT_TOKENS_BODY)
+
+ response = client.post(
+ "/v1beta/models/gemini-pro:countTokens?key=test-key",
+ json={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},
+ )
+
+ assert response.status_code == 200
+ assert response.json()["totalTokens"] == 4
+ _assert_upstream_framing_is_stripped(response)