Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): strip upstream transport framing headers from Gemini response replays (#4605)
Chen Yufeiyangdfcac78119b7
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex aceef8c62..beacd2bfb 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -27,6 +27,7 @@ from headroom.proxy.helpers import (COMPRESSION_TIMEOUT_SECONDS,extract_tags,invalid_request_body_message,+ sanitize_forwarded_response_headers,)from headroom.proxy.identity import resolve_memory_identityfrom headroom.proxy.outcome import RequestOutcome@@ -586,9 +587,7 @@ class GeminiHandlerMixin:client=client,))- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)+ response_headers = sanitize_forwarded_response_headers(response.headers)response_headers["x-headroom-tokens-before"] = str(total_input_tokens)response_headers["x-headroom-tokens-after"] = str(total_input_tokens)response_headers["x-headroom-tokens-saved"] = "0"@@ -1090,12 +1089,9 @@ class GeminiHandlerMixin:else:logger.info(f"[{request_id}] Gemini {model}: {original_tokens:,} tokens")- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- # Inject Headroom compression metrics (for SaaS metering)+ # Remove transport framing headers (the body is re-framed here)+ # and inject Headroom compression metrics (for SaaS metering).+ response_headers = sanitize_forwarded_response_headers(response.headers)response_headers["x-headroom-tokens-before"] = str(original_tokens)response_headers["x-headroom-tokens-after"] = str(optimized_tokens)response_headers["x-headroom-tokens-saved"] = str(tokens_saved)@@ -1118,9 +1114,7 @@ class GeminiHandlerMixin:)except _GeminiContinuationError as e:await self.metrics.record_failed(provider=provider_name)- response_headers = dict(e.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)+ response_headers = sanitize_forwarded_response_headers(e.headers)return Response(content=e.content, status_code=e.status_code, headers=response_headers)except Exception as e:await self.metrics.record_failed(provider=provider_name)@@ -1434,9 +1428,7 @@ class GeminiHandlerMixin:url += f"?key={query_params['key']}"response = await self._retry_request("POST", url, headers, body)- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)+ response_headers = sanitize_forwarded_response_headers(response.headers)return Response(content=response.content,status_code=response.status_code,@@ -1585,10 +1577,8 @@ class GeminiHandlerMixin:f"[{request_id}] Gemini countTokens {model}: {compressed_tokens:,} tokens")- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)+ # Remove transport framing headers (the body is re-framed here).+ response_headers = sanitize_forwarded_response_headers(response.headers)return Response(content=response.content,diff --git a/tests/test_proxy/test_gemini_response_framing_headers.py b/tests/test_proxy/test_gemini_response_framing_headers.pynew file mode 100644index 000000000..94cb8f736--- /dev/null+++ b/tests/test_proxy/test_gemini_response_framing_headers.py@@ -0,0 +1,107 @@+"""Gemini response replay must strip upstream transport framing headers.++The Gemini generateContent / countTokens replay paths copied upstream headers+by hand and popped only content-encoding / content-length. They therefore+replayed headers the shared sanitize_forwarded_response_headers exists to drop:+transfer-encoding from a chunked upstream, connection-local connection /+keep-alive, and the upstream server banner.++A stale transfer-encoding on an already-decoded, re-framed body is the #3019+class of bug: RFC 9112 makes Transfer-Encoding override Content-Length, so the+client parses plain JSON as chunked frames.+"""++from __future__ import annotations++import json++import httpx+import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient++from headroom.proxy.server import ProxyConfig, create_app++_UPSTREAM_HEADERS = {+ "content-type": "application/json",+ "server": "upstream-marker/9.9",+ "keep-alive": "timeout=5, max=100",+ "connection": "keep-alive",+ "transfer-encoding": "chunked",+}++_FRAMING_HEADERS = ("server", "keep-alive", "connection", "transfer-encoding")++_GENERATE_BODY = json.dumps(+ {+ "candidates": [+ {+ "content": {"parts": [{"text": "hi"}], "role": "model"},+ "finishReason": "STOP",+ }+ ],+ "usageMetadata": {+ "promptTokenCount": 4,+ "candidatesTokenCount": 1,+ "totalTokenCount": 5,+ },+ }+).encode()++_COUNT_TOKENS_BODY = json.dumps({"totalTokens": 4}).encode()+++def _client(body: bytes) -> TestClient:+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ image_optimize=False,+ )+ app = create_app(config)+ proxy = app.state.proxy++ async def _fake_retry(*_args: object, **_kwargs: object) -> httpx.Response:+ return httpx.Response(200, headers=_UPSTREAM_HEADERS, content=body)++ proxy._retry_request = _fake_retry # type: ignore[method-assign]+ return TestClient(app)+++def _assert_upstream_framing_is_stripped(response: httpx.Response) -> None:+ lower = {name.lower() for name in response.headers}+ for header in _FRAMING_HEADERS:+ assert header not in lower, (+ f"upstream {header!r} reached the client: {dict(response.headers)}"+ )+ assert response.headers["content-type"].startswith("application/json")+++def test_generate_content_replay_strips_upstream_framing_headers() -> None:+ client = _client(_GENERATE_BODY)++ response = client.post(+ "/v1beta/models/gemini-pro:generateContent?key=test-key",+ json={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},+ )++ assert response.status_code == 200+ assert response.json()["candidates"][0]["content"]["parts"][0]["text"] == "hi"+ _assert_upstream_framing_is_stripped(response)+++def test_count_tokens_replay_strips_upstream_framing_headers() -> None:+ client = _client(_COUNT_TOKENS_BODY)++ response = client.post(+ "/v1beta/models/gemini-pro:countTokens?key=test-key",+ json={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},+ )++ assert response.status_code == 200+ assert response.json()["totalTokens"] == 4+ _assert_upstream_framing_is_stripped(response)