Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): handle ClientDisconnect in passthrough body reads (#2033)

GUOHAO LIU9db8a6bbf661
diff --git a/headroom/providers/proxy_routes.py b/headroom/providers/proxy_routes.py
index c81b76a4b..b6bd641c3 100644
--- a/headroom/providers/proxy_routes.py
+++ b/headroom/providers/proxy_routes.py
@@ -395,7 +395,13 @@ async def _handle_chatgpt_model_metadata(
if request.url.query:
url = f"{url}?{request.url.query}"
- body = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for passthrough")
+ return Response(status_code=204)
try:
assert proxy.http_client is not None
resp = await proxy.http_client.request(
@@ -435,7 +441,13 @@ async def _handle_chatgpt_codex_images(
if request.url.query:
url = f"{url}?{request.url.query}"
- body = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for passthrough")
+ return Response(status_code=204)
try:
client = getattr(proxy, "http_client_h1", None) or getattr(proxy, "http_client", None)
if client is None:
@@ -594,7 +606,13 @@ def register_provider_routes(app: FastAPI, proxy: Any) -> None:
if request.url.query:
url = f"{url}?{request.url.query}"
- body = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for codex responses passthrough")
+ return Response(status_code=204)
try:
assert proxy.http_client is not None
resp = await proxy.http_client.request(
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index d159ee98a..600a55727 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -3632,7 +3632,13 @@ class AnthropicHandlerMixin:
request_id=None,
)
- body = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for anthropic batch passthrough")
+ return Response(status_code=204)
response = await self.http_client.request( # type: ignore[union-attr]
method=request.method,
diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.py
index 9b66ccd23..07f74defa 100644
--- a/headroom/proxy/handlers/batch.py
+++ b/headroom/proxy/handlers/batch.py
@@ -393,7 +393,13 @@ class BatchHandlerMixin:
from headroom.proxy.helpers import log_outbound_request
if body is None:
- body_content = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body_content = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for google batch passthrough")
+ return Response(status_code=204)
outbound_source = "passthrough"
body_mutated = False
else:
@@ -502,7 +508,13 @@ class BatchHandlerMixin:
else:
url = f"{url}?key={api_key}"
- body = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for gemini passthrough")
+ return Response(status_code=204)
response = await self.http_client.request( # type: ignore[union-attr]
method=request.method,
diff --git a/headroom/proxy/handlers/bedrock.py b/headroom/proxy/handlers/bedrock.py
index e3e391f23..688e1befb 100644
--- a/headroom/proxy/handlers/bedrock.py
+++ b/headroom/proxy/handlers/bedrock.py
@@ -129,6 +129,11 @@ class BedrockHandlerMixin:
try:
body, raw = await read_request_json_with_bytes(request)
except Exception as err:
+ from starlette.requests import ClientDisconnect
+
+ if isinstance(err, ClientDisconnect):
+ logger.debug("[%s] %s client disconnected during body read", request_id, LOG_TAG)
+ return Response(status_code=204)
logger.warning(
"[%s] %s could not parse body; forwarding verbatim: %s",
request_id,
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 066337172..7fce06409 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -7400,7 +7400,13 @@ class OpenAIHandlerMixin:
request_id=None,
)
- body = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for passthrough")
+ return Response(status_code=204)
headers = await apply_copilot_api_auth(headers, url=url)
# Cloudflare bot-management challenges our HTTP/2 fingerprint on
@@ -7575,7 +7581,13 @@ class OpenAIHandlerMixin:
request_id=None,
)
- body = await request.body()
+ from starlette.requests import ClientDisconnect
+
+ try:
+ body = await request.body()
+ except ClientDisconnect:
+ logger.debug("Client disconnected during body read for streaming passthrough")
+ return Response(status_code=204)
headers = await apply_copilot_api_auth(headers, url=url)
request_id = await self._next_request_id()
stream_provider = "gemini" if provider == "vertex:google" else "anthropic"
diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.py
index 2609d5fa1..3fa3e23f6 100644
--- a/tests/test_proxy_handler_helpers.py
+++ b/tests/test_proxy_handler_helpers.py
@@ -1006,3 +1006,55 @@ def test_strict_frozen_count_tool_and_function_tail_are_mutable():
)
== 3
)
+
+
+class _ClientDisconnectRequest:
+ """Mock request whose body() raises ClientDisconnect to simulate mid-stream cancel."""
+
+ method = "POST"
+ headers = {"content-type": "application/json"}
+ url = SimpleNamespace(path="/v1/chat/completions", query="")
+
+ async def body(self) -> bytes:
+ from starlette.requests import ClientDisconnect
+
+ raise ClientDisconnect()
+
+
+class _ClientDisconnectStreamRequest:
+ """Mock request for streaming passthrough with ClientDisconnect."""
+
+ method = "POST"
+ headers = {"content-type": "application/json"}
+ url = SimpleNamespace(
+ path="/v1/projects/p/locations/us-central1/publishers/google/models/gemini-2.0-flash:streamGenerateContent",
+ query="alt=sse",
+ )
+
+ async def body(self) -> bytes:
+ from starlette.requests import ClientDisconnect
+
+ raise ClientDisconnect()
+
+
+def test_handle_passthrough_client_disconnect():
+ """ClientDisconnect during body read returns 204 instead of crashing TaskGroup."""
+ handler = object.__new__(OpenAIHandlerMixin)
+ response = asyncio.run(
+ handler.handle_passthrough(_ClientDisconnectRequest(), "https://api.openai.com")
+ )
+ assert response.status_code == 204
+
+
+def test_handle_streaming_passthrough_client_disconnect():
+ """ClientDisconnect during streaming body read returns 204."""
+ handler = object.__new__(OpenAIHandlerMixin)
+ response = asyncio.run(
+ handler.handle_passthrough(
+ _ClientDisconnectStreamRequest(),
+ "https://us-central1-aiplatform.googleapis.com",
+ endpoint_name="streamRawPredict",
+ provider="vertex:google",
+ )
+ )
+ assert response.status_code == 204