Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): handle ClientDisconnect in passthrough body reads (#2033)
GUOHAO LIU9db8a6bbf661
diff --git a/headroom/providers/proxy_routes.py b/headroom/providers/proxy_routes.pyindex c81b76a4b..b6bd641c3 100644--- a/headroom/providers/proxy_routes.py+++ b/headroom/providers/proxy_routes.py@@ -395,7 +395,13 @@ async def _handle_chatgpt_model_metadata(if request.url.query:url = f"{url}?{request.url.query}"- body = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for passthrough")+ return Response(status_code=204)try:assert proxy.http_client is not Noneresp = await proxy.http_client.request(@@ -435,7 +441,13 @@ async def _handle_chatgpt_codex_images(if request.url.query:url = f"{url}?{request.url.query}"- body = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for passthrough")+ return Response(status_code=204)try:client = getattr(proxy, "http_client_h1", None) or getattr(proxy, "http_client", None)if client is None:@@ -594,7 +606,13 @@ def register_provider_routes(app: FastAPI, proxy: Any) -> None:if request.url.query:url = f"{url}?{request.url.query}"- body = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for codex responses passthrough")+ return Response(status_code=204)try:assert proxy.http_client is not Noneresp = await proxy.http_client.request(diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex d159ee98a..600a55727 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -3632,7 +3632,13 @@ class AnthropicHandlerMixin:request_id=None,)- body = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for anthropic batch passthrough")+ return Response(status_code=204)response = await self.http_client.request( # type: ignore[union-attr]method=request.method,diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.pyindex 9b66ccd23..07f74defa 100644--- a/headroom/proxy/handlers/batch.py+++ b/headroom/proxy/handlers/batch.py@@ -393,7 +393,13 @@ class BatchHandlerMixin:from headroom.proxy.helpers import log_outbound_requestif body is None:- body_content = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body_content = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for google batch passthrough")+ return Response(status_code=204)outbound_source = "passthrough"body_mutated = Falseelse:@@ -502,7 +508,13 @@ class BatchHandlerMixin:else:url = f"{url}?key={api_key}"- body = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for gemini passthrough")+ return Response(status_code=204)response = await self.http_client.request( # type: ignore[union-attr]method=request.method,diff --git a/headroom/proxy/handlers/bedrock.py b/headroom/proxy/handlers/bedrock.pyindex e3e391f23..688e1befb 100644--- a/headroom/proxy/handlers/bedrock.py+++ b/headroom/proxy/handlers/bedrock.py@@ -129,6 +129,11 @@ class BedrockHandlerMixin:try:body, raw = await read_request_json_with_bytes(request)except Exception as err:+ from starlette.requests import ClientDisconnect++ if isinstance(err, ClientDisconnect):+ logger.debug("[%s] %s client disconnected during body read", request_id, LOG_TAG)+ return Response(status_code=204)logger.warning("[%s] %s could not parse body; forwarding verbatim: %s",request_id,diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 066337172..7fce06409 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -7400,7 +7400,13 @@ class OpenAIHandlerMixin:request_id=None,)- body = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for passthrough")+ return Response(status_code=204)headers = await apply_copilot_api_auth(headers, url=url)# Cloudflare bot-management challenges our HTTP/2 fingerprint on@@ -7575,7 +7581,13 @@ class OpenAIHandlerMixin:request_id=None,)- body = await request.body()+ from starlette.requests import ClientDisconnect++ try:+ body = await request.body()+ except ClientDisconnect:+ logger.debug("Client disconnected during body read for streaming passthrough")+ return Response(status_code=204)headers = await apply_copilot_api_auth(headers, url=url)request_id = await self._next_request_id()stream_provider = "gemini" if provider == "vertex:google" else "anthropic"diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex 2609d5fa1..3fa3e23f6 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -1006,3 +1006,55 @@ def test_strict_frozen_count_tool_and_function_tail_are_mutable():)== 3)+++class _ClientDisconnectRequest:+ """Mock request whose body() raises ClientDisconnect to simulate mid-stream cancel."""++ method = "POST"+ headers = {"content-type": "application/json"}+ url = SimpleNamespace(path="/v1/chat/completions", query="")++ async def body(self) -> bytes:+ from starlette.requests import ClientDisconnect++ raise ClientDisconnect()+++class _ClientDisconnectStreamRequest:+ """Mock request for streaming passthrough with ClientDisconnect."""++ method = "POST"+ headers = {"content-type": "application/json"}+ url = SimpleNamespace(+ path="/v1/projects/p/locations/us-central1/publishers/google/models/gemini-2.0-flash:streamGenerateContent",+ query="alt=sse",+ )++ async def body(self) -> bytes:+ from starlette.requests import ClientDisconnect++ raise ClientDisconnect()+++def test_handle_passthrough_client_disconnect():+ """ClientDisconnect during body read returns 204 instead of crashing TaskGroup."""+ handler = object.__new__(OpenAIHandlerMixin)+ response = asyncio.run(+ handler.handle_passthrough(_ClientDisconnectRequest(), "https://api.openai.com")+ )+ assert response.status_code == 204+++def test_handle_streaming_passthrough_client_disconnect():+ """ClientDisconnect during streaming body read returns 204."""+ handler = object.__new__(OpenAIHandlerMixin)+ response = asyncio.run(+ handler.handle_passthrough(+ _ClientDisconnectStreamRequest(),+ "https://us-central1-aiplatform.googleapis.com",+ endpoint_name="streamRawPredict",+ provider="vertex:google",+ )+ )+ assert response.status_code == 204