Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): record a stream the client closes after its terminal event as completed (#4286)
gglucass762a337bbeba
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex f7dde7aca..da3943457 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -20,6 +20,7 @@ from headroom.proxy.helpers import (RETRYABLE_OVERLOAD_STATUSES,jitter_delay_ms,overload_retry_is_futile,+ parse_sse_events_from_byte_buffer,retry_after_ms,)from headroom.proxy.provider_usage import (@@ -1652,6 +1653,13 @@ class StreamingMixin:full_sse_bytes = bytearray()parsed_response = None # Set by memory block; used by CCR + prefix trackercompleted_normally = False+ # Set when the client hangs up after the terminal event; it marks+ # the outcome only, so pending messages are not drained into a+ # stream nobody reads.+ client_left_after_terminal = False+ # A terminal event the client was handed, not just one parsed from+ # upstream: the memory filter holds message_stop until the end.+ client_got_terminal = Falsepending_messages: list[dict] = []# Proxy-injected memory tools are invisible to the client, so their# tool_use blocks must never reach it (GH #2195).@@ -1661,6 +1669,23 @@ class StreamingMixin:else None)+ def _client_frame(frame: bytes) -> bytes:+ """``_guard_client_bytes``, noting when the terminal event goes out."""+ nonlocal client_got_terminal+ guarded = _guard_client_bytes(frame)+ if guarded and not client_got_terminal:+ if memory_filter is None:+ # Unfiltered, the client gets each whole chunk once it+ # is parsed, so every event the parser has seen.+ client_got_terminal = stream_state["stream_terminal"]+ else:+ # Filtered frames are complete events the filter decoded.+ client_got_terminal = any(+ _sse_event_outcome(name, data, stream_state["sse_dialect"])[1]+ for name, data in parse_sse_events_from_byte_buffer(bytearray(frame))+ )+ return guarded+try:async with contextlib.aclosing(upstream_response) as response:sse_chunk_index = 0@@ -1690,19 +1715,6 @@ class StreamingMixin:tail = bytes(stream_state["sse_buffer"][-MAX_SSE_BUFFER_SIZE // 2 :])stream_state["sse_buffer"] = bytearray(tail)- # Always stream immediately — buffering breaks- # real-time clients (LangGraph, LangChain, etc.)- # The context guard runs last, on exactly what the- # client receives (after the memory filter), and may- # hold back bytes only until the first complete event.- client_frames = (- [chunk] if memory_filter is None else memory_filter.feed(chunk)- )- for frame in client_frames:- guarded_frame = _guard_client_bytes(frame)- if guarded_frame:- yield guarded_frame-if _codex_wire_debug:capture_codex_wire_debug("http_stream_upstream_chunk",@@ -1735,8 +1747,14 @@ class StreamingMixin:)memory_enabled = False- # Parse complete SSE events from buffer- usage = self._parse_sse_usage_from_buffer(stream_state, provider)+ # Parse complete SSE events from buffer. A failure (e.g.+ # invalid UTF-8) is raised once the chunk is forwarded:+ # malformed upstream content still reaches the client.+ parse_error: Exception | None = None+ try:+ usage = self._parse_sse_usage_from_buffer(stream_state, provider)+ except Exception as e:+ usage, parse_error = None, eif usage:if "input_tokens" in usage:stream_state["input_tokens"] = usage["input_tokens"]@@ -1759,6 +1777,24 @@ class StreamingMixin:"cache_creation_ephemeral_1h_input_tokens"]+ # Always stream immediately — buffering breaks+ # real-time clients (LangGraph, LangChain, etc.)+ # The context guard runs last, on exactly what the+ # client receives (after the memory filter), and may+ # hold back bytes only until the first complete event.+ # Yielded only after the chunk is parsed: a client may+ # hang up on the terminal event (Codex does), closing+ # this generator at the yield.+ client_frames = (+ [chunk] if memory_filter is None else memory_filter.feed(chunk)+ )+ for frame in client_frames:+ guarded_frame = _client_frame(frame)+ if guarded_frame:+ yield guarded_frame+ if parse_error is not None:+ raise parse_error+# Memory tool handling after stream completes# Chunks were already yielded in real-time above, so we only# do silent background processing here — no yielding.@@ -1780,7 +1816,7 @@ class StreamingMixin:)if memory_filter is not None:for frame in memory_filter.closing_frames():- guarded_frame = _guard_client_bytes(frame)+ guarded_frame = _client_frame(frame)if guarded_frame:yield guarded_frameguarded_tail = _drain_context_guard()@@ -1804,7 +1840,7 @@ class StreamingMixin:stream_state=stream_state,request_id=request_id,):- guarded_frame = _guard_client_bytes(frame)+ guarded_frame = _client_frame(frame)if guarded_frame:yield guarded_framememory_filter = None@@ -1849,7 +1885,7 @@ class StreamingMixin:stream_state=stream_state,request_id=request_id,):- guarded_frame = _guard_client_bytes(frame)+ guarded_frame = _client_frame(frame)if guarded_frame:yield guarded_frame@@ -1900,6 +1936,13 @@ class StreamingMixin:stream_state["stream_terminal"] and not stream_state["stream_failed"])+ except (GeneratorExit, asyncio.CancelledError):+ # Codex closes the connection as soon as it reads+ # response.completed, before the upstream EOF ends the loop.+ client_left_after_terminal = (+ client_got_terminal and not stream_state["stream_failed"]+ )+ raiseexcept (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:logger.error(f"[{request_id}] Connection error to upstream API: {e}")guarded_tail = _drain_context_guard()@@ -1985,7 +2028,8 @@ class StreamingMixin:client=client,waste_signals=waste_signals,status_code=_stream_outcome_status(- upstream_response.status_code, completed_normally+ upstream_response.status_code,+ completed_normally or client_left_after_terminal,),conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,@@ -2143,7 +2187,6 @@ class StreamingMixin:chunk_bytes = sse_line.encode()if prefix_tracker is not None:full_sse_bytes.extend(chunk_bytes)- yield chunk_bytes# Track usage from message_start eventif event.event_type == "message_start":@@ -2188,9 +2231,15 @@ class StreamingMixin:stream_terminal = stream_terminal or terminalif failed:logger.error(f"[{request_id}] Bedrock stream error: {event.data}")+ # After the bookkeeping: the client may hang up on the terminal event.+ yield chunk_bytescompleted_normally = stream_terminal and not stream_failed+ except (GeneratorExit, asyncio.CancelledError):+ # A client that hung up after the terminal event got the whole stream.+ completed_normally = stream_terminal and not stream_failed+ raiseexcept Exception as e:error_message = format_exception_message(e)logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")@@ -2410,6 +2459,13 @@ class StreamingMixin:completed_normally = (stream_state.get("stream_terminal", False) and not stream_failed)+ except (GeneratorExit, asyncio.CancelledError):+ # A client that hung up after the terminal event got the whole stream.+ stream_failed = stream_state.get("stream_failed", False)+ completed_normally = (+ stream_state.get("stream_terminal", False) and not stream_failed+ )+ raiseexcept Exception as e:logger.error(f"[{request_id}] Backend streaming error: {e}")error_data = public_errors.openai_error_body(diff --git a/tests/test_route_advice.py b/tests/test_route_advice.pyindex b61566f93..4d412f63b 100644--- a/tests/test_route_advice.py+++ b/tests/test_route_advice.py@@ -405,6 +405,70 @@ def test_backend_clean_eof_requires_protocol_completion(provider, complete):assert outcomes[0].status_code == (200 if complete else 502)[email protected]("provider", ["anthropic", "openai"])+def test_backend_client_hangup_after_terminal_event_is_completed(provider):+ """A client may close on the terminal event, before the backend's EOF."""+ from types import MethodType++ from headroom.proxy.handlers.streaming import StreamingMixin++ class Backend(_Backend):+ async def stream_message(self, body, headers):+ yield SimpleNamespace(+ event_type="message_stop",+ raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',+ data={"type": "message_stop"},+ )+ await asyncio.Event().wait()++ async def stream_openai_message(self, body, headers):+ yield "data: [DONE]\n\n"+ await asyncio.Event().wait()++ outcomes = []++ async def record(outcome):+ outcomes.append(outcome)++ handler = _handler(Backend(provider))+ handler._record_request_outcome = record+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)+ handler._parse_sse_usage_from_buffer = MethodType(+ StreamingMixin._parse_sse_usage_from_buffer, handler+ )++ async def drive():+ arguments = {+ "body": {"messages": []},+ "headers": {},+ "model": "m",+ "request_id": "rid",+ "original_tokens": 0,+ "optimized_tokens": 0,+ "tokens_saved": 0,+ "transforms_applied": [],+ "tags": {},+ "optimization_latency": 0.0,+ "backend": handler.anthropic_backend,+ }+ if provider == "anthropic":+ response = await StreamingMixin._stream_response_bedrock(+ handler, provider=provider, **arguments+ )+ else:+ response = await StreamingMixin._stream_openai_via_backend(+ handler, start_time=0.0, **arguments+ )+ body = response.body_iterator+ while b"message_stop" not in (chunk := await body.__anext__()) and b"[DONE]" not in chunk:+ pass+ await body.aclose()++ asyncio.run(drive())+ assert len(outcomes) == 1+ assert outcomes[0].status_code == 200++@pytest.mark.parametrize("provider", ["anthropic", "openai"])@pytest.mark.parametrize("complete", [False, True])def test_backend_prefix_tracker_advances_only_after_completion(provider, complete):diff --git a/tests/test_stream_client_hangup_after_terminal.py b/tests/test_stream_client_hangup_after_terminal.pynew file mode 100644index 000000000..304bc68ef--- /dev/null+++ b/tests/test_stream_client_hangup_after_terminal.py@@ -0,0 +1,227 @@+"""A client that hangs up after the terminal event still got a complete stream.++Codex CLI closes the connection as soon as it reads ``response.completed``,+before the upstream EOF ends the proxy's read loop. The outcome was decided+only after that loop, so every such turn recorded as a 502 failure with no+usage and no cache metrics, although the client got a complete 200 response.+Success needs the terminal event the client was handed, not one only parsed+from upstream: the memory filter holds ``message_stop`` until after the stream.+"""++from __future__ import annotations++import asyncio+import json+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import httpx+import pytest++from headroom.proxy.server import HeadroomProxy+++def _event(name: str, payload: dict[str, Any]) -> bytes:+ return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode()+++START = _event("response.created", {"type": "response.created", "response": {"id": "r1"}})+COMPLETED = _event(+ "response.completed",+ {+ "type": "response.completed",+ "response": {+ "id": "r1",+ "status": "completed",+ "usage": {+ "input_tokens": 1_000,+ "input_tokens_details": {"cached_tokens": 600, "cache_write_tokens": 0},+ "output_tokens": 5,+ },+ },+ },+)++# A whole Anthropic message in one upstream chunk: the parser sees message_stop+# before the client gets the first frame the memory filter splits it into.+MESSAGE = b"".join(+ [+ _event(+ "message_start",+ {+ "type": "message_start",+ "message": {+ "id": "m1",+ "type": "message",+ "role": "assistant",+ "model": "claude-test",+ "content": [],+ "usage": {"input_tokens": 1_000, "output_tokens": 1},+ },+ },+ ),+ _event(+ "content_block_start",+ {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}},+ ),+ _event(+ "content_block_delta",+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "text_delta", "text": "ok"},+ },+ ),+ _event("content_block_stop", {"type": "content_block_stop", "index": 0}),+ _event(+ "message_delta",+ {+ "type": "message_delta",+ "delta": {"stop_reason": "end_turn"},+ "usage": {"output_tokens": 5},+ },+ ),+ _event("message_stop", {"type": "message_stop"}),+ ]+)+MEMORY_STREAM = {+ "url": "https://api.anthropic.com/v1/messages",+ "headers": {"x-api-key": "sk-test"},+ "body": {"model": "claude-test", "max_tokens": 100, "stream": True, "messages": []},+ "provider": "anthropic",+ "model": "claude-test",+ "memory_user_id": "user-1",+ "server_memory_tool_names": frozenset({"memory_save"}),+}+++def _proxy(*chunks: bytes, upstream_eof: asyncio.Event | None = None) -> HeadroomProxy:+ proxy = object.__new__(HeadroomProxy)+ proxy.http_client = MagicMock(spec=httpx.AsyncClient)+ proxy._config = MagicMock()+ proxy._config.ccr_inject_tool = False+ proxy._config.retry_max_attempts = 1+ proxy.config = proxy._config+ proxy._record_request_outcome = AsyncMock()++ response = MagicMock()+ response.headers = httpx.Headers({"content-type": "text/event-stream"})+ response.status_code = 200++ async def aiter_bytes():+ for chunk in chunks or (START, COMPLETED):+ yield chunk+ if upstream_eof is not None:+ await upstream_eof.wait() # the upstream EOF the client never waits for++ response.aiter_bytes = aiter_bytes+ response.aclose = AsyncMock()+ proxy.http_client.build_request = MagicMock(return_value=MagicMock())+ proxy.http_client.send = AsyncMock(return_value=response)+ return proxy+++async def _stream(proxy: HeadroomProxy, **overrides: Any):+ arguments: dict[str, Any] = {+ "url": "https://chatgpt.com/backend-api/codex/responses",+ "headers": {"authorization": "Bearer test"},+ "body": {"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},+ "provider": "openai",+ "model": "gpt-5.4",+ "request_id": "test-hangup",+ "original_tokens": 10,+ "optimized_tokens": 10,+ "tokens_saved": 0,+ "transforms_applied": [],+ "tags": {},+ "optimization_latency": 0.0,+ }+ if overrides.get("memory_user_id"):+ proxy.memory_handler = MagicMock()+ response = await proxy._stream_response(**{**arguments, **overrides})+ return response.body_iterator++[email protected]("hangup", ["closed_at_terminal_yield", "cancelled_awaiting_eof"])+async def test_client_hangup_after_terminal_event_records_success_with_usage(hangup):+ proxy = _proxy(upstream_eof=asyncio.Event())+ body = await _stream(proxy)+ seen = b""+ if hangup == "closed_at_terminal_yield":+ # The generator is suspended at the yield of the terminal chunk.+ while b"response.completed" not in seen:+ seen += await body.__anext__()+ await body.aclose()+ else:+ # The generator read past the terminal chunk and waits for EOF.+ async def read_all() -> None:+ nonlocal seen+ async for chunk in body:+ seen += chunk++ reader = asyncio.create_task(read_all())+ while b"response.completed" not in seen:+ await asyncio.sleep(0)+ reader.cancel()+ with pytest.raises(asyncio.CancelledError):+ await reader++ proxy._record_request_outcome.assert_awaited_once()+ outcome = proxy._record_request_outcome.await_args.args[0]+ assert outcome.status_code == 200+ assert outcome.provider_input_tokens == 1_000+ assert outcome.cache_read_tokens == 600+ assert outcome.output_tokens == 5+++async def test_client_hangup_before_terminal_event_is_still_a_failure():+ proxy = _proxy(upstream_eof=asyncio.Event())+ body = await _stream(proxy)+ assert b"response.created" in await body.__anext__()+ await body.aclose()++ outcome = proxy._record_request_outcome.await_args.args[0]+ assert outcome.status_code == 502++[email protected]("frames_read", [1, 2, 3, 4, 5])+async def test_memory_filtered_hangup_before_the_client_gets_message_stop_is_a_failure(+ frames_read,+):+ """Frames 1-4 come from the one upstream chunk; frame 5 is the held message_delta."""+ proxy = _proxy(MESSAGE)+ body = await _stream(proxy, **MEMORY_STREAM)+ frames = [await body.__anext__() for _ in range(frames_read)]+ assert b"message_start" in frames[0]+ assert not any(b"message_stop" in frame for frame in frames)+ await body.aclose()++ outcome = proxy._record_request_outcome.await_args.args[0]+ assert outcome.status_code == 502+++async def test_memory_filtered_hangup_after_message_stop_is_forwarded_records_success():+ proxy = _proxy(MESSAGE)+ body = await _stream(proxy, **MEMORY_STREAM)+ while b"message_stop" not in await body.__anext__():+ pass+ await body.aclose()++ outcome = proxy._record_request_outcome.await_args.args[0]+ assert outcome.status_code == 200+ assert outcome.provider_input_tokens == 1_000+ assert outcome.output_tokens == 5+++async def test_malformed_event_after_a_valid_one_still_forwards_the_chunk():+ """Invalid UTF-8 ends the stream with an error, after the upstream bytes."""+ valid = _event("response.output_text.delta", {"type": "response.output_text.delta"})+ malformed = b'event: response.output_text.delta\ndata: {"delta": "\xff"}\n\n'+ proxy = _proxy(START, valid + malformed)+ body = await _stream(proxy)+ frames = [frame async for frame in body]++ assert frames[:2] == [START, valid + malformed]+ assert b"error" in frames[2]+ outcome = proxy._record_request_outcome.await_args.args[0]+ assert outcome.status_code == 502