Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): record a stream the client closes after its terminal event as completed (#4286)

gglucass762a337bbeba
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index f7dde7aca..da3943457 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -20,6 +20,7 @@ from headroom.proxy.helpers import (
RETRYABLE_OVERLOAD_STATUSES,
jitter_delay_ms,
overload_retry_is_futile,
+ parse_sse_events_from_byte_buffer,
retry_after_ms,
)
from headroom.proxy.provider_usage import (
@@ -1652,6 +1653,13 @@ class StreamingMixin:
full_sse_bytes = bytearray()
parsed_response = None # Set by memory block; used by CCR + prefix tracker
completed_normally = False
+ # Set when the client hangs up after the terminal event; it marks
+ # the outcome only, so pending messages are not drained into a
+ # stream nobody reads.
+ client_left_after_terminal = False
+ # A terminal event the client was handed, not just one parsed from
+ # upstream: the memory filter holds message_stop until the end.
+ client_got_terminal = False
pending_messages: list[dict] = []
# Proxy-injected memory tools are invisible to the client, so their
# tool_use blocks must never reach it (GH #2195).
@@ -1661,6 +1669,23 @@ class StreamingMixin:
else None
)
+ def _client_frame(frame: bytes) -> bytes:
+ """``_guard_client_bytes``, noting when the terminal event goes out."""
+ nonlocal client_got_terminal
+ guarded = _guard_client_bytes(frame)
+ if guarded and not client_got_terminal:
+ if memory_filter is None:
+ # Unfiltered, the client gets each whole chunk once it
+ # is parsed, so every event the parser has seen.
+ client_got_terminal = stream_state["stream_terminal"]
+ else:
+ # Filtered frames are complete events the filter decoded.
+ client_got_terminal = any(
+ _sse_event_outcome(name, data, stream_state["sse_dialect"])[1]
+ for name, data in parse_sse_events_from_byte_buffer(bytearray(frame))
+ )
+ return guarded
+
try:
async with contextlib.aclosing(upstream_response) as response:
sse_chunk_index = 0
@@ -1690,19 +1715,6 @@ class StreamingMixin:
tail = bytes(stream_state["sse_buffer"][-MAX_SSE_BUFFER_SIZE // 2 :])
stream_state["sse_buffer"] = bytearray(tail)
- # Always stream immediately — buffering breaks
- # real-time clients (LangGraph, LangChain, etc.)
- # The context guard runs last, on exactly what the
- # client receives (after the memory filter), and may
- # hold back bytes only until the first complete event.
- client_frames = (
- [chunk] if memory_filter is None else memory_filter.feed(chunk)
- )
- for frame in client_frames:
- guarded_frame = _guard_client_bytes(frame)
- if guarded_frame:
- yield guarded_frame
-
if _codex_wire_debug:
capture_codex_wire_debug(
"http_stream_upstream_chunk",
@@ -1735,8 +1747,14 @@ class StreamingMixin:
)
memory_enabled = False
- # Parse complete SSE events from buffer
- usage = self._parse_sse_usage_from_buffer(stream_state, provider)
+ # Parse complete SSE events from buffer. A failure (e.g.
+ # invalid UTF-8) is raised once the chunk is forwarded:
+ # malformed upstream content still reaches the client.
+ parse_error: Exception | None = None
+ try:
+ usage = self._parse_sse_usage_from_buffer(stream_state, provider)
+ except Exception as e:
+ usage, parse_error = None, e
if usage:
if "input_tokens" in usage:
stream_state["input_tokens"] = usage["input_tokens"]
@@ -1759,6 +1777,24 @@ class StreamingMixin:
"cache_creation_ephemeral_1h_input_tokens"
]
+ # Always stream immediately — buffering breaks
+ # real-time clients (LangGraph, LangChain, etc.)
+ # The context guard runs last, on exactly what the
+ # client receives (after the memory filter), and may
+ # hold back bytes only until the first complete event.
+ # Yielded only after the chunk is parsed: a client may
+ # hang up on the terminal event (Codex does), closing
+ # this generator at the yield.
+ client_frames = (
+ [chunk] if memory_filter is None else memory_filter.feed(chunk)
+ )
+ for frame in client_frames:
+ guarded_frame = _client_frame(frame)
+ if guarded_frame:
+ yield guarded_frame
+ if parse_error is not None:
+ raise parse_error
+
# Memory tool handling after stream completes
# Chunks were already yielded in real-time above, so we only
# do silent background processing here — no yielding.
@@ -1780,7 +1816,7 @@ class StreamingMixin:
)
if memory_filter is not None:
for frame in memory_filter.closing_frames():
- guarded_frame = _guard_client_bytes(frame)
+ guarded_frame = _client_frame(frame)
if guarded_frame:
yield guarded_frame
guarded_tail = _drain_context_guard()
@@ -1804,7 +1840,7 @@ class StreamingMixin:
stream_state=stream_state,
request_id=request_id,
):
- guarded_frame = _guard_client_bytes(frame)
+ guarded_frame = _client_frame(frame)
if guarded_frame:
yield guarded_frame
memory_filter = None
@@ -1849,7 +1885,7 @@ class StreamingMixin:
stream_state=stream_state,
request_id=request_id,
):
- guarded_frame = _guard_client_bytes(frame)
+ guarded_frame = _client_frame(frame)
if guarded_frame:
yield guarded_frame
@@ -1900,6 +1936,13 @@ class StreamingMixin:
stream_state["stream_terminal"] and not stream_state["stream_failed"]
)
+ except (GeneratorExit, asyncio.CancelledError):
+ # Codex closes the connection as soon as it reads
+ # response.completed, before the upstream EOF ends the loop.
+ client_left_after_terminal = (
+ client_got_terminal and not stream_state["stream_failed"]
+ )
+ raise
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
logger.error(f"[{request_id}] Connection error to upstream API: {e}")
guarded_tail = _drain_context_guard()
@@ -1985,7 +2028,8 @@ class StreamingMixin:
client=client,
waste_signals=waste_signals,
status_code=_stream_outcome_status(
- upstream_response.status_code, completed_normally
+ upstream_response.status_code,
+ completed_normally or client_left_after_terminal,
),
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
@@ -2143,7 +2187,6 @@ class StreamingMixin:
chunk_bytes = sse_line.encode()
if prefix_tracker is not None:
full_sse_bytes.extend(chunk_bytes)
- yield chunk_bytes
# Track usage from message_start event
if event.event_type == "message_start":
@@ -2188,9 +2231,15 @@ class StreamingMixin:
stream_terminal = stream_terminal or terminal
if failed:
logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
+ # After the bookkeeping: the client may hang up on the terminal event.
+ yield chunk_bytes
completed_normally = stream_terminal and not stream_failed
+ except (GeneratorExit, asyncio.CancelledError):
+ # A client that hung up after the terminal event got the whole stream.
+ completed_normally = stream_terminal and not stream_failed
+ raise
except Exception as e:
error_message = format_exception_message(e)
logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")
@@ -2410,6 +2459,13 @@ class StreamingMixin:
completed_normally = (
stream_state.get("stream_terminal", False) and not stream_failed
)
+ except (GeneratorExit, asyncio.CancelledError):
+ # A client that hung up after the terminal event got the whole stream.
+ stream_failed = stream_state.get("stream_failed", False)
+ completed_normally = (
+ stream_state.get("stream_terminal", False) and not stream_failed
+ )
+ raise
except Exception as e:
logger.error(f"[{request_id}] Backend streaming error: {e}")
error_data = public_errors.openai_error_body(
diff --git a/tests/test_route_advice.py b/tests/test_route_advice.py
index b61566f93..4d412f63b 100644
--- a/tests/test_route_advice.py
+++ b/tests/test_route_advice.py
@@ -405,6 +405,70 @@ def test_backend_clean_eof_requires_protocol_completion(provider, complete):
assert outcomes[0].status_code == (200 if complete else 502)
[email protected]("provider", ["anthropic", "openai"])
+def test_backend_client_hangup_after_terminal_event_is_completed(provider):
+ """A client may close on the terminal event, before the backend's EOF."""
+ from types import MethodType
+
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ class Backend(_Backend):
+ async def stream_message(self, body, headers):
+ yield SimpleNamespace(
+ event_type="message_stop",
+ raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',
+ data={"type": "message_stop"},
+ )
+ await asyncio.Event().wait()
+
+ async def stream_openai_message(self, body, headers):
+ yield "data: [DONE]\n\n"
+ await asyncio.Event().wait()
+
+ outcomes = []
+
+ async def record(outcome):
+ outcomes.append(outcome)
+
+ handler = _handler(Backend(provider))
+ handler._record_request_outcome = record
+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
+ handler._parse_sse_usage_from_buffer = MethodType(
+ StreamingMixin._parse_sse_usage_from_buffer, handler
+ )
+
+ async def drive():
+ arguments = {
+ "body": {"messages": []},
+ "headers": {},
+ "model": "m",
+ "request_id": "rid",
+ "original_tokens": 0,
+ "optimized_tokens": 0,
+ "tokens_saved": 0,
+ "transforms_applied": [],
+ "tags": {},
+ "optimization_latency": 0.0,
+ "backend": handler.anthropic_backend,
+ }
+ if provider == "anthropic":
+ response = await StreamingMixin._stream_response_bedrock(
+ handler, provider=provider, **arguments
+ )
+ else:
+ response = await StreamingMixin._stream_openai_via_backend(
+ handler, start_time=0.0, **arguments
+ )
+ body = response.body_iterator
+ while b"message_stop" not in (chunk := await body.__anext__()) and b"[DONE]" not in chunk:
+ pass
+ await body.aclose()
+
+ asyncio.run(drive())
+ assert len(outcomes) == 1
+ assert outcomes[0].status_code == 200
+
+
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
@pytest.mark.parametrize("complete", [False, True])
def test_backend_prefix_tracker_advances_only_after_completion(provider, complete):
diff --git a/tests/test_stream_client_hangup_after_terminal.py b/tests/test_stream_client_hangup_after_terminal.py
new file mode 100644
index 000000000..304bc68ef
--- /dev/null
+++ b/tests/test_stream_client_hangup_after_terminal.py
@@ -0,0 +1,227 @@
+"""A client that hangs up after the terminal event still got a complete stream.
+
+Codex CLI closes the connection as soon as it reads ``response.completed``,
+before the upstream EOF ends the proxy's read loop. The outcome was decided
+only after that loop, so every such turn recorded as a 502 failure with no
+usage and no cache metrics, although the client got a complete 200 response.
+Success needs the terminal event the client was handed, not one only parsed
+from upstream: the memory filter holds ``message_stop`` until after the stream.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+
+from headroom.proxy.server import HeadroomProxy
+
+
+def _event(name: str, payload: dict[str, Any]) -> bytes:
+ return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
+START = _event("response.created", {"type": "response.created", "response": {"id": "r1"}})
+COMPLETED = _event(
+ "response.completed",
+ {
+ "type": "response.completed",
+ "response": {
+ "id": "r1",
+ "status": "completed",
+ "usage": {
+ "input_tokens": 1_000,
+ "input_tokens_details": {"cached_tokens": 600, "cache_write_tokens": 0},
+ "output_tokens": 5,
+ },
+ },
+ },
+)
+
+# A whole Anthropic message in one upstream chunk: the parser sees message_stop
+# before the client gets the first frame the memory filter splits it into.
+MESSAGE = b"".join(
+ [
+ _event(
+ "message_start",
+ {
+ "type": "message_start",
+ "message": {
+ "id": "m1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-test",
+ "content": [],
+ "usage": {"input_tokens": 1_000, "output_tokens": 1},
+ },
+ },
+ ),
+ _event(
+ "content_block_start",
+ {"type": "content_block_start", "index": 0, "content_block": {"type": "text"}},
+ ),
+ _event(
+ "content_block_delta",
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "text_delta", "text": "ok"},
+ },
+ ),
+ _event("content_block_stop", {"type": "content_block_stop", "index": 0}),
+ _event(
+ "message_delta",
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": "end_turn"},
+ "usage": {"output_tokens": 5},
+ },
+ ),
+ _event("message_stop", {"type": "message_stop"}),
+ ]
+)
+MEMORY_STREAM = {
+ "url": "https://api.anthropic.com/v1/messages",
+ "headers": {"x-api-key": "sk-test"},
+ "body": {"model": "claude-test", "max_tokens": 100, "stream": True, "messages": []},
+ "provider": "anthropic",
+ "model": "claude-test",
+ "memory_user_id": "user-1",
+ "server_memory_tool_names": frozenset({"memory_save"}),
+}
+
+
+def _proxy(*chunks: bytes, upstream_eof: asyncio.Event | None = None) -> HeadroomProxy:
+ proxy = object.__new__(HeadroomProxy)
+ proxy.http_client = MagicMock(spec=httpx.AsyncClient)
+ proxy._config = MagicMock()
+ proxy._config.ccr_inject_tool = False
+ proxy._config.retry_max_attempts = 1
+ proxy.config = proxy._config
+ proxy._record_request_outcome = AsyncMock()
+
+ response = MagicMock()
+ response.headers = httpx.Headers({"content-type": "text/event-stream"})
+ response.status_code = 200
+
+ async def aiter_bytes():
+ for chunk in chunks or (START, COMPLETED):
+ yield chunk
+ if upstream_eof is not None:
+ await upstream_eof.wait() # the upstream EOF the client never waits for
+
+ response.aiter_bytes = aiter_bytes
+ response.aclose = AsyncMock()
+ proxy.http_client.build_request = MagicMock(return_value=MagicMock())
+ proxy.http_client.send = AsyncMock(return_value=response)
+ return proxy
+
+
+async def _stream(proxy: HeadroomProxy, **overrides: Any):
+ arguments: dict[str, Any] = {
+ "url": "https://chatgpt.com/backend-api/codex/responses",
+ "headers": {"authorization": "Bearer test"},
+ "body": {"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},
+ "provider": "openai",
+ "model": "gpt-5.4",
+ "request_id": "test-hangup",
+ "original_tokens": 10,
+ "optimized_tokens": 10,
+ "tokens_saved": 0,
+ "transforms_applied": [],
+ "tags": {},
+ "optimization_latency": 0.0,
+ }
+ if overrides.get("memory_user_id"):
+ proxy.memory_handler = MagicMock()
+ response = await proxy._stream_response(**{**arguments, **overrides})
+ return response.body_iterator
+
+
[email protected]("hangup", ["closed_at_terminal_yield", "cancelled_awaiting_eof"])
+async def test_client_hangup_after_terminal_event_records_success_with_usage(hangup):
+ proxy = _proxy(upstream_eof=asyncio.Event())
+ body = await _stream(proxy)
+ seen = b""
+ if hangup == "closed_at_terminal_yield":
+ # The generator is suspended at the yield of the terminal chunk.
+ while b"response.completed" not in seen:
+ seen += await body.__anext__()
+ await body.aclose()
+ else:
+ # The generator read past the terminal chunk and waits for EOF.
+ async def read_all() -> None:
+ nonlocal seen
+ async for chunk in body:
+ seen += chunk
+
+ reader = asyncio.create_task(read_all())
+ while b"response.completed" not in seen:
+ await asyncio.sleep(0)
+ reader.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await reader
+
+ proxy._record_request_outcome.assert_awaited_once()
+ outcome = proxy._record_request_outcome.await_args.args[0]
+ assert outcome.status_code == 200
+ assert outcome.provider_input_tokens == 1_000
+ assert outcome.cache_read_tokens == 600
+ assert outcome.output_tokens == 5
+
+
+async def test_client_hangup_before_terminal_event_is_still_a_failure():
+ proxy = _proxy(upstream_eof=asyncio.Event())
+ body = await _stream(proxy)
+ assert b"response.created" in await body.__anext__()
+ await body.aclose()
+
+ outcome = proxy._record_request_outcome.await_args.args[0]
+ assert outcome.status_code == 502
+
+
[email protected]("frames_read", [1, 2, 3, 4, 5])
+async def test_memory_filtered_hangup_before_the_client_gets_message_stop_is_a_failure(
+ frames_read,
+):
+ """Frames 1-4 come from the one upstream chunk; frame 5 is the held message_delta."""
+ proxy = _proxy(MESSAGE)
+ body = await _stream(proxy, **MEMORY_STREAM)
+ frames = [await body.__anext__() for _ in range(frames_read)]
+ assert b"message_start" in frames[0]
+ assert not any(b"message_stop" in frame for frame in frames)
+ await body.aclose()
+
+ outcome = proxy._record_request_outcome.await_args.args[0]
+ assert outcome.status_code == 502
+
+
+async def test_memory_filtered_hangup_after_message_stop_is_forwarded_records_success():
+ proxy = _proxy(MESSAGE)
+ body = await _stream(proxy, **MEMORY_STREAM)
+ while b"message_stop" not in await body.__anext__():
+ pass
+ await body.aclose()
+
+ outcome = proxy._record_request_outcome.await_args.args[0]
+ assert outcome.status_code == 200
+ assert outcome.provider_input_tokens == 1_000
+ assert outcome.output_tokens == 5
+
+
+async def test_malformed_event_after_a_valid_one_still_forwards_the_chunk():
+ """Invalid UTF-8 ends the stream with an error, after the upstream bytes."""
+ valid = _event("response.output_text.delta", {"type": "response.output_text.delta"})
+ malformed = b'event: response.output_text.delta\ndata: {"delta": "\xff"}\n\n'
+ proxy = _proxy(START, valid + malformed)
+ body = await _stream(proxy)
+ frames = [frame async for frame in body]
+
+ assert frames[:2] == [START, valid + malformed]
+ assert b"error" in frames[2]
+ outcome = proxy._record_request_outcome.await_args.args[0]
+ assert outcome.status_code == 502