Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): record outcome when a passthrough stream ends mid-character (#4429)

HetCreep75cfbba91d9b
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 643d8c7f9..c9e7b0934 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -113,6 +113,7 @@ from headroom.proxy.buffered_ccr_response import (
from headroom.proxy.compression_decision import CompressionDecision
from headroom.proxy.cost import header_safe_transforms
from headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_value
+from headroom.proxy.handlers.streaming import _stream_outcome_status
from headroom.proxy.image_isolation import run_image_compression_isolated
from headroom.proxy.outcome import RequestOutcome
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
@@ -11759,10 +11760,20 @@ class OpenAIHandlerMixin:
)
yield chunk
finally:
+ completed_normally = True
buf = stream_state["sse_buffer"]
if len(buf) > 0:
buf.extend(b"\n\n")
- _absorb_usage(self._parse_sse_usage_from_buffer(stream_state, stream_provider))
+ # A truncated stream can end inside a multi-byte character;
+ # the strict decoder then raises and would skip the outcome
+ # below. Keep the usage already parsed, drop the unusable
+ # tail, and report the stream as failed.
+ try:
+ _absorb_usage(
+ self._parse_sse_usage_from_buffer(stream_state, stream_provider)
+ )
+ except UnicodeDecodeError:
+ completed_normally = False
output_tokens = stream_state["output_tokens"] or 0
cache_read_tokens = stream_state["cache_read_input_tokens"] or 0
@@ -11803,6 +11814,9 @@ class OpenAIHandlerMixin:
total_latency_ms=(time.time() - start_time) * 1000,
ttfb_ms=stream_state["ttfb_ms"] or 0,
tags=tags,
+ status_code=_stream_outcome_status(
+ upstream_response.status_code, completed_normally
+ ),
client=client,
)
)
diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.py
index fea3022c2..443615e70 100644
--- a/tests/test_proxy_handler_helpers.py
+++ b/tests/test_proxy_handler_helpers.py
@@ -22,6 +22,7 @@ from headroom.proxy.helpers import (
_headroom_bypass_enabled,
relocate_system_messages_to_top_level,
)
+from headroom.proxy.outcome import emit_request_outcome
from headroom.proxy.server import HeadroomProxy
@@ -849,6 +850,78 @@ def test_vertex_stream_passthrough_preserves_chunks_and_records_usage() -> None:
assert outcome.cache_read_tokens == 2
+def test_vertex_stream_passthrough_records_outcome_when_stream_ends_mid_character() -> None:
+ closed: list[bool] = []
+
+ class _CountingChunks(_AsyncChunks):
+ async def aclose(self) -> None:
+ closed.append(True)
+
+ class _TruncatedClient(_VertexStreamClient):
+ async def send(self, request, stream=False): # noqa: ANN001, ANN201
+ return httpx.Response(
+ 200,
+ request=request,
+ headers={"content-type": "text/event-stream"},
+ stream=_CountingChunks(
+ [
+ b'data: {"usageMetadata":{"promptTokenCount":13,'
+ b'"candidatesTokenCount":5}}\n\n',
+ b'data: {"t":"\xe2\x82', # first 2 bytes of a 3-byte character
+ ]
+ ),
+ )
+
+ handler = object.__new__(HeadroomProxy)
+ handler.http_client = _TruncatedClient()
+ outcomes = []
+
+ async def next_request_id(): # noqa: ANN202
+ return "req_vertex_truncated"
+
+ async def record(outcome): # noqa: ANN001, ANN202
+ outcomes.append(outcome)
+
+ handler._next_request_id = next_request_id
+ handler._record_request_outcome = record
+
+ response = asyncio.run(
+ handler.handle_passthrough(
+ _VertexStreamPassthroughRequest(),
+ "https://vertex.test",
+ "streamGenerateContent",
+ "vertex:google",
+ )
+ )
+
+ async def collect(): # noqa: ANN202
+ return [chunk async for chunk in response.body_iterator]
+
+ assert len(asyncio.run(collect())) == 2
+ assert len(outcomes) == 1
+ assert closed == [True]
+ # Usage already received is kept, but a stream cut mid-character did not
+ # complete: it takes the same failure status as other post-start stream
+ # failures, so the outcome funnel books it as failed, not as a success.
+ assert outcomes[0].optimized_tokens == 13
+ assert outcomes[0].output_tokens == 5
+ assert outcomes[0].status_code == 502
+
+ failed: list[str] = []
+ requested: list[dict] = []
+
+ class _Metrics:
+ async def record_failed(self, provider): # noqa: ANN001, ANN202
+ failed.append(provider)
+
+ async def record_request(self, **kwargs): # noqa: ANN003, ANN202
+ requested.append(kwargs)
+
+ asyncio.run(emit_request_outcome(SimpleNamespace(metrics=_Metrics()), outcomes[0]))
+ assert failed == [outcomes[0].provider]
+ assert requested == []
+
+
def test_stream_finalizer_records_vertex_provider_for_dashboard() -> None:
handler = object.__new__(HeadroomProxy)
handler.config = SimpleNamespace(log_full_messages=False)