Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): record outcome when a passthrough stream ends mid-character (#4429)
HetCreep75cfbba91d9b
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 643d8c7f9..c9e7b0934 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -113,6 +113,7 @@ from headroom.proxy.buffered_ccr_response import (from headroom.proxy.compression_decision import CompressionDecisionfrom headroom.proxy.cost import header_safe_transformsfrom headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_value+from headroom.proxy.handlers.streaming import _stream_outcome_statusfrom headroom.proxy.image_isolation import run_image_compression_isolatedfrom headroom.proxy.outcome import RequestOutcomefrom headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for@@ -11759,10 +11760,20 @@ class OpenAIHandlerMixin:)yield chunkfinally:+ completed_normally = Truebuf = stream_state["sse_buffer"]if len(buf) > 0:buf.extend(b"\n\n")- _absorb_usage(self._parse_sse_usage_from_buffer(stream_state, stream_provider))+ # A truncated stream can end inside a multi-byte character;+ # the strict decoder then raises and would skip the outcome+ # below. Keep the usage already parsed, drop the unusable+ # tail, and report the stream as failed.+ try:+ _absorb_usage(+ self._parse_sse_usage_from_buffer(stream_state, stream_provider)+ )+ except UnicodeDecodeError:+ completed_normally = Falseoutput_tokens = stream_state["output_tokens"] or 0cache_read_tokens = stream_state["cache_read_input_tokens"] or 0@@ -11803,6 +11814,9 @@ class OpenAIHandlerMixin:total_latency_ms=(time.time() - start_time) * 1000,ttfb_ms=stream_state["ttfb_ms"] or 0,tags=tags,+ status_code=_stream_outcome_status(+ upstream_response.status_code, completed_normally+ ),client=client,))diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex fea3022c2..443615e70 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -22,6 +22,7 @@ from headroom.proxy.helpers import (_headroom_bypass_enabled,relocate_system_messages_to_top_level,)+from headroom.proxy.outcome import emit_request_outcomefrom headroom.proxy.server import HeadroomProxy@@ -849,6 +850,78 @@ def test_vertex_stream_passthrough_preserves_chunks_and_records_usage() -> None:assert outcome.cache_read_tokens == 2+def test_vertex_stream_passthrough_records_outcome_when_stream_ends_mid_character() -> None:+ closed: list[bool] = []++ class _CountingChunks(_AsyncChunks):+ async def aclose(self) -> None:+ closed.append(True)++ class _TruncatedClient(_VertexStreamClient):+ async def send(self, request, stream=False): # noqa: ANN001, ANN201+ return httpx.Response(+ 200,+ request=request,+ headers={"content-type": "text/event-stream"},+ stream=_CountingChunks(+ [+ b'data: {"usageMetadata":{"promptTokenCount":13,'+ b'"candidatesTokenCount":5}}\n\n',+ b'data: {"t":"\xe2\x82', # first 2 bytes of a 3-byte character+ ]+ ),+ )++ handler = object.__new__(HeadroomProxy)+ handler.http_client = _TruncatedClient()+ outcomes = []++ async def next_request_id(): # noqa: ANN202+ return "req_vertex_truncated"++ async def record(outcome): # noqa: ANN001, ANN202+ outcomes.append(outcome)++ handler._next_request_id = next_request_id+ handler._record_request_outcome = record++ response = asyncio.run(+ handler.handle_passthrough(+ _VertexStreamPassthroughRequest(),+ "https://vertex.test",+ "streamGenerateContent",+ "vertex:google",+ )+ )++ async def collect(): # noqa: ANN202+ return [chunk async for chunk in response.body_iterator]++ assert len(asyncio.run(collect())) == 2+ assert len(outcomes) == 1+ assert closed == [True]+ # Usage already received is kept, but a stream cut mid-character did not+ # complete: it takes the same failure status as other post-start stream+ # failures, so the outcome funnel books it as failed, not as a success.+ assert outcomes[0].optimized_tokens == 13+ assert outcomes[0].output_tokens == 5+ assert outcomes[0].status_code == 502++ failed: list[str] = []+ requested: list[dict] = []++ class _Metrics:+ async def record_failed(self, provider): # noqa: ANN001, ANN202+ failed.append(provider)++ async def record_request(self, **kwargs): # noqa: ANN003, ANN202+ requested.append(kwargs)++ asyncio.run(emit_request_outcome(SimpleNamespace(metrics=_Metrics()), outcomes[0]))+ assert failed == [outcomes[0].provider]+ assert requested == []++def test_stream_finalizer_records_vertex_provider_for_dashboard() -> None:handler = object.__new__(HeadroomProxy)handler.config = SimpleNamespace(log_full_messages=False)