Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): classify failed streams as failed requests (#3747)
lorenzozane42d4f58b434b
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 198ec5d1c..84fa502e0 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -12,6 +12,7 @@ import loggingimport timefrom collections.abc import AsyncIteratorfrom typing import TYPE_CHECKING, Any+from urllib.parse import urlsplitfrom headroom.proxy.auth_mode import classify_client, supports_mid_turn_coalescingfrom headroom.proxy.handlers._debug_dump import write_upstream_error_dump@@ -40,6 +41,104 @@ from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokefrom headroom.utils import format_exception_messagelogger = logging.getLogger("headroom.proxy")+STREAM_FAILURE_STATUS = 502+++def _stream_outcome_status(status_code: int, completed_normally: bool) -> int:+ """Keep post-start stream failures out of the success counter."""+ return status_code if completed_normally else STREAM_FAILURE_STATUS+++def _gemini_response_data(data: Any) -> dict[str, Any]:+ if not isinstance(data, dict):+ return {}+ response = data.get("response")+ return response if isinstance(response, dict) else data+++def _gemini_finished_candidates(data: Any) -> set[int]:+ candidates = _gemini_response_data(data).get("candidates")+ finished: set[int] = set()+ if not isinstance(candidates, list):+ return finished+ for position, candidate in enumerate(candidates):+ if not isinstance(candidate, dict):+ continue+ reason = candidate.get("finishReason")+ index = candidate.get("index", position)+ if (+ isinstance(reason, str)+ and reason.strip()+ and reason != "FINISH_REASON_UNSPECIFIED"+ and isinstance(index, int)+ and not isinstance(index, bool)+ and index >= 0+ ):+ finished.add(index)+ return finished+++def _gemini_requested_candidates(body: dict[str, Any]) -> int:+ request = body.get("request")+ request = request if isinstance(request, dict) else body+ config = request.get("generationConfig") or request.get("generation_config")+ if not isinstance(config, dict):+ return 1+ count = config.get("candidateCount", config.get("candidate_count", 1))+ return count if isinstance(count, int) and not isinstance(count, bool) and count > 0 else 1+++def _sse_event_outcome(event_name: str | None, data_str: str, dialect: str) -> tuple[bool, bool]:+ """Classify a complete protocol event, never text inside a model delta."""+ try:+ data = json.loads(data_str)+ except (TypeError, ValueError):+ data = None+ event_type = data.get("type") if isinstance(data, dict) else None+ if not isinstance(event_type, str):+ event_type = None+ failed = event_name in {"error", "response.failed"} or event_type in {+ "error",+ "response.failed",+ }+ if isinstance(data, dict) and data.get("error") is not None:+ failed = True+ if dialect == "gemini":+ response = _gemini_response_data(data)+ feedback = response.get("promptFeedback")+ block_reason = feedback.get("blockReason") if isinstance(feedback, dict) else None+ failed = (+ failed+ or response.get("error") is not None+ or (+ isinstance(block_reason, str)+ and bool(block_reason.strip())+ and block_reason != "BLOCK_REASON_UNSPECIFIED"+ )+ )+ terminal = bool(_gemini_finished_candidates(data))+ elif dialect == "anthropic":+ terminal = event_name == "message_stop" or event_type == "message_stop"+ elif dialect == "responses":+ # Incomplete is a normal terminal result (e.g. the output-token limit),+ # unlike response.failed. It still incurred a successful model request.+ terminal = event_name in {"response.completed", "response.incomplete"} or event_type in {+ "response.completed",+ "response.incomplete",+ }+ else:+ terminal = data_str.strip() == "[DONE]"+ return failed, terminal+++def _sse_contains_error_event(payload: bytes) -> bool:+ from headroom.proxy.helpers import parse_sse_events_from_byte_buffer++ return any(+ _sse_event_outcome(event_name, data, "chat")[0]+ for event_name, data in parse_sse_events_from_byte_buffer(bytearray(payload))+ )+_ROUND_USAGE_KEYS = ("input_tokens",@@ -335,6 +434,16 @@ class StreamingMixin:# ``stream_state``, no reassignment is needed.events = parse_sse_events_from_byte_buffer(buffer)for _event_name, data_str in events:+ failed, terminal = _sse_event_outcome(+ _event_name,+ data_str,+ stream_state.get(+ "sse_dialect",+ provider if provider in {"anthropic", "gemini"} else "chat",+ ),+ )+ stream_state["stream_failed"] = stream_state.get("stream_failed", False) or failed+ stream_state["stream_terminal"] = stream_state.get("stream_terminal", False) or terminalif not data_str or data_str == "[DONE]":continue@@ -343,6 +452,12 @@ class StreamingMixin:except json.JSONDecodeError:continue+ if provider == "gemini":+ finished = stream_state.setdefault("gemini_finished_candidates", set())+ finished.update(_gemini_finished_candidates(data))+ expected = stream_state.get("gemini_candidate_count", 1)+ stream_state["stream_terminal"] = set(range(expected)).issubset(finished)+if provider == "anthropic":event_type = data.get("type", "")if event_type == "message_start":@@ -1266,6 +1381,18 @@ class StreamingMixin:# not corrupt downstream parsing."sse_buffer": bytearray(),"ttfb_ms": None, # Time to first byte from upstream+ "stream_failed": False,+ "stream_terminal": False,+ "gemini_candidate_count": _gemini_requested_candidates(body),+ "sse_dialect": (+ "anthropic"+ if provider == "anthropic"+ else "gemini"+ if provider == "gemini"+ else "responses"+ if urlsplit(url).path.rstrip("/").endswith("/responses")+ else "chat"+ ),}# Track if we need to handle memory tools@@ -1829,7 +1956,9 @@ class StreamingMixin:status_code=upstream_response.status_code,metadata={"total_bytes": stream_state["total_bytes"]},)- completed_normally = True+ completed_normally = (+ stream_state["stream_terminal"] and not stream_state["stream_failed"]+ )except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:logger.error(f"[{request_id}] Connection error to upstream API: {e}")@@ -1915,7 +2044,9 @@ class StreamingMixin:parsed_response=parsed_response,client=client,waste_signals=waste_signals,- status_code=upstream_response.status_code,+ status_code=_stream_outcome_status(+ upstream_response.status_code, completed_normally+ ),conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,)@@ -2012,8 +2143,12 @@ class StreamingMixin:# stream closes (see finally: block below). Not on the hot path# for anything the client sees.full_sse_bytes = bytearray()+ completed_normally = False+ stream_failed = False+ stream_terminal = Falseasync def generate():+ nonlocal completed_normally, stream_failed, stream_terminaltry:assert backend is not None@@ -2088,9 +2223,16 @@ class StreamingMixin:]# Handle errors- if event.event_type == "error":+ failed, terminal = _sse_event_outcome(+ event.event_type, json.dumps(event.data), "anthropic"+ )+ stream_failed = stream_failed or failed+ stream_terminal = stream_terminal or terminal+ if failed:logger.error(f"[{request_id}] Bedrock stream error: {event.data}")+ completed_normally = stream_terminal and not stream_failed+except Exception as e:error_message = format_exception_message(e)logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")@@ -2110,7 +2252,7 @@ class StreamingMixin:# and _stream_openai_via_backend (OpenAI-via-backend# sibling). Run before the outcome funnel so prefix state# is consistent regardless of metric path.- if prefix_tracker is not None:+ if completed_normally and prefix_tracker is not None:import copy as _copytracker_messages = (@@ -2169,6 +2311,7 @@ class StreamingMixin:overhead_ms=optimization_latency,tags=tags,client=client,+ status_code=_stream_outcome_status(200, completed_normally),log_full_messages=getattr(self.config, "log_full_messages", False),cache_read_tokens=stream_state["cache_read_input_tokens"],cache_write_tokens=stream_state["cache_creation_input_tokens"],@@ -2263,6 +2406,8 @@ class StreamingMixin:# closes (cheap, no buffering of in-flight chunks back to# the client).full_sse_bytes = bytearray()+ completed_normally = False+ stream_failed = Falsedef _absorb(usage: dict[str, int] | None) -> None:if not usage:@@ -2288,6 +2433,10 @@ class StreamingMixin:if parsed is not None and not stream_state["output_tokens"]:stream_state["output_tokens"] = parsedyield chunk_bytes+ stream_failed = stream_state.get("stream_failed", False)+ completed_normally = (+ stream_state.get("stream_terminal", False) and not stream_failed+ )except Exception as e:logger.error(f"[{request_id}] Backend streaming error: {e}")error_data = public_errors.openai_error_body(@@ -2341,7 +2490,7 @@ class StreamingMixin:# the non-streaming sibling. Done before outcome funnel# so prefix state is consistent regardless of metric# path.- if prefix_tracker is not None:+ if completed_normally and prefix_tracker is not None:tracker_messages = (optimized_messagesif optimized_messages is not None@@ -2389,6 +2538,7 @@ class StreamingMixin:overhead_ms=optimization_latency,tags=tags,client=client,+ status_code=_stream_outcome_status(200, completed_normally),log_full_messages=getattr(self.config, "log_full_messages", False),cache_read_tokens=cache_read_tokens,cache_write_tokens=cache_write_tokens,diff --git a/tests/test_gemini_prefix_freeze.py b/tests/test_gemini_prefix_freeze.pyindex e5120c09e..79f46d73e 100644--- a/tests/test_gemini_prefix_freeze.py+++ b/tests/test_gemini_prefix_freeze.py@@ -112,7 +112,7 @@ def _gemini_provider(sent: list[dict[str, Any]]):if ":streamGenerateContent" in str(request.url):chunks = ('data: {"candidates": [{"content": {"role": "model", '- '"parts": [{"text": "ack"}]}}]}\n\n'+ '"parts": [{"text": "ack"}]}, "finishReason": "STOP"}]}\n\n'+ f"data: {json.dumps({'usageMetadata': usage})}\n\n")return httpx.Response(diff --git a/tests/test_mid_turn_steering.py b/tests/test_mid_turn_steering.pyindex 81534ae0d..0277e374f 100644--- a/tests/test_mid_turn_steering.py+++ b/tests/test_mid_turn_steering.py@@ -81,7 +81,8 @@ class TestMidTurnSteering:proxy._config.retry_max_delay_ms = 0proxy.config = proxy._configproxy.memory_handler = None- proxy._parse_sse_usage_from_buffer = MagicMock(return_value=None)+ # Exercise real event parsing: successful completion requires the+ # message_stop marker before pending messages can be drained.proxy._finalize_stream_response = AsyncMock(return_value=None)return proxydiff --git a/tests/test_route_advice.py b/tests/test_route_advice.pyindex 9e84bee9c..812f3abc7 100644--- a/tests/test_route_advice.py+++ b/tests/test_route_advice.py@@ -183,6 +183,77 @@ class _Backend:yield # pragma: no cover+class _FailingBackend(_Backend):+ async def stream_message(self, body, headers):+ self.served = True+ yield SimpleNamespace(+ event_type="message_start",+ raw_sse="event: message_start\ndata: {}\n\n",+ data={},+ )+ raise RuntimeError("stream finalization failed")+++class _ErrorEventBackend(_Backend):+ async def stream_message(self, body, headers):+ self.served = True+ yield SimpleNamespace(+ event_type="error",+ raw_sse='event: error\ndata: {"error": {"type": "api_error"}}\n\n',+ data={"error": {"type": "api_error"}},+ )++ async def stream_openai_message(self, body, headers):+ self.served = True+ yield 'data: {"type": "er'+ yield 'ror", "message": "provider failed"}\n\n'+ yield "data: [DONE]\n\n"+++async def _drive_with_outcome(handler, backend):+ from headroom.proxy.handlers.streaming import StreamingMixin++ response = await StreamingMixin._stream_response_bedrock(+ handler,+ {"messages": []},+ {},+ "anthropic",+ "m",+ "rid",+ 0,+ 0,+ 0,+ [],+ {},+ 0.0,+ backend=backend,+ )+ async for _ in response.body_iterator:+ pass+++async def _drive_openai_with_outcome(handler, backend):+ from headroom.proxy.handlers.streaming import StreamingMixin++ response = await StreamingMixin._stream_openai_via_backend(+ handler,+ {"messages": []},+ {},+ "m",+ "rid",+ 0.0,+ 0,+ 0,+ 0,+ [],+ {},+ 0.0,+ backend=backend,+ )+ async for _ in response.body_iterator:+ pass++async def _drive(handler, **kw):from headroom.proxy.handlers.streaming import StreamingMixin@@ -240,6 +311,271 @@ def test_streaming_without_a_route_uses_the_configured_backend():assert configured.served+def test_failed_stream_finalization_is_not_counted_as_completed():+ backend = _FailingBackend("anthropic")+ outcomes = []++ async def record_outcome(outcome):+ outcomes.append(outcome)++ handler = _handler(backend)+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)+ handler._record_request_outcome = record_outcome+ asyncio.run(_drive_with_outcome(handler, backend))++ assert len(outcomes) == 1+ assert outcomes[0].status_code == 502+++def test_stream_outcome_status_preserves_success_and_reclassifies_failure():+ from headroom.proxy.handlers.streaming import _stream_outcome_status++ assert _stream_outcome_status(200, True) == 200+ assert _stream_outcome_status(200, False) == 502+ assert _stream_outcome_status(529, False) == 502+++def test_sse_error_detection_handles_native_and_split_events():+ from headroom.proxy.handlers.streaming import _sse_contains_error_event++ assert _sse_contains_error_event(b'event: error\ndata: {"type":"error"}\n\n')+ assert _sse_contains_error_event(b'data: {"type":"er' + b'ror"}\n\n')+ assert not _sse_contains_error_event(b'data: {"type":"message"}\n\n')+++def test_sse_error_text_in_response_content_is_not_an_error():+ from headroom.proxy.handlers.streaming import _sse_contains_error_event++ assert not _sse_contains_error_event(b'data: {"text":"event: error"}\n\n')+++def test_responses_failed_envelope_is_an_error():+ from headroom.proxy.handlers.streaming import _sse_contains_error_event++ assert _sse_contains_error_event(+ b'data: {"type":"response.failed","response":{"status":"failed",'+ b'"error":{"code":"server_error"}}}\n\n'+ )++[email protected]("provider", ["anthropic", "openai"])[email protected]("complete", [False, True])+def test_backend_clean_eof_requires_protocol_completion(provider, complete):+ from types import MethodType++ from headroom.proxy.handlers.streaming import StreamingMixin++ class Backend(_Backend):+ async def stream_message(self, body, headers):+ yield SimpleNamespace(+ event_type="message_start", raw_sse="event: message_start\ndata: {}\n\n", data={}+ )+ if complete:+ yield SimpleNamespace(+ event_type="message_stop",+ raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',+ data={"type": "message_stop"},+ )++ async def stream_openai_message(self, body, headers):+ yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'+ if complete:+ yield "data: [DONE]\n\n"++ outcomes = []++ async def record(outcome):+ outcomes.append(outcome)++ handler = _handler(Backend(provider))+ handler._record_request_outcome = record+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)+ handler._parse_sse_usage_from_buffer = MethodType(+ StreamingMixin._parse_sse_usage_from_buffer, handler+ )+ drive = _drive_with_outcome if provider == "anthropic" else _drive_openai_with_outcome+ asyncio.run(drive(handler, handler.anthropic_backend))+ assert len(outcomes) == 1+ assert outcomes[0].status_code == (200 if complete else 502)++[email protected]("provider", ["anthropic", "openai"])[email protected]("complete", [False, True])+def test_backend_prefix_tracker_advances_only_after_completion(provider, complete):+ from types import MethodType+ from unittest.mock import MagicMock++ from headroom.proxy.handlers.streaming import StreamingMixin++ class Backend(_Backend):+ async def stream_message(self, body, headers):+ yield SimpleNamespace(event_type="message_start", raw_sse="", data={})+ if complete:+ yield SimpleNamespace(event_type="message_stop", raw_sse="", data={})++ async def stream_openai_message(self, body, headers):+ yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'+ if complete:+ yield "data: [DONE]\n\n"++ backend = Backend(provider)+ handler = _handler(backend)+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)+ handler._parse_sse_usage_from_buffer = MethodType(+ StreamingMixin._parse_sse_usage_from_buffer, handler+ )+ handler._parse_sse_to_response = lambda *args: None+ handler._assistant_message_from_response_json = lambda response: None+ tracker = MagicMock()+ tracker.classify_cache_miss.return_value = SimpleNamespace(is_miss=False)++ async def drive():+ arguments = {+ "body": {"messages": []},+ "headers": {},+ "model": "m",+ "request_id": "rid",+ "original_tokens": 0,+ "optimized_tokens": 0,+ "tokens_saved": 0,+ "transforms_applied": [],+ "tags": {},+ "optimization_latency": 0.0,+ "backend": backend,+ "prefix_tracker": tracker,+ }+ if provider == "anthropic":+ response = await StreamingMixin._stream_response_bedrock(+ handler, provider=provider, **arguments+ )+ else:+ response = await StreamingMixin._stream_openai_via_backend(+ handler, start_time=0.0, **arguments+ )+ async for _ in response.body_iterator:+ pass++ asyncio.run(drive())+ assert tracker.update_from_response.call_count == int(complete)+++ "event, data, expected_failed, expected_terminal",+ [+ ("response.completed", '{"type":"response.completed"}', False, True),+ ("response.incomplete", '{"type":"response.incomplete"}', False, True),+ ("response.failed", '{"type":"response.failed"}', True, False),+ (None, "[DONE]", False, False),+ ],+)+def test_responses_terminal_semantics(event, data, expected_failed, expected_terminal):+ from headroom.proxy.handlers.streaming import _sse_event_outcome++ assert _sse_event_outcome(event, data, "responses") == (expected_failed, expected_terminal)++[email protected]("event_type", ['["error"]', '{"unexpected":true}', "7", "true"])+def test_malformed_event_type_is_not_interpreted_as_a_protocol_marker(event_type):+ from headroom.proxy.handlers.streaming import _sse_contains_error_event++ payload = ('data: {"type":' + event_type + "}\n\ndata: [DONE]\n\n").encode()+ assert not _sse_contains_error_event(payload)++[email protected]("wrapped", [False, True])[email protected]("finish_reason", ["STOP", "MAX_TOKENS"])+def test_gemini_finish_reason_is_a_terminal_event(wrapped, finish_reason):+ import json++ from headroom.proxy.handlers.streaming import _sse_event_outcome++ payload = {"candidates": [{"index": 0, "finishReason": finish_reason}]}+ if wrapped:+ payload = {"response": payload}+ assert _sse_event_outcome(None, json.dumps(payload), "gemini") == (False, True)++[email protected]("wrapped", [False, True])+def test_gemini_prompt_block_is_a_failed_outcome(wrapped):+ import json++ from headroom.proxy.handlers.streaming import _sse_event_outcome++ payload = {"promptFeedback": {"blockReason": "SAFETY"}}+ if wrapped:+ payload = {"response": payload}+ failed, _ = _sse_event_outcome(None, json.dumps(payload), "gemini")+ assert failed+++def test_gemini_completion_requires_all_requested_candidates():+ from headroom.proxy.handlers.streaming import StreamingMixin++ parser = StreamingMixin()+ state = {"sse_buffer": bytearray(), "gemini_candidate_count": 2}+ state["sse_buffer"].extend(b'data: {"candidates":[{"index":0,"finishReason":"STOP"}]}\n\n')+ parser._parse_sse_usage_from_buffer(state, "gemini")+ assert not state.get("stream_terminal", False)+ state["sse_buffer"].extend(b'data: {"candidates":[{"index":1,"finishReason":"STOP"}]}\n\n')+ parser._parse_sse_usage_from_buffer(state, "gemini")+ assert state["stream_terminal"]+++def test_outcome_classification_runs_once_for_a_chunked_complete_event(monkeypatch):+ from headroom.proxy.handlers import streaming++ seen = []+ classify = streaming._sse_event_outcome++ def record(*args):+ seen.append(args)+ return classify(*args)++ monkeypatch.setattr(streaming, "_sse_event_outcome", record)+ state = {"sse_buffer": bytearray()}+ parser = streaming.StreamingMixin()+ payload = b'data: {"text":"' + b"x" * 10_000 + b'"}\n\n'+ for offset in range(0, len(payload), 101):+ state["sse_buffer"].extend(payload[offset : offset + 101])+ parser._parse_sse_usage_from_buffer(state, "openai")+ assert len(seen) == 1+ assert not state["sse_buffer"]+++def test_backend_error_event_is_not_counted_as_completed():+ backend = _ErrorEventBackend("anthropic")+ outcomes = []++ async def record_outcome(outcome):+ outcomes.append(outcome)++ handler = _handler(backend)+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)+ handler._record_request_outcome = record_outcome+ asyncio.run(_drive_with_outcome(handler, backend))++ assert backend.served+ assert len(outcomes) == 1+ assert outcomes[0].status_code == 502+++def test_openai_backend_error_event_is_not_counted_as_completed():+ backend = _ErrorEventBackend("openai")+ outcomes = []++ async def record_outcome(outcome):+ outcomes.append(outcome)++ handler = _handler(backend)+ handler._parse_sse_usage_from_buffer = lambda state, provider: None+ handler._record_request_outcome = record_outcome+ asyncio.run(_drive_openai_with_outcome(handler, backend))++ assert backend.served+ assert len(outcomes) == 1+ assert outcomes[0].status_code == 502++async def _drive_openai(handler, **kw):from headroom.proxy.handlers.streaming import StreamingMixin