Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): classify failed streams as failed requests (#3747)

lorenzozane42d4f58b434b
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 198ec5d1c..84fa502e0 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -12,6 +12,7 @@ import logging
import time
from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
+from urllib.parse import urlsplit
from headroom.proxy.auth_mode import classify_client, supports_mid_turn_coalescing
from headroom.proxy.handlers._debug_dump import write_upstream_error_dump
@@ -40,6 +41,104 @@ from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_toke
from headroom.utils import format_exception_message
logger = logging.getLogger("headroom.proxy")
+STREAM_FAILURE_STATUS = 502
+
+
+def _stream_outcome_status(status_code: int, completed_normally: bool) -> int:
+ """Keep post-start stream failures out of the success counter."""
+ return status_code if completed_normally else STREAM_FAILURE_STATUS
+
+
+def _gemini_response_data(data: Any) -> dict[str, Any]:
+ if not isinstance(data, dict):
+ return {}
+ response = data.get("response")
+ return response if isinstance(response, dict) else data
+
+
+def _gemini_finished_candidates(data: Any) -> set[int]:
+ candidates = _gemini_response_data(data).get("candidates")
+ finished: set[int] = set()
+ if not isinstance(candidates, list):
+ return finished
+ for position, candidate in enumerate(candidates):
+ if not isinstance(candidate, dict):
+ continue
+ reason = candidate.get("finishReason")
+ index = candidate.get("index", position)
+ if (
+ isinstance(reason, str)
+ and reason.strip()
+ and reason != "FINISH_REASON_UNSPECIFIED"
+ and isinstance(index, int)
+ and not isinstance(index, bool)
+ and index >= 0
+ ):
+ finished.add(index)
+ return finished
+
+
+def _gemini_requested_candidates(body: dict[str, Any]) -> int:
+ request = body.get("request")
+ request = request if isinstance(request, dict) else body
+ config = request.get("generationConfig") or request.get("generation_config")
+ if not isinstance(config, dict):
+ return 1
+ count = config.get("candidateCount", config.get("candidate_count", 1))
+ return count if isinstance(count, int) and not isinstance(count, bool) and count > 0 else 1
+
+
+def _sse_event_outcome(event_name: str | None, data_str: str, dialect: str) -> tuple[bool, bool]:
+ """Classify a complete protocol event, never text inside a model delta."""
+ try:
+ data = json.loads(data_str)
+ except (TypeError, ValueError):
+ data = None
+ event_type = data.get("type") if isinstance(data, dict) else None
+ if not isinstance(event_type, str):
+ event_type = None
+ failed = event_name in {"error", "response.failed"} or event_type in {
+ "error",
+ "response.failed",
+ }
+ if isinstance(data, dict) and data.get("error") is not None:
+ failed = True
+ if dialect == "gemini":
+ response = _gemini_response_data(data)
+ feedback = response.get("promptFeedback")
+ block_reason = feedback.get("blockReason") if isinstance(feedback, dict) else None
+ failed = (
+ failed
+ or response.get("error") is not None
+ or (
+ isinstance(block_reason, str)
+ and bool(block_reason.strip())
+ and block_reason != "BLOCK_REASON_UNSPECIFIED"
+ )
+ )
+ terminal = bool(_gemini_finished_candidates(data))
+ elif dialect == "anthropic":
+ terminal = event_name == "message_stop" or event_type == "message_stop"
+ elif dialect == "responses":
+ # Incomplete is a normal terminal result (e.g. the output-token limit),
+ # unlike response.failed. It still incurred a successful model request.
+ terminal = event_name in {"response.completed", "response.incomplete"} or event_type in {
+ "response.completed",
+ "response.incomplete",
+ }
+ else:
+ terminal = data_str.strip() == "[DONE]"
+ return failed, terminal
+
+
+def _sse_contains_error_event(payload: bytes) -> bool:
+ from headroom.proxy.helpers import parse_sse_events_from_byte_buffer
+
+ return any(
+ _sse_event_outcome(event_name, data, "chat")[0]
+ for event_name, data in parse_sse_events_from_byte_buffer(bytearray(payload))
+ )
+
_ROUND_USAGE_KEYS = (
"input_tokens",
@@ -335,6 +434,16 @@ class StreamingMixin:
# ``stream_state``, no reassignment is needed.
events = parse_sse_events_from_byte_buffer(buffer)
for _event_name, data_str in events:
+ failed, terminal = _sse_event_outcome(
+ _event_name,
+ data_str,
+ stream_state.get(
+ "sse_dialect",
+ provider if provider in {"anthropic", "gemini"} else "chat",
+ ),
+ )
+ stream_state["stream_failed"] = stream_state.get("stream_failed", False) or failed
+ stream_state["stream_terminal"] = stream_state.get("stream_terminal", False) or terminal
if not data_str or data_str == "[DONE]":
continue
@@ -343,6 +452,12 @@ class StreamingMixin:
except json.JSONDecodeError:
continue
+ if provider == "gemini":
+ finished = stream_state.setdefault("gemini_finished_candidates", set())
+ finished.update(_gemini_finished_candidates(data))
+ expected = stream_state.get("gemini_candidate_count", 1)
+ stream_state["stream_terminal"] = set(range(expected)).issubset(finished)
+
if provider == "anthropic":
event_type = data.get("type", "")
if event_type == "message_start":
@@ -1266,6 +1381,18 @@ class StreamingMixin:
# not corrupt downstream parsing.
"sse_buffer": bytearray(),
"ttfb_ms": None, # Time to first byte from upstream
+ "stream_failed": False,
+ "stream_terminal": False,
+ "gemini_candidate_count": _gemini_requested_candidates(body),
+ "sse_dialect": (
+ "anthropic"
+ if provider == "anthropic"
+ else "gemini"
+ if provider == "gemini"
+ else "responses"
+ if urlsplit(url).path.rstrip("/").endswith("/responses")
+ else "chat"
+ ),
}
# Track if we need to handle memory tools
@@ -1829,7 +1956,9 @@ class StreamingMixin:
status_code=upstream_response.status_code,
metadata={"total_bytes": stream_state["total_bytes"]},
)
- completed_normally = True
+ completed_normally = (
+ stream_state["stream_terminal"] and not stream_state["stream_failed"]
+ )
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
logger.error(f"[{request_id}] Connection error to upstream API: {e}")
@@ -1915,7 +2044,9 @@ class StreamingMixin:
parsed_response=parsed_response,
client=client,
waste_signals=waste_signals,
- status_code=upstream_response.status_code,
+ status_code=_stream_outcome_status(
+ upstream_response.status_code, completed_normally
+ ),
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
)
@@ -2012,8 +2143,12 @@ class StreamingMixin:
# stream closes (see finally: block below). Not on the hot path
# for anything the client sees.
full_sse_bytes = bytearray()
+ completed_normally = False
+ stream_failed = False
+ stream_terminal = False
async def generate():
+ nonlocal completed_normally, stream_failed, stream_terminal
try:
assert backend is not None
@@ -2088,9 +2223,16 @@ class StreamingMixin:
]
# Handle errors
- if event.event_type == "error":
+ failed, terminal = _sse_event_outcome(
+ event.event_type, json.dumps(event.data), "anthropic"
+ )
+ stream_failed = stream_failed or failed
+ stream_terminal = stream_terminal or terminal
+ if failed:
logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
+ completed_normally = stream_terminal and not stream_failed
+
except Exception as e:
error_message = format_exception_message(e)
logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")
@@ -2110,7 +2252,7 @@ class StreamingMixin:
# and _stream_openai_via_backend (OpenAI-via-backend
# sibling). Run before the outcome funnel so prefix state
# is consistent regardless of metric path.
- if prefix_tracker is not None:
+ if completed_normally and prefix_tracker is not None:
import copy as _copy
tracker_messages = (
@@ -2169,6 +2311,7 @@ class StreamingMixin:
overhead_ms=optimization_latency,
tags=tags,
client=client,
+ status_code=_stream_outcome_status(200, completed_normally),
log_full_messages=getattr(self.config, "log_full_messages", False),
cache_read_tokens=stream_state["cache_read_input_tokens"],
cache_write_tokens=stream_state["cache_creation_input_tokens"],
@@ -2263,6 +2406,8 @@ class StreamingMixin:
# closes (cheap, no buffering of in-flight chunks back to
# the client).
full_sse_bytes = bytearray()
+ completed_normally = False
+ stream_failed = False
def _absorb(usage: dict[str, int] | None) -> None:
if not usage:
@@ -2288,6 +2433,10 @@ class StreamingMixin:
if parsed is not None and not stream_state["output_tokens"]:
stream_state["output_tokens"] = parsed
yield chunk_bytes
+ stream_failed = stream_state.get("stream_failed", False)
+ completed_normally = (
+ stream_state.get("stream_terminal", False) and not stream_failed
+ )
except Exception as e:
logger.error(f"[{request_id}] Backend streaming error: {e}")
error_data = public_errors.openai_error_body(
@@ -2341,7 +2490,7 @@ class StreamingMixin:
# the non-streaming sibling. Done before outcome funnel
# so prefix state is consistent regardless of metric
# path.
- if prefix_tracker is not None:
+ if completed_normally and prefix_tracker is not None:
tracker_messages = (
optimized_messages
if optimized_messages is not None
@@ -2389,6 +2538,7 @@ class StreamingMixin:
overhead_ms=optimization_latency,
tags=tags,
client=client,
+ status_code=_stream_outcome_status(200, completed_normally),
log_full_messages=getattr(self.config, "log_full_messages", False),
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
diff --git a/tests/test_gemini_prefix_freeze.py b/tests/test_gemini_prefix_freeze.py
index e5120c09e..79f46d73e 100644
--- a/tests/test_gemini_prefix_freeze.py
+++ b/tests/test_gemini_prefix_freeze.py
@@ -112,7 +112,7 @@ def _gemini_provider(sent: list[dict[str, Any]]):
if ":streamGenerateContent" in str(request.url):
chunks = (
'data: {"candidates": [{"content": {"role": "model", '
- '"parts": [{"text": "ack"}]}}]}\n\n'
+ '"parts": [{"text": "ack"}]}, "finishReason": "STOP"}]}\n\n'
+ f"data: {json.dumps({'usageMetadata': usage})}\n\n"
)
return httpx.Response(
diff --git a/tests/test_mid_turn_steering.py b/tests/test_mid_turn_steering.py
index 81534ae0d..0277e374f 100644
--- a/tests/test_mid_turn_steering.py
+++ b/tests/test_mid_turn_steering.py
@@ -81,7 +81,8 @@ class TestMidTurnSteering:
proxy._config.retry_max_delay_ms = 0
proxy.config = proxy._config
proxy.memory_handler = None
- proxy._parse_sse_usage_from_buffer = MagicMock(return_value=None)
+ # Exercise real event parsing: successful completion requires the
+ # message_stop marker before pending messages can be drained.
proxy._finalize_stream_response = AsyncMock(return_value=None)
return proxy
diff --git a/tests/test_route_advice.py b/tests/test_route_advice.py
index 9e84bee9c..812f3abc7 100644
--- a/tests/test_route_advice.py
+++ b/tests/test_route_advice.py
@@ -183,6 +183,77 @@ class _Backend:
yield # pragma: no cover
+class _FailingBackend(_Backend):
+ async def stream_message(self, body, headers):
+ self.served = True
+ yield SimpleNamespace(
+ event_type="message_start",
+ raw_sse="event: message_start\ndata: {}\n\n",
+ data={},
+ )
+ raise RuntimeError("stream finalization failed")
+
+
+class _ErrorEventBackend(_Backend):
+ async def stream_message(self, body, headers):
+ self.served = True
+ yield SimpleNamespace(
+ event_type="error",
+ raw_sse='event: error\ndata: {"error": {"type": "api_error"}}\n\n',
+ data={"error": {"type": "api_error"}},
+ )
+
+ async def stream_openai_message(self, body, headers):
+ self.served = True
+ yield 'data: {"type": "er'
+ yield 'ror", "message": "provider failed"}\n\n'
+ yield "data: [DONE]\n\n"
+
+
+async def _drive_with_outcome(handler, backend):
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ response = await StreamingMixin._stream_response_bedrock(
+ handler,
+ {"messages": []},
+ {},
+ "anthropic",
+ "m",
+ "rid",
+ 0,
+ 0,
+ 0,
+ [],
+ {},
+ 0.0,
+ backend=backend,
+ )
+ async for _ in response.body_iterator:
+ pass
+
+
+async def _drive_openai_with_outcome(handler, backend):
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ response = await StreamingMixin._stream_openai_via_backend(
+ handler,
+ {"messages": []},
+ {},
+ "m",
+ "rid",
+ 0.0,
+ 0,
+ 0,
+ 0,
+ [],
+ {},
+ 0.0,
+ backend=backend,
+ )
+ async for _ in response.body_iterator:
+ pass
+
+
async def _drive(handler, **kw):
from headroom.proxy.handlers.streaming import StreamingMixin
@@ -240,6 +311,271 @@ def test_streaming_without_a_route_uses_the_configured_backend():
assert configured.served
+def test_failed_stream_finalization_is_not_counted_as_completed():
+ backend = _FailingBackend("anthropic")
+ outcomes = []
+
+ async def record_outcome(outcome):
+ outcomes.append(outcome)
+
+ handler = _handler(backend)
+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
+ handler._record_request_outcome = record_outcome
+ asyncio.run(_drive_with_outcome(handler, backend))
+
+ assert len(outcomes) == 1
+ assert outcomes[0].status_code == 502
+
+
+def test_stream_outcome_status_preserves_success_and_reclassifies_failure():
+ from headroom.proxy.handlers.streaming import _stream_outcome_status
+
+ assert _stream_outcome_status(200, True) == 200
+ assert _stream_outcome_status(200, False) == 502
+ assert _stream_outcome_status(529, False) == 502
+
+
+def test_sse_error_detection_handles_native_and_split_events():
+ from headroom.proxy.handlers.streaming import _sse_contains_error_event
+
+ assert _sse_contains_error_event(b'event: error\ndata: {"type":"error"}\n\n')
+ assert _sse_contains_error_event(b'data: {"type":"er' + b'ror"}\n\n')
+ assert not _sse_contains_error_event(b'data: {"type":"message"}\n\n')
+
+
+def test_sse_error_text_in_response_content_is_not_an_error():
+ from headroom.proxy.handlers.streaming import _sse_contains_error_event
+
+ assert not _sse_contains_error_event(b'data: {"text":"event: error"}\n\n')
+
+
+def test_responses_failed_envelope_is_an_error():
+ from headroom.proxy.handlers.streaming import _sse_contains_error_event
+
+ assert _sse_contains_error_event(
+ b'data: {"type":"response.failed","response":{"status":"failed",'
+ b'"error":{"code":"server_error"}}}\n\n'
+ )
+
+
[email protected]("provider", ["anthropic", "openai"])
[email protected]("complete", [False, True])
+def test_backend_clean_eof_requires_protocol_completion(provider, complete):
+ from types import MethodType
+
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ class Backend(_Backend):
+ async def stream_message(self, body, headers):
+ yield SimpleNamespace(
+ event_type="message_start", raw_sse="event: message_start\ndata: {}\n\n", data={}
+ )
+ if complete:
+ yield SimpleNamespace(
+ event_type="message_stop",
+ raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',
+ data={"type": "message_stop"},
+ )
+
+ async def stream_openai_message(self, body, headers):
+ yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'
+ if complete:
+ yield "data: [DONE]\n\n"
+
+ outcomes = []
+
+ async def record(outcome):
+ outcomes.append(outcome)
+
+ handler = _handler(Backend(provider))
+ handler._record_request_outcome = record
+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
+ handler._parse_sse_usage_from_buffer = MethodType(
+ StreamingMixin._parse_sse_usage_from_buffer, handler
+ )
+ drive = _drive_with_outcome if provider == "anthropic" else _drive_openai_with_outcome
+ asyncio.run(drive(handler, handler.anthropic_backend))
+ assert len(outcomes) == 1
+ assert outcomes[0].status_code == (200 if complete else 502)
+
+
[email protected]("provider", ["anthropic", "openai"])
[email protected]("complete", [False, True])
+def test_backend_prefix_tracker_advances_only_after_completion(provider, complete):
+ from types import MethodType
+ from unittest.mock import MagicMock
+
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ class Backend(_Backend):
+ async def stream_message(self, body, headers):
+ yield SimpleNamespace(event_type="message_start", raw_sse="", data={})
+ if complete:
+ yield SimpleNamespace(event_type="message_stop", raw_sse="", data={})
+
+ async def stream_openai_message(self, body, headers):
+ yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'
+ if complete:
+ yield "data: [DONE]\n\n"
+
+ backend = Backend(provider)
+ handler = _handler(backend)
+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
+ handler._parse_sse_usage_from_buffer = MethodType(
+ StreamingMixin._parse_sse_usage_from_buffer, handler
+ )
+ handler._parse_sse_to_response = lambda *args: None
+ handler._assistant_message_from_response_json = lambda response: None
+ tracker = MagicMock()
+ tracker.classify_cache_miss.return_value = SimpleNamespace(is_miss=False)
+
+ async def drive():
+ arguments = {
+ "body": {"messages": []},
+ "headers": {},
+ "model": "m",
+ "request_id": "rid",
+ "original_tokens": 0,
+ "optimized_tokens": 0,
+ "tokens_saved": 0,
+ "transforms_applied": [],
+ "tags": {},
+ "optimization_latency": 0.0,
+ "backend": backend,
+ "prefix_tracker": tracker,
+ }
+ if provider == "anthropic":
+ response = await StreamingMixin._stream_response_bedrock(
+ handler, provider=provider, **arguments
+ )
+ else:
+ response = await StreamingMixin._stream_openai_via_backend(
+ handler, start_time=0.0, **arguments
+ )
+ async for _ in response.body_iterator:
+ pass
+
+ asyncio.run(drive())
+ assert tracker.update_from_response.call_count == int(complete)
+
+
+ "event, data, expected_failed, expected_terminal",
+ [
+ ("response.completed", '{"type":"response.completed"}', False, True),
+ ("response.incomplete", '{"type":"response.incomplete"}', False, True),
+ ("response.failed", '{"type":"response.failed"}', True, False),
+ (None, "[DONE]", False, False),
+ ],
+)
+def test_responses_terminal_semantics(event, data, expected_failed, expected_terminal):
+ from headroom.proxy.handlers.streaming import _sse_event_outcome
+
+ assert _sse_event_outcome(event, data, "responses") == (expected_failed, expected_terminal)
+
+
[email protected]("event_type", ['["error"]', '{"unexpected":true}', "7", "true"])
+def test_malformed_event_type_is_not_interpreted_as_a_protocol_marker(event_type):
+ from headroom.proxy.handlers.streaming import _sse_contains_error_event
+
+ payload = ('data: {"type":' + event_type + "}\n\ndata: [DONE]\n\n").encode()
+ assert not _sse_contains_error_event(payload)
+
+
[email protected]("wrapped", [False, True])
[email protected]("finish_reason", ["STOP", "MAX_TOKENS"])
+def test_gemini_finish_reason_is_a_terminal_event(wrapped, finish_reason):
+ import json
+
+ from headroom.proxy.handlers.streaming import _sse_event_outcome
+
+ payload = {"candidates": [{"index": 0, "finishReason": finish_reason}]}
+ if wrapped:
+ payload = {"response": payload}
+ assert _sse_event_outcome(None, json.dumps(payload), "gemini") == (False, True)
+
+
[email protected]("wrapped", [False, True])
+def test_gemini_prompt_block_is_a_failed_outcome(wrapped):
+ import json
+
+ from headroom.proxy.handlers.streaming import _sse_event_outcome
+
+ payload = {"promptFeedback": {"blockReason": "SAFETY"}}
+ if wrapped:
+ payload = {"response": payload}
+ failed, _ = _sse_event_outcome(None, json.dumps(payload), "gemini")
+ assert failed
+
+
+def test_gemini_completion_requires_all_requested_candidates():
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ parser = StreamingMixin()
+ state = {"sse_buffer": bytearray(), "gemini_candidate_count": 2}
+ state["sse_buffer"].extend(b'data: {"candidates":[{"index":0,"finishReason":"STOP"}]}\n\n')
+ parser._parse_sse_usage_from_buffer(state, "gemini")
+ assert not state.get("stream_terminal", False)
+ state["sse_buffer"].extend(b'data: {"candidates":[{"index":1,"finishReason":"STOP"}]}\n\n')
+ parser._parse_sse_usage_from_buffer(state, "gemini")
+ assert state["stream_terminal"]
+
+
+def test_outcome_classification_runs_once_for_a_chunked_complete_event(monkeypatch):
+ from headroom.proxy.handlers import streaming
+
+ seen = []
+ classify = streaming._sse_event_outcome
+
+ def record(*args):
+ seen.append(args)
+ return classify(*args)
+
+ monkeypatch.setattr(streaming, "_sse_event_outcome", record)
+ state = {"sse_buffer": bytearray()}
+ parser = streaming.StreamingMixin()
+ payload = b'data: {"text":"' + b"x" * 10_000 + b'"}\n\n'
+ for offset in range(0, len(payload), 101):
+ state["sse_buffer"].extend(payload[offset : offset + 101])
+ parser._parse_sse_usage_from_buffer(state, "openai")
+ assert len(seen) == 1
+ assert not state["sse_buffer"]
+
+
+def test_backend_error_event_is_not_counted_as_completed():
+ backend = _ErrorEventBackend("anthropic")
+ outcomes = []
+
+ async def record_outcome(outcome):
+ outcomes.append(outcome)
+
+ handler = _handler(backend)
+ handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
+ handler._record_request_outcome = record_outcome
+ asyncio.run(_drive_with_outcome(handler, backend))
+
+ assert backend.served
+ assert len(outcomes) == 1
+ assert outcomes[0].status_code == 502
+
+
+def test_openai_backend_error_event_is_not_counted_as_completed():
+ backend = _ErrorEventBackend("openai")
+ outcomes = []
+
+ async def record_outcome(outcome):
+ outcomes.append(outcome)
+
+ handler = _handler(backend)
+ handler._parse_sse_usage_from_buffer = lambda state, provider: None
+ handler._record_request_outcome = record_outcome
+ asyncio.run(_drive_openai_with_outcome(handler, backend))
+
+ assert backend.served
+ assert len(outcomes) == 1
+ assert outcomes[0].status_code == 502
+
+
async def _drive_openai(handler, **kw):
from headroom.proxy.handlers.streaming import StreamingMixin