Skip to content

Agent Zero AI framework

Respect user stream_options keys when remembering usage rejections

linkliti16fcdfcec1a8
diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.py
index 1b80c0328..9cedac2af 100644
--- a/helpers/litellm_transport.py
+++ b/helpers/litellm_transport.py
@@ -366,12 +366,6 @@ class LiteLLMTransport:
self.stream_usage_retried = True
if self.policy.cache_key:
STREAM_USAGE_UNSUPPORTED_CACHE.add(self.policy.cache_key)
- options = dict(self.kwargs.get("stream_options") or {})
- options.pop("include_usage", None)
- if options:
- self.kwargs["stream_options"] = options
- else:
- self.kwargs.pop("stream_options", None)
return True
def _chat_request(self, *, stream: bool) -> dict[str, Any]:
@@ -392,14 +386,15 @@ class LiteLLMTransport:
"stream": stream,
**chat_kwargs,
}
- if stream and not self.stream_usage_retried:
- options = {**(request.get("stream_options") or {})}
- if self.policy.cache_key in STREAM_USAGE_UNSUPPORTED_CACHE:
- options.pop("include_usage", None)
- else:
- options["include_usage"] = True
- if options:
- request["stream_options"] = options
+ if (
+ stream
+ and not self.stream_usage_retried
+ and self.policy.cache_key not in STREAM_USAGE_UNSUPPORTED_CACHE
+ ):
+ request["stream_options"] = {
+ **(request.get("stream_options") or {}),
+ "include_usage": True,
+ }
if self.stop is not None:
request["stop"] = self.stop
return request
diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.md
index 65e499444..277670b48 100644
--- a/helpers/litellm_transport.py.dox.md
+++ b/helpers/litellm_transport.py.dox.md
@@ -36,7 +36,7 @@
- Fall back to Chat Completions when LiteLLM's Responses mock streaming path tries to JSON-decode a real SSE stream before any output.
- Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items.
- Preserve provider usage and LiteLLM response cost for both transports only when the response or stream actually supplies them; do not synthesize unavailable provider accounting.
-- Streaming Chat Completions requests include `stream_options.include_usage` so terminal usage events reach `LLMResult.usage` for OpenAI-compatible endpoints and Messages API providers alike. A caller-provided `stream_options` keeps its extra keys, and a pre-output rejection of the option retries once without `include_usage`, remembering the rejection per model configuration in a process-global cache that naturally clears when the framework process restarts; tests reset it through `clear_transport_capability_cache`.
+- Streaming Chat Completions requests include `stream_options.include_usage` so terminal usage events reach `LLMResult.usage` for OpenAI-compatible endpoints and Messages API providers alike, unless the model configuration previously rejected the option or the request already carries it. A pre-output rejection retries once without the transport-injected `include_usage` and remembers the rejection per model configuration in a process-global cache that naturally clears when the framework process restarts; tests reset it through `clear_transport_capability_cache`. User-provided `stream_options` keys are always passed through untouched, even when the provider rejects them.
- Recover all Responses output items from stream events when a terminal completed envelope omits them, including encrypted reasoning. Merge by item/call identity in output-index order; terminal non-null fields win without duplicating calls.
- Stream named native function arguments through canonical tool envelopes for display. Buffer interleaved calls until the active envelope closes; preserve independent original call metadata. Native execution still waits for the completed model turn.
- Fail incomplete Responses generations and native-call streams that end without completion; never promote partial call previews to a Chat result.
diff --git a/tests/test_responses_architecture.py b/tests/test_responses_architecture.py
index 68e710e1e..135805d11 100644
--- a/tests/test_responses_architecture.py
+++ b/tests/test_responses_architecture.py
@@ -312,23 +312,17 @@ async def test_chat_stream_retries_without_usage_when_stream_options_rejected(
transport = litellm_transport.LiteLLMTransport(
model="custom/model",
messages=[{"role": "user", "content": "question"}],
- kwargs={
- "a0_api_mode": "chat_completions",
- "stream_options": {"include_usage": True, "custom": "keep"},
- },
+ kwargs={"a0_api_mode": "chat_completions"},
)
async for _chunk in transport.astream():
pass
- assert calls[0]["stream_options"] == {
- "include_usage": True,
- "custom": "keep",
- }
- assert calls[1]["stream_options"] == {"custom": "keep"}
+ assert calls[0]["stream_options"] == {"include_usage": True}
+ assert "stream_options" not in calls[1]
assert len(calls) == 2
- next_transport = litellm_transport.LiteLLMTransport(
+ user_transport = litellm_transport.LiteLLMTransport(
model="custom/model",
messages=[{"role": "user", "content": "again"}],
kwargs={
@@ -336,8 +330,11 @@ async def test_chat_stream_retries_without_usage_when_stream_options_rejected(
"stream_options": {"include_usage": True, "custom": "keep"},
},
)
- request = next_transport._chat_request(stream=True)
- assert request["stream_options"] == {"custom": "keep"}
+ request = user_transport._chat_request(stream=True)
+ assert request["stream_options"] == {
+ "include_usage": True,
+ "custom": "keep",
+ }
def test_responses_provider_state_uses_previous_response_and_new_items():