Agent Zero AI framework
Preserve provider usage in LLM results
Alessandroaf56d211c09b
diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.pyindex 6a1498b71..92b04b32a 100644--- a/helpers/litellm_transport.py+++ b/helpers/litellm_transport.py@@ -214,10 +214,11 @@ class LiteLLMTransport:while True:try:if self.policy.mode is TransportMode.CHAT_COMPLETIONS:- parsed = ChatCompletionsTransport.parse(- completion(**self._chat_request(stream=False))+ raw_response = completion(**self._chat_request(stream=False))+ parsed = ChatCompletionsTransport.parse(raw_response)+ self.last_result = self._llm_result_from_chat(+ parsed, raw_response)- self.last_result = self._llm_result_from_chat(parsed)return parsedrequest = self._responses_request(stream=False)raw_response = responses(**request)@@ -235,10 +236,13 @@ class LiteLLMTransport:while True:try:if self.policy.mode is TransportMode.CHAT_COMPLETIONS:- parsed = ChatCompletionsTransport.parse(- await acompletion(**self._chat_request(stream=False))+ raw_response = await acompletion(+ **self._chat_request(stream=False)+ )+ parsed = ChatCompletionsTransport.parse(raw_response)+ self.last_result = self._llm_result_from_chat(+ parsed, raw_response)- self.last_result = self._llm_result_from_chat(parsed)return parsedrequest = self._responses_request(stream=False)raw_response = await aresponses(**request)@@ -400,10 +404,13 @@ class LiteLLMTransport:**response_kwargs,}- def _llm_result_from_chat(self, parsed: ChatChunk) -> LLMResult:+ def _llm_result_from_chat(+ self, parsed: ChatChunk, response: Any = None+ ) -> LLMResult:return LLMResult.from_chat(response=parsed["response_delta"],reasoning=parsed["reasoning_delta"],+ usage=_reported_usage(response),input_items=ResponsesTransport.input_from_messages(self.messages),output_items=parsed.get("_output_items"),provider_model_key=self.model,@@ -413,7 +420,7 @@ class LiteLLMTransport:def _llm_result_from_response(self, response: Any, request: dict[str, Any]) -> LLMResult:- return LLMResult.from_response(+ result = LLMResult.from_response(response,input_items=_as_list(request.get("input")),previous_response_id=str(request.get("previous_response_id") or ""),@@ -422,6 +429,8 @@ class LiteLLMTransport:state=self.last_request_state,capability=self._capability_metadata(),)+ result.usage = _reported_usage(response)+ return resultdef _stream_result_from_parser(self, parser: "ResponsesEventParser", request: dict[str, Any]@@ -440,10 +449,11 @@ class LiteLLMTransport:self, parser: "ChatCompletionsStreamParser") -> LLMResult | None:output_items = parser.output_items()- if not output_items:+ if not output_items and not parser.usage:return Nonereturn LLMResult.from_chat(response=parser.function_calls_text(),+ usage=parser.usage,input_items=ResponsesTransport.input_from_messages(self.messages),output_items=output_items,provider_model_key=self.model,@@ -589,8 +599,11 @@ class ChatCompletionsStreamParser:self.tool_calls: dict[str, dict[str, Any]] = {}self.order: list[str] = []self.emitted = False+ self.usage: dict[str, Any] = {}def parse(self, chunk: Any) -> ChatChunk:+ if usage := _reported_usage(chunk):+ self.usage.update(usage)parsed = ChatCompletionsTransport.parse(chunk)choice = _first_choice(chunk)delta = _get_value(choice, "delta") or {}@@ -1699,6 +1712,16 @@ def _object_to_dict(obj: Any) -> dict[str, Any]:return {}+def _reported_usage(response: Any) -> dict[str, Any]:+ usage = _object_to_dict(_get_value(response, "usage"))+ hidden = _object_to_dict(_get_value(response, "_hidden_params"))+ if usage.get("cost") is None:+ usage.pop("cost", None)+ if hidden.get("response_cost") is not None:+ usage["cost"] = hidden["response_cost"]+ return usage++def _normalize_reasoning_effort(effort: Any) -> str | None:if isinstance(effort, str):normalized = effort.strip().lower()diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.mdindex 1a8e20795..d122ccd5f 100644--- a/helpers/litellm_transport.py.dox.md+++ b/helpers/litellm_transport.py.dox.md@@ -33,6 +33,7 @@- Fall back to Chat Completions when a Responses endpoint fails before output with an endpoint-specific server error, proxy path-unavailable error, or LiteLLM proxy-extra import error.- Fall back to Chat Completions when LiteLLM's Responses mock streaming path tries to JSON-decode a real SSE stream before any output.- Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items.+- Preserve provider usage and LiteLLM response cost for both transports only when the response or stream actually supplies them; do not synthesize unavailable provider accounting.- Preserve Responses function calls collected from stream events when a terminal completed event omits them.- Preserve provider-state metadata when Responses API calls succeed, and fall back to local replay when provider state is unsupported.- Keep prompt-cache markers only for providers that accept them.diff --git a/helpers/llm_result.py b/helpers/llm_result.pyindex a39234eab..4e645c886 100644--- a/helpers/llm_result.py+++ b/helpers/llm_result.py@@ -124,6 +124,7 @@ class LLMResult:*,response: str,reasoning: str = "",+ usage: dict[str, Any] | None = None,input_items: list[dict[str, Any]] | None = None,output_items: list[dict[str, Any]] | None = None,provider_model_key: str = "",@@ -160,6 +161,7 @@ class LLMResult:provider_model_key=provider_model_key,mode="chat_completions",state="off",+ usage=object_to_dict(usage or {}),capability=dict(capability or {}),)if not result.response and result.function_calls:diff --git a/helpers/llm_result.py.dox.md b/helpers/llm_result.py.dox.mdindex 4d98b9a23..aceaa83d1 100644--- a/helpers/llm_result.py.dox.md+++ b/helpers/llm_result.py.dox.md@@ -19,7 +19,7 @@- `LLMResult.metadata()` stores only durable provider state under `RESPONSE_METADATA_KEY`: response IDs, structured output items, provider/mode/state, usage, and capability data. Runtime prompt inputs, raw responses, and duplicated response/reasoning text are not persisted in history.- `from_response(...)` must preserve provider `response_id`, `previous_response_id`, raw output items, usage, and capability metadata.-- `from_chat(...)` must produce an equivalent chat-completions result with `mode="chat_completions"` and `state="off"`, preserving optional function-call output items when the chat transport supplies them.+- `from_chat(...)` must produce an equivalent chat-completions result with `mode="chat_completions"` and `state="off"`, preserving optional function-call output items and provider usage when the chat transport supplies them.- Function-call output items must preserve `call_id` and optional acknowledged safety checks.- Argument parsing must tolerate JSON strings, dictionaries, and malformed values without throwing.diff --git a/tests/test_responses_architecture.py b/tests/test_responses_architecture.pyindex f344e9ebf..3eebf4c20 100644--- a/tests/test_responses_architecture.py+++ b/tests/test_responses_architecture.py@@ -129,6 +129,37 @@ def test_history_migrates_legacy_ai_metadata_and_preserves_tool_inputs():assert old.sequence == 0+async def test_chat_completion_transport_preserves_reported_usage(monkeypatch):+ async def fake_acompletion(**kwargs):+ return {+ "choices": [{"message": {"content": "done"}}],+ "usage": {+ "prompt_tokens": 120,+ "completion_tokens": 8,+ "total_tokens": 128,+ },+ "_hidden_params": {"response_cost": 0.0042},+ }++ monkeypatch.setattr(litellm_transport, "acompletion", fake_acompletion)+ transport = litellm_transport.LiteLLMTransport(+ model="custom/model",+ messages=[{"role": "user", "content": "question"}],+ kwargs={"a0_api_mode": "chat_completions"},+ )++ await transport.acomplete()++ assert transport.last_result is not None+ assert transport.last_result.usage == {+ "prompt_tokens": 120,+ "completion_tokens": 8,+ "total_tokens": 128,+ "cost": 0.0042,+ }++def test_responses_provider_state_uses_previous_response_and_new_items():new_items = [{"type": "function_call_output", "call_id": "call_1", "output": "done"}]local_items = [{"role": "user", "content": "full replay"}]@@ -362,6 +393,7 @@ async def test_unified_turn_waits_for_completed_native_responses_calls(monkeypat"id": "resp_parallel","output": calls,"usage": {"input_tokens": 10, "output_tokens": 5},+ "_hidden_params": {"response_cost": 0.0012},},},]@@ -394,7 +426,11 @@ async def test_unified_turn_waits_for_completed_native_responses_calls(monkeypatassert stream.closed is Falseassert result.mode == "responses"assert result.response_id == "resp_parallel"- assert result.usage == {"input_tokens": 10, "output_tokens": 5}+ assert result.usage == {+ "input_tokens": 10,+ "output_tokens": 5,+ "cost": 0.0012,+ }assert [call.name for call in result.function_calls] == ["lookup", "summarize"]assert json.loads(result.response) == {"tool_name": "parallel_tool_calls",diff --git a/tests/test_stream_tool_early_stop.py b/tests/test_stream_tool_early_stop.pyindex 6c26570e4..91f61b627 100644--- a/tests/test_stream_tool_early_stop.py+++ b/tests/test_stream_tool_early_stop.py@@ -1527,6 +1527,38 @@ def test_chat_completions_stream_parser_reads_dumped_tool_calls():}+def test_chat_completions_stream_parser_preserves_optional_usage():+ parser = litellm_transport.ChatCompletionsStreamParser()+ parser.parse(+ {+ "choices": [],+ "usage": {"prompt_tokens": 240},+ "_hidden_params": {"response_cost": 0.0084},+ }+ )+ parser.parse(+ {+ "choices": [],+ "usage": {"completion_tokens": 16, "total_tokens": 256},+ }+ )+ transport = litellm_transport.LiteLLMTransport(+ model="custom/model",+ messages=[{"role": "user", "content": "question"}],+ kwargs={"a0_api_mode": "chat_completions"},+ )++ result = transport._stream_result_from_chat_parser(parser)++ assert result is not None+ assert result.usage == {+ "prompt_tokens": 240,+ "completion_tokens": 16,+ "total_tokens": 256,+ "cost": 0.0084,+ }++@pytest.mark.asyncioasync def test_unified_turn_preserves_chat_streaming_tool_calls(monkeypatch):async def fake_acompletion(*args, **kwargs):