Skip to content

Agent Zero AI framework

Display native commentary as streamed thoughts

Alessandro55aee9d8574f
diff --git a/AGENTS.md b/AGENTS.md
index ff274ae6f..19dfbd60a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -20,6 +20,7 @@
- Native function calls use canonical call content for history and repeat comparison, even when accompanied by commentary. The canonical content passes through the normal history template/masking hook; provider output metadata remains intact.
- `initialize.py` owns framework initialization.
- `models.py` owns model-provider configuration and LiteLLM integration.
+- Native agent-turn response callbacks may receive a display-only commentary preview as their full text; legacy callbacks, accumulated model output, usage and native dispatch retain the original text and result metadata.
- `run_ui.py` is the WebUI entry point.
- `DockerfileLocal` must remain compatible with the contracts under `docker/`.
- Runtime or user state under `usr/` and `tmp/` is intentionally outside tracked DOX unless the user explicitly asks otherwise.
diff --git a/extensions/python/message_loop_result/AGENTS.md b/extensions/python/message_loop_result/AGENTS.md
index e1d2783a7..6453e7924 100644
--- a/extensions/python/message_loop_result/AGENTS.md
+++ b/extensions/python/message_loop_result/AGENTS.md
@@ -10,6 +10,7 @@
- `_20_empty_response.py` retries fully empty turns (no response and no reasoning): count them toward the unusable-response limit without adding a warning to model history, and use `fw.msg_empty_response.md` for agent-prefixed UI warning text only.
- `_20_empty_response.py` retries reasoning-only turns: add an agent-facing `fw.msg_reasoning_only.md` history warning with a separate `fw.msg_reasoning_only_response.md` user notice, and count them through the stop-unusable-response-loop extension.
- `_30_repeat_response.py` compares canonical function-call content when native calls exist, otherwise response text; it retries content that exactly matches `loop_data.last_response`, regardless of reasoning, using `fw.msg_repeat.md` for history and `fw.msg_repeat_response.md` for the agent-prefixed UI warning text. State advancement is owned by `hist_add_ai_response`; do not call `_remember_llm_result_state` manually.
+- `_40_native_thoughts.py` folds public commentary accompanying native function calls into the existing completed-generation log `thoughts` array. It preserves provider output items, canonical history/repeat comparison and dispatch. It respects handled results and excludes reasoning/encrypted items and explicit final-answer messages. Normal log masking applies.
## Local Contracts
diff --git a/extensions/python/message_loop_result/_40_native_thoughts.py b/extensions/python/message_loop_result/_40_native_thoughts.py
new file mode 100644
index 000000000..793b5bf6d
--- /dev/null
+++ b/extensions/python/message_loop_result/_40_native_thoughts.py
@@ -0,0 +1,35 @@
+"""Show public commentary beside native calls in the completed generation log."""
+import json
+
+from helpers.extension import Extension
+from extensions.python.before_main_llm_call._10_log_for_stream import build_heading
+
+
+class NativeThoughts(Extension):
+ def execute(self, result_data, loop_data, **kwargs):
+ if not self.agent or result_data.get("skip_default_processing"):
+ return
+ result = result_data["llm_result"]
+ log_item = loop_data.params_temporary.get("log_item_generating")
+ if not result.function_calls or log_item is None:
+ return
+ thoughts = []
+ for item in result.output_items:
+ if (item.type != "message" or item.data.get("phase") not in (None, "commentary")
+ or item.data.get("role") not in (None, "assistant")):
+ continue
+ content = item.data.get("content")
+ if not isinstance(content, list):
+ continue
+ text = "".join(block["text"] for block in content
+ if isinstance(block, dict) and block.get("type") == "output_text"
+ and isinstance(block.get("text"), str)).strip()
+ if text:
+ thoughts.append(text)
+ if not thoughts:
+ return
+ action = {"thoughts": thoughts, **json.loads(result.function_calls_text())}
+ kvps = {**(log_item.kvps or {}), **action}
+ kvps.pop("step", None)
+ log_item.update(heading=build_heading(self.agent, f"Using {action['tool_name']}"),
+ content=json.dumps(action, ensure_ascii=False), kvps=kvps)
diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.py
index 9cedac2af..aab6399f0 100644
--- a/helpers/litellm_transport.py
+++ b/helpers/litellm_transport.py
@@ -586,14 +586,16 @@ class ChatCompletionsTransport:
reasoning_delta, final=reasoning_filter is None
)
parsed = {"reasoning_delta": reasoning_delta, "response_delta": response_delta}
- if not response_delta:
- tool_calls = _as_list(_get_value(message, "tool_calls"))
- response_delta = ChatCompletionsTransport.tool_calls_text(tool_calls)
+ tool_calls = _as_list(_get_value(message, "tool_calls"))
+ call_text = ChatCompletionsTransport.tool_calls_text(tool_calls)
+ if call_text:
+ parsed["response_delta"] = call_text
+ parsed["_output_items"] = ChatCompletionsTransport.output_items(tool_calls)
if response_delta:
- parsed["response_delta"] = response_delta
- parsed["_output_items"] = ChatCompletionsTransport.output_items(
- tool_calls
- )
+ parsed["_output_items"].insert(0, {
+ "type": "message", "role": "assistant",
+ "content": [{"type": "output_text", "text": response_delta}],
+ })
return parsed
@classmethod
@@ -1236,6 +1238,7 @@ class ResponsesEventParser:
self.emitted_function_calls: set[str] = set()
self.streamed_function_calls: dict[str, str] = {}
self.pending_function_calls: dict[str, Any] = {}
+ self.commentary: dict[str, str] = {}
self.seen_response_delta = False
self.seen_reasoning_delta = False
self.completed_response: Any = None
@@ -1281,6 +1284,7 @@ class ResponsesEventParser:
event_type = _get_value(event, "type") or ""
response_delta = ""
reasoning_delta = ""
+ is_commentary = False
if event_type in {
"response.output_text.delta",
@@ -1288,6 +1292,11 @@ class ResponsesEventParser:
"response.text.delta",
}:
response_delta = str(_get_value(event, "delta") or "")
+ key = self._event_key(event)
+ item = self.output_items.get(key, {})
+ if event_type != "response.refusal.delta" and item.get("phase") == "commentary":
+ self.commentary[key] = self.commentary.get(key, "") + response_delta
+ is_commentary = True
elif event_type in {
"response.reasoning_summary_text.delta",
"response.reasoning_text.delta",
@@ -1317,7 +1326,30 @@ class ResponsesEventParser:
if reasoning_delta:
self.seen_reasoning_delta = True
- return {"reasoning_delta": reasoning_delta, "response_delta": response_delta}
+ parsed = {"reasoning_delta": reasoning_delta, "response_delta": response_delta}
+ is_call = event_type in {
+ "response.function_call_arguments.delta", "response.function_call_arguments.done",
+ } or event_type == "response.output_item.done"
+ if self.commentary and response_delta and (is_commentary or is_call):
+ parsed["response_preview"] = self._commentary_preview()
+ return parsed
+
+ def _commentary_preview(self) -> str:
+ preview = {"thoughts": list(self.commentary.values())}
+ calls = [(key, item) for key, item in self.function_calls.items()
+ if key in self.emitted_function_calls or key in self.streamed_function_calls]
+ if len(calls) == 1:
+ key, item = calls[0]
+ if key in self.streamed_function_calls:
+ return (json.dumps(preview, ensure_ascii=False)[:-1]
+ + ',"tool_name":' + json.dumps(item["name"], ensure_ascii=False)
+ + ',"tool_args":' + item["arguments"])
+ preview.update(ResponsesTransport.function_call_object(item))
+ elif calls:
+ preview.update(tool_name="parallel_tool_calls", tool_args={
+ "calls": [ResponsesTransport.function_call_object(item) for _, item in calls],
+ })
+ return json.dumps(preview, ensure_ascii=False)
def _remember_output_item(self, item: Any, event: Any) -> str:
item_type = _get_value(item, "type")
diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.md
index 277670b48..42bbf5e83 100644
--- a/helpers/litellm_transport.py.dox.md
+++ b/helpers/litellm_transport.py.dox.md
@@ -34,7 +34,8 @@
- Treat opaque type-discrimination errors such as `cannot determine type` from OpenAI-compatible Responses endpoints as shape-specific rejections.
- Fall back to Chat Completions when a Responses endpoint fails before output with an endpoint-specific server error, proxy path-unavailable error, or LiteLLM proxy-extra import error.
- Fall back to Chat Completions when LiteLLM's Responses mock streaming path tries to JSON-decode a real SSE stream before any output.
-- Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items.
+- Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items. Non-streaming calls stay canonical when accompanied by commentary; retain that commentary as a separate output message.
+- Responses messages explicitly marked `phase: commentary` produce an incremental `response_preview` with public text in `thoughts`, followed by the native tool preview. Previews reuse parsed call state; multiple calls form one parallel envelope. Only native agent-turn callbacks consume the preview: legacy callbacks, response/reasoning deltas, completed output items and call IDs remain unchanged. Unmarked/final-answer text and reasoning are not reclassified as commentary.
- Preserve provider usage and LiteLLM response cost for both transports only when the response or stream actually supplies them; do not synthesize unavailable provider accounting.
- Streaming Chat Completions requests include `stream_options.include_usage` so terminal usage events reach `LLMResult.usage` for OpenAI-compatible endpoints and Messages API providers alike, unless the model configuration previously rejected the option or the request already carries it. A pre-output rejection retries once without the transport-injected `include_usage` and remembers the rejection per model configuration in a process-global cache that naturally clears when the framework process restarts; tests reset it through `clear_transport_capability_cache`. User-provided `stream_options` keys are always passed through untouched, even when the provider rejects them.
- Recover all Responses output items from stream events when a terminal completed envelope omits them, including encrypted reasoning. Merge by item/call identity in output-index order; terminal non-null fields win without duplicating calls.
diff --git a/models.py b/models.py
index 8730832f4..2ed41dd5c 100644
--- a/models.py
+++ b/models.py
@@ -727,7 +727,8 @@ class LiteLLMChatWrapper(SimpleChatModel):
if output["response_delta"]:
if response_callback:
stop_response = await response_callback(
- output["response_delta"], result.response
+ output["response_delta"],
+ parsed.get("response_preview", result.response),
)
if tokens_callback:
await tokens_callback(
diff --git a/prompts/AGENTS.md b/prompts/AGENTS.md
index 0f54384d9..fe976a894 100644
--- a/prompts/AGENTS.md
+++ b/prompts/AGENTS.md
@@ -17,6 +17,7 @@
- Do not include secrets, real API keys, or private user data in prompt templates.
- Keep placeholder names, include aliases, and template assumptions synchronized with prompt-loading code and extensions.
- `agent.system.main.communication.native.md` owns native function-call formatting; profiles may override it alongside the legacy communication template. Shared communication additions remain transport-neutral; the brace terminator belongs only to legacy JSON formatting.
+- Native communication requests public commentary without a sentence limit; the display maps it to thoughts without adding fields to native arguments.
- Prompt changes can alter agent behavior; keep edits narrow and intentional.
- Shared solving guidance owns observable success, evidence-driven replanning, task progress notes, source-preserving synthesis, and bug reproduction; specialist methods belong in profile specifics.
- Maintain clear separation between core behavior prompts and profile/plugin-specific customization.
diff --git a/prompts/agent.system.main.communication.native.md b/prompts/agent.system.main.communication.native.md
index c95d29e9f..1cf5becc7 100644
--- a/prompts/agent.system.main.communication.native.md
+++ b/prompts/agent.system.main.communication.native.md
@@ -1,6 +1,7 @@
## Communication
- Use the provided native functions for tool calls. Put only the function arguments in the call.
- Do not emit an Agent Zero JSON envelope, `thoughts`, `headline`, `tool_name`, or `tool_args` as a text response.
+- Before native calls, provide public commentary.
- Tool examples in older messages describe previous actions; follow the current native function definitions for new calls.
- Call one tool, then wait for its result before choosing the next dependent action. Do not append a second tool call as text.
- Finish through the native response function when the task is done. Its text is the answer shown to the user.
diff --git a/tests/test_responses_architecture.py b/tests/test_responses_architecture.py
index 135805d11..0f2644f10 100644
--- a/tests/test_responses_architecture.py
+++ b/tests/test_responses_architecture.py
@@ -44,6 +44,131 @@ class _AsyncEventStream:
self.closed = True
+async def test_commentary_streams_as_thoughts_without_changing_native_result(monkeypatch):
+ message = {"type": "message", "id": "msg_progress", "role": "assistant",
+ "phase": "commentary", "content": []}
+ call = {"type": "function_call", "id": "fc_progress", "call_id": "call_progress",
+ "name": "lookup", "arguments": '{"q":"a0"}'}
+ message_done = {**message, "content": [{"type": "output_text", "text": 'Reading "a0" now.'}]}
+ stream = _AsyncEventStream([
+ {"type": "response.output_item.added", "output_index": 0, "item": message},
+ {"type": "response.output_text.delta", "item_id": message["id"], "delta": 'Reading "a0"'},
+ {"type": "response.output_text.delta", "item_id": message["id"], "delta": " now."},
+ {"type": "response.output_item.done", "output_index": 0, "item": message_done},
+ {"type": "response.output_item.added", "output_index": 1, "item": {**call, "arguments": ""}},
+ {"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '{"q":'},
+ {"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '"a0"}'},
+ {"type": "response.function_call_arguments.done", "item_id": call["id"], "arguments": call["arguments"]},
+ {"type": "response.completed", "response": {"id": "resp_progress", "output": [message_done, call]}},
+ ])
+
+ async def fake_responses(**kwargs):
+ return stream
+
+ async def no_limiter(*args, **kwargs):
+ return None
+
+ monkeypatch.setattr(litellm_transport, "aresponses", fake_responses)
+ monkeypatch.setattr(models, "apply_rate_limiter", no_limiter)
+ wrapper = models.LiteLLMChatWrapper(model="test", provider="openai", model_config=None,
+ a0_api_mode="responses")
+ previews = []
+
+ async def preview(chunk, full):
+ previews.append((stream.index, chunk, full))
+ return full if extract_tools.extract_tool_request(full) else None
+
+ result = await wrapper.unified_turn(messages=[HumanMessage(content="hi")], response_callback=preview)
+ assert [json.loads(row[2])["thoughts"] for row in previews[:2]] == [['Reading "a0"'], ['Reading "a0" now.']]
+ assert [row[1] for row in previews[:2]] == ['Reading "a0"', ' now.']
+ assert previews[0][0] == 2 and previews[1][0] == 3
+ assert json.loads(previews[-1][2]) == {"thoughts": ['Reading "a0" now.'], "tool_name": "lookup", "tool_args": {"q": "a0"}}
+ assert stream.index == 9 and not stream.closed
+ assert [item.to_dict() for item in result.output_items] == [message_done, call]
+ assert json.loads(result.function_calls_text()) == {"tool_name": "lookup", "tool_args": {"q": "a0"}}
+
+
[email protected]("phase", [None, "final_answer"])
+def test_non_commentary_stream_text_has_no_thoughts_preview(phase):
+ parser = litellm_transport.ResponsesEventParser()
+ parser.parse({"type": "response.output_item.added", "output_index": 0,
+ "item": {"type": "message", "id": "msg_final", "phase": phase}})
+ assert parser.parse({"type": "response.output_text.delta", "item_id": "msg_final", "delta": "Done"}) == {
+ "response_delta": "Done", "reasoning_delta": "",
+ }
+
+
+async def test_legacy_callback_still_receives_accumulated_model_text(monkeypatch):
+ stream = _AsyncEventStream([
+ {"type": "response.output_item.added", "item": {"id": "progress", "type": "message", "phase": "commentary"}},
+ {"type": "response.output_text.delta", "item_id": "progress", "delta": "Reading now."},
+ ])
+
+ async def fake_responses(**kwargs):
+ return stream
+
+ async def no_limiter(*args, **kwargs):
+ return None
+
+ async def stop_after_chunk(chunk, full):
+ assert full == chunk == "Reading now."
+ return full
+
+ monkeypatch.setattr(litellm_transport, "aresponses", fake_responses)
+ monkeypatch.setattr(models, "apply_rate_limiter", no_limiter)
+ wrapper = models.LiteLLMChatWrapper(model="test", provider="openai", model_config=None,
+ a0_api_mode="responses")
+ response, _ = await wrapper.unified_call(user_message="hi", response_callback=stop_after_chunk)
+ assert response == "Reading now."
+
+
+def test_commentary_preview_keeps_multiple_native_calls_in_one_envelope():
+ parser = litellm_transport.ResponsesEventParser()
+ parser.parse({"type": "response.output_item.added", "item": {"id": "progress", "type": "message", "phase": "commentary"}})
+ parser.parse({"type": "response.output_text.delta", "item_id": "progress", "delta": "Reading both files."})
+ for index in (1, 2):
+ call = {"id": f"fc_{index}", "call_id": f"call_{index}", "type": "function_call", "name": "lookup", "arguments": ""}
+ parser.parse({"type": "response.output_item.added", "output_index": index, "item": call})
+ parser.parse({"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '{"q":'})
+ if index == 2:
+ partial = parser.parse({"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '"second"'})
+ assert json.loads(partial["response_preview"])["tool_name"] == "parallel_tool_calls"
+ parsed = parser.parse({"type": "response.function_call_arguments.done", "item_id": call["id"], "arguments": json.dumps({"q": index})})
+ assert json.loads(parsed["response_preview"]) == {
+ "thoughts": ["Reading both files."], "tool_name": "parallel_tool_calls",
+ "tool_args": {"calls": [{"tool_name": "lookup", "tool_args": {"q": index}} for index in (1, 2)]},
+ }
+
+
+def test_native_generation_thoughts_preserve_result_and_mask_log(monkeypatch):
+ from types import SimpleNamespace
+ from extensions.python.message_loop_result._40_native_thoughts import NativeThoughts
+
+ result = LLMResult.from_response({"output": [
+ {"type": "message", "phase": "commentary", "content": [
+ {"type": "output_text", "text": "Reading sample-secret"}]},
+ {"type": "reasoning", "encrypted_content": "opaque"},
+ {"type": "function_call", "name": "lookup", "call_id": "call_a", "arguments": '{"q":"a0"}'},
+ ]})
+ original = json.dumps(result.to_dict(), sort_keys=True)
+ monkeypatch.setattr("helpers.log.get_secrets_manager", lambda context: SimpleNamespace(
+ mask_values=lambda text: text.replace("sample-secret", "[MASKED]")))
+ log = Log()
+ item = log.log(type="agent", kvps={"reasoning": "existing summary"})
+ loop = LoopData()
+ loop.params_temporary["log_item_generating"] = item
+ agent = SimpleNamespace(agent_name="A0")
+ NativeThoughts(agent).execute({"llm_result": result}, loop)
+ assert item.kvps["thoughts"] == ["Reading [MASKED]"]
+ assert "headline" not in item.kvps
+ assert item.heading == "A0: Using lookup"
+ assert item.kvps["reasoning"] == "existing summary"
+ assert json.loads(item.content) == {"thoughts": ["Reading [MASKED]"], "tool_name": "lookup", "tool_args": {"q": "a0"}}
+ assert json.dumps(result.to_dict(), sort_keys=True) == original
+
+
def test_responses_function_call_text_preserves_non_ascii_tool_args():
result = LLMResult.from_response(
{
diff --git a/tests/test_stream_tool_early_stop.py b/tests/test_stream_tool_early_stop.py
index 90a71935b..704cd9275 100644
--- a/tests/test_stream_tool_early_stop.py
+++ b/tests/test_stream_tool_early_stop.py
@@ -1449,12 +1449,14 @@ def test_responses_response_parser_extracts_text_reasoning_and_function_calls():
}
-def test_chat_completions_response_parser_extracts_tool_calls():
[email protected]("commentary", [None, "Looking it up."])
+def test_chat_completions_response_parser_extracts_tool_calls(commentary):
parsed = litellm_transport.ChatCompletionsTransport.parse(
{
"choices": [
{
"message": {
+ "content": commentary,
"tool_calls": [
{
"id": "call_1",
@@ -1475,7 +1477,17 @@ def test_chat_completions_response_parser_extracts_tool_calls():
"tool_name": "lookup",
"tool_args": {"q": "a0"},
}
- assert parsed["_output_items"][0]["name"] == "lookup"
+ items = parsed["_output_items"]
+ assert items[-1]["name"] == "lookup"
+ assert items[-1]["call_id"] == "call_1"
+ if commentary:
+ assert items[0] == {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": commentary}],
+ }
+ else:
+ assert len(items) == 1
def test_chat_completions_stream_parser_accumulates_tool_call_arguments():