Agent Zero AI framework
Display native commentary as streamed thoughts
Alessandro55aee9d8574f
diff --git a/AGENTS.md b/AGENTS.mdindex ff274ae6f..19dfbd60a 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -20,6 +20,7 @@- Native function calls use canonical call content for history and repeat comparison, even when accompanied by commentary. The canonical content passes through the normal history template/masking hook; provider output metadata remains intact.- `initialize.py` owns framework initialization.- `models.py` owns model-provider configuration and LiteLLM integration.+- Native agent-turn response callbacks may receive a display-only commentary preview as their full text; legacy callbacks, accumulated model output, usage and native dispatch retain the original text and result metadata.- `run_ui.py` is the WebUI entry point.- `DockerfileLocal` must remain compatible with the contracts under `docker/`.- Runtime or user state under `usr/` and `tmp/` is intentionally outside tracked DOX unless the user explicitly asks otherwise.diff --git a/extensions/python/message_loop_result/AGENTS.md b/extensions/python/message_loop_result/AGENTS.mdindex e1d2783a7..6453e7924 100644--- a/extensions/python/message_loop_result/AGENTS.md+++ b/extensions/python/message_loop_result/AGENTS.md@@ -10,6 +10,7 @@- `_20_empty_response.py` retries fully empty turns (no response and no reasoning): count them toward the unusable-response limit without adding a warning to model history, and use `fw.msg_empty_response.md` for agent-prefixed UI warning text only.- `_20_empty_response.py` retries reasoning-only turns: add an agent-facing `fw.msg_reasoning_only.md` history warning with a separate `fw.msg_reasoning_only_response.md` user notice, and count them through the stop-unusable-response-loop extension.- `_30_repeat_response.py` compares canonical function-call content when native calls exist, otherwise response text; it retries content that exactly matches `loop_data.last_response`, regardless of reasoning, using `fw.msg_repeat.md` for history and `fw.msg_repeat_response.md` for the agent-prefixed UI warning text. State advancement is owned by `hist_add_ai_response`; do not call `_remember_llm_result_state` manually.+- `_40_native_thoughts.py` folds public commentary accompanying native function calls into the existing completed-generation log `thoughts` array. It preserves provider output items, canonical history/repeat comparison and dispatch. It respects handled results and excludes reasoning/encrypted items and explicit final-answer messages. Normal log masking applies.## Local Contractsdiff --git a/extensions/python/message_loop_result/_40_native_thoughts.py b/extensions/python/message_loop_result/_40_native_thoughts.pynew file mode 100644index 000000000..793b5bf6d--- /dev/null+++ b/extensions/python/message_loop_result/_40_native_thoughts.py@@ -0,0 +1,35 @@+"""Show public commentary beside native calls in the completed generation log."""+import json++from helpers.extension import Extension+from extensions.python.before_main_llm_call._10_log_for_stream import build_heading+++class NativeThoughts(Extension):+ def execute(self, result_data, loop_data, **kwargs):+ if not self.agent or result_data.get("skip_default_processing"):+ return+ result = result_data["llm_result"]+ log_item = loop_data.params_temporary.get("log_item_generating")+ if not result.function_calls or log_item is None:+ return+ thoughts = []+ for item in result.output_items:+ if (item.type != "message" or item.data.get("phase") not in (None, "commentary")+ or item.data.get("role") not in (None, "assistant")):+ continue+ content = item.data.get("content")+ if not isinstance(content, list):+ continue+ text = "".join(block["text"] for block in content+ if isinstance(block, dict) and block.get("type") == "output_text"+ and isinstance(block.get("text"), str)).strip()+ if text:+ thoughts.append(text)+ if not thoughts:+ return+ action = {"thoughts": thoughts, **json.loads(result.function_calls_text())}+ kvps = {**(log_item.kvps or {}), **action}+ kvps.pop("step", None)+ log_item.update(heading=build_heading(self.agent, f"Using {action['tool_name']}"),+ content=json.dumps(action, ensure_ascii=False), kvps=kvps)diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.pyindex 9cedac2af..aab6399f0 100644--- a/helpers/litellm_transport.py+++ b/helpers/litellm_transport.py@@ -586,14 +586,16 @@ class ChatCompletionsTransport:reasoning_delta, final=reasoning_filter is None)parsed = {"reasoning_delta": reasoning_delta, "response_delta": response_delta}- if not response_delta:- tool_calls = _as_list(_get_value(message, "tool_calls"))- response_delta = ChatCompletionsTransport.tool_calls_text(tool_calls)+ tool_calls = _as_list(_get_value(message, "tool_calls"))+ call_text = ChatCompletionsTransport.tool_calls_text(tool_calls)+ if call_text:+ parsed["response_delta"] = call_text+ parsed["_output_items"] = ChatCompletionsTransport.output_items(tool_calls)if response_delta:- parsed["response_delta"] = response_delta- parsed["_output_items"] = ChatCompletionsTransport.output_items(- tool_calls- )+ parsed["_output_items"].insert(0, {+ "type": "message", "role": "assistant",+ "content": [{"type": "output_text", "text": response_delta}],+ })return parsed@classmethod@@ -1236,6 +1238,7 @@ class ResponsesEventParser:self.emitted_function_calls: set[str] = set()self.streamed_function_calls: dict[str, str] = {}self.pending_function_calls: dict[str, Any] = {}+ self.commentary: dict[str, str] = {}self.seen_response_delta = Falseself.seen_reasoning_delta = Falseself.completed_response: Any = None@@ -1281,6 +1284,7 @@ class ResponsesEventParser:event_type = _get_value(event, "type") or ""response_delta = ""reasoning_delta = ""+ is_commentary = Falseif event_type in {"response.output_text.delta",@@ -1288,6 +1292,11 @@ class ResponsesEventParser:"response.text.delta",}:response_delta = str(_get_value(event, "delta") or "")+ key = self._event_key(event)+ item = self.output_items.get(key, {})+ if event_type != "response.refusal.delta" and item.get("phase") == "commentary":+ self.commentary[key] = self.commentary.get(key, "") + response_delta+ is_commentary = Trueelif event_type in {"response.reasoning_summary_text.delta","response.reasoning_text.delta",@@ -1317,7 +1326,30 @@ class ResponsesEventParser:if reasoning_delta:self.seen_reasoning_delta = True- return {"reasoning_delta": reasoning_delta, "response_delta": response_delta}+ parsed = {"reasoning_delta": reasoning_delta, "response_delta": response_delta}+ is_call = event_type in {+ "response.function_call_arguments.delta", "response.function_call_arguments.done",+ } or event_type == "response.output_item.done"+ if self.commentary and response_delta and (is_commentary or is_call):+ parsed["response_preview"] = self._commentary_preview()+ return parsed++ def _commentary_preview(self) -> str:+ preview = {"thoughts": list(self.commentary.values())}+ calls = [(key, item) for key, item in self.function_calls.items()+ if key in self.emitted_function_calls or key in self.streamed_function_calls]+ if len(calls) == 1:+ key, item = calls[0]+ if key in self.streamed_function_calls:+ return (json.dumps(preview, ensure_ascii=False)[:-1]+ + ',"tool_name":' + json.dumps(item["name"], ensure_ascii=False)+ + ',"tool_args":' + item["arguments"])+ preview.update(ResponsesTransport.function_call_object(item))+ elif calls:+ preview.update(tool_name="parallel_tool_calls", tool_args={+ "calls": [ResponsesTransport.function_call_object(item) for _, item in calls],+ })+ return json.dumps(preview, ensure_ascii=False)def _remember_output_item(self, item: Any, event: Any) -> str:item_type = _get_value(item, "type")diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.mdindex 277670b48..42bbf5e83 100644--- a/helpers/litellm_transport.py.dox.md+++ b/helpers/litellm_transport.py.dox.md@@ -34,7 +34,8 @@- Treat opaque type-discrimination errors such as `cannot determine type` from OpenAI-compatible Responses endpoints as shape-specific rejections.- Fall back to Chat Completions when a Responses endpoint fails before output with an endpoint-specific server error, proxy path-unavailable error, or LiteLLM proxy-extra import error.- Fall back to Chat Completions when LiteLLM's Responses mock streaming path tries to JSON-decode a real SSE stream before any output.-- Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items.+- Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items. Non-streaming calls stay canonical when accompanied by commentary; retain that commentary as a separate output message.+- Responses messages explicitly marked `phase: commentary` produce an incremental `response_preview` with public text in `thoughts`, followed by the native tool preview. Previews reuse parsed call state; multiple calls form one parallel envelope. Only native agent-turn callbacks consume the preview: legacy callbacks, response/reasoning deltas, completed output items and call IDs remain unchanged. Unmarked/final-answer text and reasoning are not reclassified as commentary.- Preserve provider usage and LiteLLM response cost for both transports only when the response or stream actually supplies them; do not synthesize unavailable provider accounting.- Streaming Chat Completions requests include `stream_options.include_usage` so terminal usage events reach `LLMResult.usage` for OpenAI-compatible endpoints and Messages API providers alike, unless the model configuration previously rejected the option or the request already carries it. A pre-output rejection retries once without the transport-injected `include_usage` and remembers the rejection per model configuration in a process-global cache that naturally clears when the framework process restarts; tests reset it through `clear_transport_capability_cache`. User-provided `stream_options` keys are always passed through untouched, even when the provider rejects them.- Recover all Responses output items from stream events when a terminal completed envelope omits them, including encrypted reasoning. Merge by item/call identity in output-index order; terminal non-null fields win without duplicating calls.diff --git a/models.py b/models.pyindex 8730832f4..2ed41dd5c 100644--- a/models.py+++ b/models.py@@ -727,7 +727,8 @@ class LiteLLMChatWrapper(SimpleChatModel):if output["response_delta"]:if response_callback:stop_response = await response_callback(- output["response_delta"], result.response+ output["response_delta"],+ parsed.get("response_preview", result.response),)if tokens_callback:await tokens_callback(diff --git a/prompts/AGENTS.md b/prompts/AGENTS.mdindex 0f54384d9..fe976a894 100644--- a/prompts/AGENTS.md+++ b/prompts/AGENTS.md@@ -17,6 +17,7 @@- Do not include secrets, real API keys, or private user data in prompt templates.- Keep placeholder names, include aliases, and template assumptions synchronized with prompt-loading code and extensions.- `agent.system.main.communication.native.md` owns native function-call formatting; profiles may override it alongside the legacy communication template. Shared communication additions remain transport-neutral; the brace terminator belongs only to legacy JSON formatting.+- Native communication requests public commentary without a sentence limit; the display maps it to thoughts without adding fields to native arguments.- Prompt changes can alter agent behavior; keep edits narrow and intentional.- Shared solving guidance owns observable success, evidence-driven replanning, task progress notes, source-preserving synthesis, and bug reproduction; specialist methods belong in profile specifics.- Maintain clear separation between core behavior prompts and profile/plugin-specific customization.diff --git a/prompts/agent.system.main.communication.native.md b/prompts/agent.system.main.communication.native.mdindex c95d29e9f..1cf5becc7 100644--- a/prompts/agent.system.main.communication.native.md+++ b/prompts/agent.system.main.communication.native.md@@ -1,6 +1,7 @@## Communication- Use the provided native functions for tool calls. Put only the function arguments in the call.- Do not emit an Agent Zero JSON envelope, `thoughts`, `headline`, `tool_name`, or `tool_args` as a text response.+- Before native calls, provide public commentary.- Tool examples in older messages describe previous actions; follow the current native function definitions for new calls.- Call one tool, then wait for its result before choosing the next dependent action. Do not append a second tool call as text.- Finish through the native response function when the task is done. Its text is the answer shown to the user.diff --git a/tests/test_responses_architecture.py b/tests/test_responses_architecture.pyindex 135805d11..0f2644f10 100644--- a/tests/test_responses_architecture.py+++ b/tests/test_responses_architecture.py@@ -44,6 +44,131 @@ class _AsyncEventStream:self.closed = True+async def test_commentary_streams_as_thoughts_without_changing_native_result(monkeypatch):+ message = {"type": "message", "id": "msg_progress", "role": "assistant",+ "phase": "commentary", "content": []}+ call = {"type": "function_call", "id": "fc_progress", "call_id": "call_progress",+ "name": "lookup", "arguments": '{"q":"a0"}'}+ message_done = {**message, "content": [{"type": "output_text", "text": 'Reading "a0" now.'}]}+ stream = _AsyncEventStream([+ {"type": "response.output_item.added", "output_index": 0, "item": message},+ {"type": "response.output_text.delta", "item_id": message["id"], "delta": 'Reading "a0"'},+ {"type": "response.output_text.delta", "item_id": message["id"], "delta": " now."},+ {"type": "response.output_item.done", "output_index": 0, "item": message_done},+ {"type": "response.output_item.added", "output_index": 1, "item": {**call, "arguments": ""}},+ {"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '{"q":'},+ {"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '"a0"}'},+ {"type": "response.function_call_arguments.done", "item_id": call["id"], "arguments": call["arguments"]},+ {"type": "response.completed", "response": {"id": "resp_progress", "output": [message_done, call]}},+ ])++ async def fake_responses(**kwargs):+ return stream++ async def no_limiter(*args, **kwargs):+ return None++ monkeypatch.setattr(litellm_transport, "aresponses", fake_responses)+ monkeypatch.setattr(models, "apply_rate_limiter", no_limiter)+ wrapper = models.LiteLLMChatWrapper(model="test", provider="openai", model_config=None,+ a0_api_mode="responses")+ previews = []++ async def preview(chunk, full):+ previews.append((stream.index, chunk, full))+ return full if extract_tools.extract_tool_request(full) else None++ result = await wrapper.unified_turn(messages=[HumanMessage(content="hi")], response_callback=preview)+ assert [json.loads(row[2])["thoughts"] for row in previews[:2]] == [['Reading "a0"'], ['Reading "a0" now.']]+ assert [row[1] for row in previews[:2]] == ['Reading "a0"', ' now.']+ assert previews[0][0] == 2 and previews[1][0] == 3+ assert json.loads(previews[-1][2]) == {"thoughts": ['Reading "a0" now.'], "tool_name": "lookup", "tool_args": {"q": "a0"}}+ assert stream.index == 9 and not stream.closed+ assert [item.to_dict() for item in result.output_items] == [message_done, call]+ assert json.loads(result.function_calls_text()) == {"tool_name": "lookup", "tool_args": {"q": "a0"}}++[email protected]("phase", [None, "final_answer"])+def test_non_commentary_stream_text_has_no_thoughts_preview(phase):+ parser = litellm_transport.ResponsesEventParser()+ parser.parse({"type": "response.output_item.added", "output_index": 0,+ "item": {"type": "message", "id": "msg_final", "phase": phase}})+ assert parser.parse({"type": "response.output_text.delta", "item_id": "msg_final", "delta": "Done"}) == {+ "response_delta": "Done", "reasoning_delta": "",+ }+++async def test_legacy_callback_still_receives_accumulated_model_text(monkeypatch):+ stream = _AsyncEventStream([+ {"type": "response.output_item.added", "item": {"id": "progress", "type": "message", "phase": "commentary"}},+ {"type": "response.output_text.delta", "item_id": "progress", "delta": "Reading now."},+ ])++ async def fake_responses(**kwargs):+ return stream++ async def no_limiter(*args, **kwargs):+ return None++ async def stop_after_chunk(chunk, full):+ assert full == chunk == "Reading now."+ return full++ monkeypatch.setattr(litellm_transport, "aresponses", fake_responses)+ monkeypatch.setattr(models, "apply_rate_limiter", no_limiter)+ wrapper = models.LiteLLMChatWrapper(model="test", provider="openai", model_config=None,+ a0_api_mode="responses")+ response, _ = await wrapper.unified_call(user_message="hi", response_callback=stop_after_chunk)+ assert response == "Reading now."+++def test_commentary_preview_keeps_multiple_native_calls_in_one_envelope():+ parser = litellm_transport.ResponsesEventParser()+ parser.parse({"type": "response.output_item.added", "item": {"id": "progress", "type": "message", "phase": "commentary"}})+ parser.parse({"type": "response.output_text.delta", "item_id": "progress", "delta": "Reading both files."})+ for index in (1, 2):+ call = {"id": f"fc_{index}", "call_id": f"call_{index}", "type": "function_call", "name": "lookup", "arguments": ""}+ parser.parse({"type": "response.output_item.added", "output_index": index, "item": call})+ parser.parse({"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '{"q":'})+ if index == 2:+ partial = parser.parse({"type": "response.function_call_arguments.delta", "item_id": call["id"], "delta": '"second"'})+ assert json.loads(partial["response_preview"])["tool_name"] == "parallel_tool_calls"+ parsed = parser.parse({"type": "response.function_call_arguments.done", "item_id": call["id"], "arguments": json.dumps({"q": index})})+ assert json.loads(parsed["response_preview"]) == {+ "thoughts": ["Reading both files."], "tool_name": "parallel_tool_calls",+ "tool_args": {"calls": [{"tool_name": "lookup", "tool_args": {"q": index}} for index in (1, 2)]},+ }+++def test_native_generation_thoughts_preserve_result_and_mask_log(monkeypatch):+ from types import SimpleNamespace+ from extensions.python.message_loop_result._40_native_thoughts import NativeThoughts++ result = LLMResult.from_response({"output": [+ {"type": "message", "phase": "commentary", "content": [+ {"type": "output_text", "text": "Reading sample-secret"}]},+ {"type": "reasoning", "encrypted_content": "opaque"},+ {"type": "function_call", "name": "lookup", "call_id": "call_a", "arguments": '{"q":"a0"}'},+ ]})+ original = json.dumps(result.to_dict(), sort_keys=True)+ monkeypatch.setattr("helpers.log.get_secrets_manager", lambda context: SimpleNamespace(+ mask_values=lambda text: text.replace("sample-secret", "[MASKED]")))+ log = Log()+ item = log.log(type="agent", kvps={"reasoning": "existing summary"})+ loop = LoopData()+ loop.params_temporary["log_item_generating"] = item+ agent = SimpleNamespace(agent_name="A0")+ NativeThoughts(agent).execute({"llm_result": result}, loop)+ assert item.kvps["thoughts"] == ["Reading [MASKED]"]+ assert "headline" not in item.kvps+ assert item.heading == "A0: Using lookup"+ assert item.kvps["reasoning"] == "existing summary"+ assert json.loads(item.content) == {"thoughts": ["Reading [MASKED]"], "tool_name": "lookup", "tool_args": {"q": "a0"}}+ assert json.dumps(result.to_dict(), sort_keys=True) == original++def test_responses_function_call_text_preserves_non_ascii_tool_args():result = LLMResult.from_response({diff --git a/tests/test_stream_tool_early_stop.py b/tests/test_stream_tool_early_stop.pyindex 90a71935b..704cd9275 100644--- a/tests/test_stream_tool_early_stop.py+++ b/tests/test_stream_tool_early_stop.py@@ -1449,12 +1449,14 @@ def test_responses_response_parser_extracts_text_reasoning_and_function_calls():}-def test_chat_completions_response_parser_extracts_tool_calls():[email protected]("commentary", [None, "Looking it up."])+def test_chat_completions_response_parser_extracts_tool_calls(commentary):parsed = litellm_transport.ChatCompletionsTransport.parse({"choices": [{"message": {+ "content": commentary,"tool_calls": [{"id": "call_1",@@ -1475,7 +1477,17 @@ def test_chat_completions_response_parser_extracts_tool_calls():"tool_name": "lookup","tool_args": {"q": "a0"},}- assert parsed["_output_items"][0]["name"] == "lookup"+ items = parsed["_output_items"]+ assert items[-1]["name"] == "lookup"+ assert items[-1]["call_id"] == "call_1"+ if commentary:+ assert items[0] == {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": commentary}],+ }+ else:+ assert len(items) == 1def test_chat_completions_stream_parser_accumulates_tool_call_arguments():