Skip to content

A programming framework for agentic AI

fix: Address tool call execution scenario when model produces empty tool call ids (#5509)

Eric Zhue7a3c7859458
diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py
index 6bba99241..3b109c196 100644
--- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py
+++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py
@@ -11,6 +11,7 @@ from typing import (
List,
Mapping,
Sequence,
+ Tuple,
)
from autogen_core import CancellationToken, Component, ComponentModel, FunctionCall
@@ -441,28 +442,26 @@ class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]):
inner_messages.append(tool_call_msg)
yield tool_call_msg
- # Execute the tool calls.
- exec_results = await asyncio.gather(
+ # Execute the tool calls and hanoff calls.
+ executed_calls_and_results = await asyncio.gather(
*[self._execute_tool_call(call, cancellation_token) for call in model_result.content]
)
+ # Collect the execution results in a list.
+ exec_results = [result for _, result in executed_calls_and_results]
+ # Add the execution results to output and model context.
tool_call_result_msg = ToolCallExecutionEvent(content=exec_results, source=self.name)
event_logger.debug(tool_call_result_msg)
await self._model_context.add_message(FunctionExecutionResultMessage(content=exec_results))
inner_messages.append(tool_call_result_msg)
yield tool_call_result_msg
- # Correlate tool call results with tool calls.
- tool_calls = [call for call in model_result.content if call.name not in self._handoffs]
+ # Separate out tool calls and tool call results from handoff requests.
+ tool_calls: List[FunctionCall] = []
tool_call_results: List[FunctionExecutionResult] = []
- for tool_call in tool_calls:
- found = False
- for exec_result in exec_results:
- if exec_result.call_id == tool_call.id:
- found = True
- tool_call_results.append(exec_result)
- break
- if not found:
- raise RuntimeError(f"Tool call result not found for call id: {tool_call.id}")
+ for exec_call, exec_result in executed_calls_and_results:
+ if exec_call.name not in self._handoffs:
+ tool_calls.append(exec_call)
+ tool_call_results.append(exec_result)
# Detect handoff requests.
handoff_reqs = [call for call in model_result.content if call.name in self._handoffs]
@@ -546,7 +545,7 @@ class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]):
async def _execute_tool_call(
self, tool_call: FunctionCall, cancellation_token: CancellationToken
- ) -> FunctionExecutionResult:
+ ) -> Tuple[FunctionCall, FunctionExecutionResult]:
"""Execute a tool call and return the result."""
try:
if not self._tools + self._handoff_tools:
@@ -557,9 +556,9 @@ class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]):
arguments = json.loads(tool_call.arguments)
result = await tool.run_json(arguments, cancellation_token)
result_as_str = tool.return_value_as_string(result)
- return FunctionExecutionResult(content=result_as_str, call_id=tool_call.id, is_error=False)
+ return (tool_call, FunctionExecutionResult(content=result_as_str, call_id=tool_call.id, is_error=False))
except Exception as e:
- return FunctionExecutionResult(content=f"Error: {e}", call_id=tool_call.id, is_error=True)
+ return (tool_call, FunctionExecutionResult(content=f"Error: {e}", call_id=tool_call.id, is_error=True))
async def on_reset(self, cancellation_token: CancellationToken) -> None:
"""Reset the assistant agent to its initialization state."""
diff --git a/python/packages/autogen-agentchat/tests/test_assistant_agent.py b/python/packages/autogen-agentchat/tests/test_assistant_agent.py
index 2d82158ef..827a54766 100644
--- a/python/packages/autogen-agentchat/tests/test_assistant_agent.py
+++ b/python/packages/autogen-agentchat/tests/test_assistant_agent.py
@@ -427,6 +427,142 @@ async def test_run_with_parallel_tools(monkeypatch: pytest.MonkeyPatch) -> None:
assert state == state2
+async def test_run_with_parallel_tools_with_empty_call_ids(monkeypatch: pytest.MonkeyPatch) -> None:
+ model = "gpt-4o-2024-05-13"
+ chat_completions = [
+ ChatCompletion(
+ id="id1",
+ choices=[
+ Choice(
+ finish_reason="tool_calls",
+ index=0,
+ message=ChatCompletionMessage(
+ content=None,
+ tool_calls=[
+ ChatCompletionMessageToolCall(
+ id="",
+ type="function",
+ function=Function(
+ name="_pass_function",
+ arguments=json.dumps({"input": "task1"}),
+ ),
+ ),
+ ChatCompletionMessageToolCall(
+ id="",
+ type="function",
+ function=Function(
+ name="_pass_function",
+ arguments=json.dumps({"input": "task2"}),
+ ),
+ ),
+ ChatCompletionMessageToolCall(
+ id="",
+ type="function",
+ function=Function(
+ name="_echo_function",
+ arguments=json.dumps({"input": "task3"}),
+ ),
+ ),
+ ],
+ role="assistant",
+ ),
+ )
+ ],
+ created=0,
+ model=model,
+ object="chat.completion",
+ usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=0),
+ ),
+ ChatCompletion(
+ id="id2",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ message=ChatCompletionMessage(content="pass", role="assistant"),
+ )
+ ],
+ created=0,
+ model=model,
+ object="chat.completion",
+ usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=0),
+ ),
+ ChatCompletion(
+ id="id2",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ message=ChatCompletionMessage(content="TERMINATE", role="assistant"),
+ )
+ ],
+ created=0,
+ model=model,
+ object="chat.completion",
+ usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=0),
+ ),
+ ]
+ mock = _MockChatCompletion(chat_completions)
+ monkeypatch.setattr(AsyncCompletions, "create", mock.mock_create)
+ agent = AssistantAgent(
+ "tool_use_agent",
+ model_client=OpenAIChatCompletionClient(model=model, api_key=""),
+ tools=[
+ _pass_function,
+ _fail_function,
+ FunctionTool(_echo_function, description="Echo"),
+ ],
+ )
+ result = await agent.run(task="task")
+
+ assert len(result.messages) == 4
+ assert isinstance(result.messages[0], TextMessage)
+ assert result.messages[0].models_usage is None
+ assert isinstance(result.messages[1], ToolCallRequestEvent)
+ assert result.messages[1].content == [
+ FunctionCall(id="", arguments=r'{"input": "task1"}', name="_pass_function"),
+ FunctionCall(id="", arguments=r'{"input": "task2"}', name="_pass_function"),
+ FunctionCall(id="", arguments=r'{"input": "task3"}', name="_echo_function"),
+ ]
+ assert result.messages[1].models_usage is not None
+ assert result.messages[1].models_usage.completion_tokens == 5
+ assert result.messages[1].models_usage.prompt_tokens == 10
+ assert isinstance(result.messages[2], ToolCallExecutionEvent)
+ expected_content = [
+ FunctionExecutionResult(call_id="", content="pass", is_error=False),
+ FunctionExecutionResult(call_id="", content="pass", is_error=False),
+ FunctionExecutionResult(call_id="", content="task3", is_error=False),
+ ]
+ for expected in expected_content:
+ assert expected in result.messages[2].content
+ assert result.messages[2].models_usage is None
+ assert isinstance(result.messages[3], ToolCallSummaryMessage)
+ assert result.messages[3].content == "pass\npass\ntask3"
+ assert result.messages[3].models_usage is None
+
+ # Test streaming.
+ mock.curr_index = 0 # Reset the mock
+ index = 0
+ async for message in agent.run_stream(task="task"):
+ if isinstance(message, TaskResult):
+ assert message == result
+ else:
+ assert message == result.messages[index]
+ index += 1
+
+ # Test state saving and loading.
+ state = await agent.save_state()
+ agent2 = AssistantAgent(
+ "tool_use_agent",
+ model_client=OpenAIChatCompletionClient(model=model, api_key=""),
+ tools=[_pass_function, _fail_function, FunctionTool(_echo_function, description="Echo")],
+ )
+ await agent2.load_state(state)
+ state2 = await agent2.save_state()
+ assert state == state2
+
+
@pytest.mark.asyncio
async def test_handoffs(monkeypatch: pytest.MonkeyPatch) -> None:
handoff = Handoff(target="agent2")
diff --git a/python/packages/autogen-core/src/autogen_core/models/_types.py b/python/packages/autogen-core/src/autogen_core/models/_types.py
index 22b1cff3b..76d87ab8d 100644
--- a/python/packages/autogen-core/src/autogen_core/models/_types.py
+++ b/python/packages/autogen-core/src/autogen_core/models/_types.py
@@ -21,6 +21,8 @@ class SystemMessage(BaseModel):
"""
content: str
+ """The content of the message."""
+
type: Literal["SystemMessage"] = "SystemMessage"
@@ -28,9 +30,10 @@ class UserMessage(BaseModel):
"""User message contains input from end users, or a catch-all for data provided to the model."""
content: Union[str, List[Union[str, Image]]]
+ """The content of the message."""
- # Name of the agent that sent this message
source: str
+ """The name of the agent that sent this message."""
type: Literal["UserMessage"] = "UserMessage"
@@ -39,9 +42,10 @@ class AssistantMessage(BaseModel):
"""Assistant message are sampled from the language model."""
content: Union[str, List[FunctionCall]]
+ """The content of the message."""
- # Name of the agent that sent this message
source: str
+ """The name of the agent that sent this message."""
type: Literal["AssistantMessage"] = "AssistantMessage"
@@ -50,8 +54,13 @@ class FunctionExecutionResult(BaseModel):
"""Function execution result contains the output of a function call."""
content: str
+ """The output of the function call."""
+
call_id: str
+ """The ID of the function call. Note this ID may be empty for some models."""
+
is_error: bool | None = None
+ """Whether the function call resulted in an error."""
class FunctionExecutionResultMessage(BaseModel):