Skip to content

Build and run agents you can see, understand and trust.

fix(formatter): put parallel Gemini function responses in one user turn (#2631)

Max Freedom Pollard35cd00c752fc
diff --git a/src/agentscope/formatter/_gemini_formatter.py b/src/agentscope/formatter/_gemini_formatter.py
index c69e1c9d0..94df778dc 100644
--- a/src/agentscope/formatter/_gemini_formatter.py
+++ b/src/agentscope/formatter/_gemini_formatter.py
@@ -129,6 +129,30 @@ class GeminiChatFormatter(_GeminiFormatterBase):
),
)
+ @staticmethod
+ def _flush_tool_results(
+ messages: list[dict],
+ tool_result_parts: list,
+ promoted_parts: list,
+ ) -> None:
+ """Append the gathered tool results as one `user` content, followed
+ by one content holding the media promoted out of them.
+
+ Args:
+ messages (`list[dict]`):
+ The formatted messages, appended to in place.
+ tool_result_parts (`list`):
+ The `function_response` parts gathered so far.
+ promoted_parts (`list`):
+ The media parts promoted out of those tool results.
+ """
+ if not tool_result_parts:
+ return
+
+ messages.append({"role": "user", "parts": tool_result_parts})
+ if promoted_parts:
+ messages.append({"role": "user", "parts": promoted_parts})
+
async def format(
self,
msgs: list[Msg],
@@ -150,8 +174,24 @@ class GeminiChatFormatter(_GeminiFormatterBase):
while i < len(msgs):
msg = msgs[i]
parts: list = []
+ # Group consecutive tool results into one user turn, followed by
+ # any media promoted from those results.
+ tool_result_parts: list = []
+ promoted_parts: list = []
for block in msg.get_content_blocks():
+ if tool_result_parts and not isinstance(
+ block,
+ ToolResultBlock,
+ ):
+ self._flush_tool_results(
+ messages,
+ tool_result_parts,
+ promoted_parts,
+ )
+ tool_result_parts = []
+ promoted_parts = []
+
if isinstance(block, TextBlock):
if block.text:
parts.append({"text": block.text})
@@ -231,38 +271,28 @@ class GeminiChatFormatter(_GeminiFormatterBase):
multimodal_data,
) = self.convert_tool_result_to_string(block.output)
- messages.append(
+ tool_result_parts.append(
{
- "role": "user",
- "parts": [
- {
- "function_response": {
- "id": block.id,
- "name": block.name,
- "response": {
- "output": textual_output,
- },
- },
+ "function_response": {
+ "id": block.id,
+ "name": block.name,
+ "response": {
+ "output": textual_output,
},
- ],
+ },
},
)
if multimodal_data:
- promo_parts = []
for item in multimodal_data:
if isinstance(item, TextBlock):
- promo_parts.append({"text": item.text})
+ promoted_parts.append({"text": item.text})
elif isinstance(item, DataBlock):
fmt_item = self._format_gemini_data_block(
item,
)
if fmt_item is not None:
- promo_parts.append(fmt_item)
- if promo_parts:
- messages.append(
- {"role": "user", "parts": promo_parts},
- )
+ promoted_parts.append(fmt_item)
else:
logger.warning(
@@ -281,6 +311,12 @@ class GeminiChatFormatter(_GeminiFormatterBase):
},
)
+ self._flush_tool_results(
+ messages,
+ tool_result_parts,
+ promoted_parts,
+ )
+
i += 1
return messages
diff --git a/tests/formatter_gemini_test.py b/tests/formatter_gemini_test.py
index 7c4e1d890..ba2c35f6e 100644
--- a/tests/formatter_gemini_test.py
+++ b/tests/formatter_gemini_test.py
@@ -760,11 +760,6 @@ class TestGeminiFormatter(IsolatedAsyncioTestCase):
"response": {"output": "result_1"},
},
},
- ],
- },
- {
- "role": "user",
- "parts": [
{
"function_response": {
"id": "call_2",
@@ -835,6 +830,212 @@ class TestGeminiFormatter(IsolatedAsyncioTestCase):
res,
)
+ async def test_parallel_tool_results_share_one_user_turn(self) -> None:
+ """Parallel function responses go into a single user content, so
+ their count matches the function calls of the model turn."""
+ fmt = GeminiChatFormatter()
+ res = await fmt.format(
+ [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(
+ id="call_1",
+ name="get_weather",
+ input='{"city": "Beijing"}',
+ ),
+ ToolCallBlock(
+ id="call_2",
+ name="get_weather",
+ input='{"city": "Shanghai"}',
+ ),
+ ToolResultBlock(
+ id="call_1",
+ name="get_weather",
+ output=[TextBlock(text="Beijing: sunny")],
+ state=ToolResultState.SUCCESS,
+ ),
+ ToolResultBlock(
+ id="call_2",
+ name="get_weather",
+ output=[TextBlock(text="Shanghai: rainy")],
+ state=ToolResultState.SUCCESS,
+ ),
+ ],
+ ),
+ ],
+ )
+ self.assertListEqual(
+ [
+ {
+ "role": "model",
+ "parts": [
+ {
+ "function_call": {
+ "id": "call_1",
+ "name": "get_weather",
+ "args": {"city": "Beijing"},
+ },
+ },
+ {
+ "function_call": {
+ "id": "call_2",
+ "name": "get_weather",
+ "args": {"city": "Shanghai"},
+ },
+ },
+ ],
+ },
+ {
+ "role": "user",
+ "parts": [
+ {
+ "function_response": {
+ "id": "call_1",
+ "name": "get_weather",
+ "response": {"output": "Beijing: sunny"},
+ },
+ },
+ {
+ "function_response": {
+ "id": "call_2",
+ "name": "get_weather",
+ "response": {"output": "Shanghai: rainy"},
+ },
+ },
+ ],
+ },
+ ],
+ res,
+ )
+
+ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(
+ self,
+ ) -> None:
+ """Media promoted from a tool result must not split the function
+ responses that answer one model turn's function calls."""
+ fmt = GeminiChatFormatter()
+ msgs = [
+ AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(
+ id="call_shot",
+ name="screenshot",
+ input="{}",
+ ),
+ ToolCallBlock(
+ id="call_title",
+ name="get_title",
+ input="{}",
+ ),
+ ToolResultBlock(
+ id="call_shot",
+ name="screenshot",
+ output=[
+ TextBlock(text="Screenshot taken."),
+ DataBlock(
+ id=_FIXED_ID,
+ source=Base64Source(
+ data=self.image_b64,
+ media_type="image/png",
+ ),
+ ),
+ ],
+ state=ToolResultState.SUCCESS,
+ ),
+ ToolResultBlock(
+ id="call_title",
+ name="get_title",
+ output=[TextBlock(text="Example Domain")],
+ state=ToolResultState.SUCCESS,
+ ),
+ TextBlock(text="The page is Example Domain."),
+ ],
+ ),
+ ]
+
+ res = await fmt.format(msgs)
+
+ shot_output = (
+ "Screenshot taken.\n"
+ "<system-reminder>A(n) image file is returned and will be "
+ "presented to you with the identifier "
+ f"[{_FIXED_ID}].</system-reminder>"
+ )
+
+ self.assertListEqual(
+ [
+ {
+ "role": "model",
+ "parts": [
+ {
+ "function_call": {
+ "id": "call_shot",
+ "name": "screenshot",
+ "args": {},
+ },
+ },
+ {
+ "function_call": {
+ "id": "call_title",
+ "name": "get_title",
+ "args": {},
+ },
+ },
+ ],
+ },
+ {
+ "role": "user",
+ "parts": [
+ {
+ "function_response": {
+ "id": "call_shot",
+ "name": "screenshot",
+ "response": {"output": shot_output},
+ },
+ },
+ {
+ "function_response": {
+ "id": "call_title",
+ "name": "get_title",
+ "response": {"output": "Example Domain"},
+ },
+ },
+ ],
+ },
+ {
+ "role": "user",
+ "parts": [
+ {
+ "text": (
+ "<system-reminder>The multimodal data "
+ "and their identifiers are listed as "
+ "follows:"
+ ),
+ },
+ {
+ "text": f"- {_FIXED_ID} (image file): ",
+ },
+ {
+ "inline_data": {
+ "data": self.image_b64,
+ "mime_type": "image/png",
+ },
+ },
+ {"text": "</system-reminder>"},
+ ],
+ },
+ {
+ "role": "model",
+ "parts": [
+ {"text": "The page is Example Domain."},
+ ],
+ },
+ ],
+ res,
+ )
+
async def test_chat_formatter_hint_block(self) -> None:
"""HintBlock flushes preceding content and becomes a user message."""
fmt = GeminiChatFormatter()