Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(formatter): put parallel Gemini function responses in one user turn (#2631)
Max Freedom Pollard35cd00c752fc
diff --git a/src/agentscope/formatter/_gemini_formatter.py b/src/agentscope/formatter/_gemini_formatter.pyindex c69e1c9d0..94df778dc 100644--- a/src/agentscope/formatter/_gemini_formatter.py+++ b/src/agentscope/formatter/_gemini_formatter.py@@ -129,6 +129,30 @@ class GeminiChatFormatter(_GeminiFormatterBase):),)+ @staticmethod+ def _flush_tool_results(+ messages: list[dict],+ tool_result_parts: list,+ promoted_parts: list,+ ) -> None:+ """Append the gathered tool results as one `user` content, followed+ by one content holding the media promoted out of them.++ Args:+ messages (`list[dict]`):+ The formatted messages, appended to in place.+ tool_result_parts (`list`):+ The `function_response` parts gathered so far.+ promoted_parts (`list`):+ The media parts promoted out of those tool results.+ """+ if not tool_result_parts:+ return++ messages.append({"role": "user", "parts": tool_result_parts})+ if promoted_parts:+ messages.append({"role": "user", "parts": promoted_parts})+async def format(self,msgs: list[Msg],@@ -150,8 +174,24 @@ class GeminiChatFormatter(_GeminiFormatterBase):while i < len(msgs):msg = msgs[i]parts: list = []+ # Group consecutive tool results into one user turn, followed by+ # any media promoted from those results.+ tool_result_parts: list = []+ promoted_parts: list = []for block in msg.get_content_blocks():+ if tool_result_parts and not isinstance(+ block,+ ToolResultBlock,+ ):+ self._flush_tool_results(+ messages,+ tool_result_parts,+ promoted_parts,+ )+ tool_result_parts = []+ promoted_parts = []+if isinstance(block, TextBlock):if block.text:parts.append({"text": block.text})@@ -231,38 +271,28 @@ class GeminiChatFormatter(_GeminiFormatterBase):multimodal_data,) = self.convert_tool_result_to_string(block.output)- messages.append(+ tool_result_parts.append({- "role": "user",- "parts": [- {- "function_response": {- "id": block.id,- "name": block.name,- "response": {- "output": textual_output,- },- },+ "function_response": {+ "id": block.id,+ "name": block.name,+ "response": {+ "output": textual_output,},- ],+ },},)if multimodal_data:- promo_parts = []for item in multimodal_data:if isinstance(item, TextBlock):- promo_parts.append({"text": item.text})+ promoted_parts.append({"text": item.text})elif isinstance(item, DataBlock):fmt_item = self._format_gemini_data_block(item,)if fmt_item is not None:- promo_parts.append(fmt_item)- if promo_parts:- messages.append(- {"role": "user", "parts": promo_parts},- )+ promoted_parts.append(fmt_item)else:logger.warning(@@ -281,6 +311,12 @@ class GeminiChatFormatter(_GeminiFormatterBase):},)+ self._flush_tool_results(+ messages,+ tool_result_parts,+ promoted_parts,+ )+i += 1return messagesdiff --git a/tests/formatter_gemini_test.py b/tests/formatter_gemini_test.pyindex 7c4e1d890..ba2c35f6e 100644--- a/tests/formatter_gemini_test.py+++ b/tests/formatter_gemini_test.py@@ -760,11 +760,6 @@ class TestGeminiFormatter(IsolatedAsyncioTestCase):"response": {"output": "result_1"},},},- ],- },- {- "role": "user",- "parts": [{"function_response": {"id": "call_2",@@ -835,6 +830,212 @@ class TestGeminiFormatter(IsolatedAsyncioTestCase):res,)+ async def test_parallel_tool_results_share_one_user_turn(self) -> None:+ """Parallel function responses go into a single user content, so+ their count matches the function calls of the model turn."""+ fmt = GeminiChatFormatter()+ res = await fmt.format(+ [+ AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(+ id="call_1",+ name="get_weather",+ input='{"city": "Beijing"}',+ ),+ ToolCallBlock(+ id="call_2",+ name="get_weather",+ input='{"city": "Shanghai"}',+ ),+ ToolResultBlock(+ id="call_1",+ name="get_weather",+ output=[TextBlock(text="Beijing: sunny")],+ state=ToolResultState.SUCCESS,+ ),+ ToolResultBlock(+ id="call_2",+ name="get_weather",+ output=[TextBlock(text="Shanghai: rainy")],+ state=ToolResultState.SUCCESS,+ ),+ ],+ ),+ ],+ )+ self.assertListEqual(+ [+ {+ "role": "model",+ "parts": [+ {+ "function_call": {+ "id": "call_1",+ "name": "get_weather",+ "args": {"city": "Beijing"},+ },+ },+ {+ "function_call": {+ "id": "call_2",+ "name": "get_weather",+ "args": {"city": "Shanghai"},+ },+ },+ ],+ },+ {+ "role": "user",+ "parts": [+ {+ "function_response": {+ "id": "call_1",+ "name": "get_weather",+ "response": {"output": "Beijing: sunny"},+ },+ },+ {+ "function_response": {+ "id": "call_2",+ "name": "get_weather",+ "response": {"output": "Shanghai: rainy"},+ },+ },+ ],+ },+ ],+ res,+ )++ async def test_chat_formatter_parallel_tool_media_after_tool_msgs(+ self,+ ) -> None:+ """Media promoted from a tool result must not split the function+ responses that answer one model turn's function calls."""+ fmt = GeminiChatFormatter()+ msgs = [+ AssistantMsg(+ name="assistant",+ content=[+ ToolCallBlock(+ id="call_shot",+ name="screenshot",+ input="{}",+ ),+ ToolCallBlock(+ id="call_title",+ name="get_title",+ input="{}",+ ),+ ToolResultBlock(+ id="call_shot",+ name="screenshot",+ output=[+ TextBlock(text="Screenshot taken."),+ DataBlock(+ id=_FIXED_ID,+ source=Base64Source(+ data=self.image_b64,+ media_type="image/png",+ ),+ ),+ ],+ state=ToolResultState.SUCCESS,+ ),+ ToolResultBlock(+ id="call_title",+ name="get_title",+ output=[TextBlock(text="Example Domain")],+ state=ToolResultState.SUCCESS,+ ),+ TextBlock(text="The page is Example Domain."),+ ],+ ),+ ]++ res = await fmt.format(msgs)++ shot_output = (+ "Screenshot taken.\n"+ "<system-reminder>A(n) image file is returned and will be "+ "presented to you with the identifier "+ f"[{_FIXED_ID}].</system-reminder>"+ )++ self.assertListEqual(+ [+ {+ "role": "model",+ "parts": [+ {+ "function_call": {+ "id": "call_shot",+ "name": "screenshot",+ "args": {},+ },+ },+ {+ "function_call": {+ "id": "call_title",+ "name": "get_title",+ "args": {},+ },+ },+ ],+ },+ {+ "role": "user",+ "parts": [+ {+ "function_response": {+ "id": "call_shot",+ "name": "screenshot",+ "response": {"output": shot_output},+ },+ },+ {+ "function_response": {+ "id": "call_title",+ "name": "get_title",+ "response": {"output": "Example Domain"},+ },+ },+ ],+ },+ {+ "role": "user",+ "parts": [+ {+ "text": (+ "<system-reminder>The multimodal data "+ "and their identifiers are listed as "+ "follows:"+ ),+ },+ {+ "text": f"- {_FIXED_ID} (image file): ",+ },+ {+ "inline_data": {+ "data": self.image_b64,+ "mime_type": "image/png",+ },+ },+ {"text": "</system-reminder>"},+ ],+ },+ {+ "role": "model",+ "parts": [+ {"text": "The page is Example Domain."},+ ],+ },+ ],+ res,+ )+async def test_chat_formatter_hint_block(self) -> None:"""HintBlock flushes preceding content and becomes a user message."""fmt = GeminiChatFormatter()