Skip to content

Build and run agents you can see, understand and trust.

fix(formatter): preserve image-only messages and history prompts in Ollama multi-agent formatter (#3790)

hdu_zhou2f6fa6a12e4b
diff --git a/src/agentscope/formatter/_ollama_formatter.py b/src/agentscope/formatter/_ollama_formatter.py
index 403d12ef0..6827d59fd 100644
--- a/src/agentscope/formatter/_ollama_formatter.py
+++ b/src/agentscope/formatter/_ollama_formatter.py
@@ -383,17 +383,22 @@ class OllamaMultiAgentFormatter(_OllamaFormatterBase):
await self._format_tool_sequence(group),
)
case "agent_message":
- (
- formatted_group,
- has_history,
- ) = await self._format_agent_message_with_hints(
- group,
- is_first_agent_message,
- self._format_agent_message,
- )
- formatted_msgs.extend(formatted_group)
- if has_history:
- is_first_agent_message = False
+ segments = self._split_agent_messages_at_hints(group)
+ for is_hint, segment in segments:
+ if is_hint:
+ formatted_msgs.extend(
+ await self._format_hint_message(segment),
+ )
+ continue
+
+ formatted_group = await self._format_agent_message(
+ segment,
+ is_first_agent_message,
+ )
+ formatted_msgs.extend(formatted_group)
+ # Image-only history does not emit the prompt.
+ if any(msg["content"] for msg in formatted_group):
+ is_first_agent_message = False
return formatted_msgs
@@ -494,7 +499,7 @@ class OllamaMultiAgentFormatter(_OllamaFormatterBase):
user_message["images"] = images
formatted_msgs = []
- if conversation_blocks:
+ if conversation_blocks or images:
formatted_msgs.append(user_message)
return formatted_msgs
diff --git a/tests/formatter_ollama_test.py b/tests/formatter_ollama_test.py
index 90629be48..4e070fd82 100644
--- a/tests/formatter_ollama_test.py
+++ b/tests/formatter_ollama_test.py
@@ -9,6 +9,7 @@ from unittest import IsolatedAsyncioTestCase
from agentscope.formatter import OllamaChatFormatter, OllamaMultiAgentFormatter
from agentscope.message import (
+ Msg,
UserMsg,
AssistantMsg,
SystemMsg,
@@ -623,6 +624,131 @@ class TestOllamaFormatter(IsolatedAsyncioTestCase):
# Empty
self.assertListEqual([], await fmt.format([]))
+ async def test_multiagent_formatter_image_only_message_kept(self) -> None:
+ """An agent message carrying only an image (no text) is preserved.
+
+ Regression test: the image-only turn used to be dropped because the
+ guard only considered text blocks.
+ """
+ fmt = OllamaMultiAgentFormatter()
+ msgs = [
+ Msg(
+ name="",
+ role="user",
+ content=[
+ DataBlock(
+ source=Base64Source(
+ data=self.image_b64,
+ media_type="image/png",
+ ),
+ ),
+ ],
+ ),
+ ]
+ res = await fmt.format(msgs)
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": "",
+ "images": [self.image_b64],
+ },
+ ],
+ res,
+ )
+
+ async def test_multiagent_image_history_prompt(self) -> None:
+ """Emit the history prompt once, on the first textual history."""
+ fmt = OllamaMultiAgentFormatter(
+ conversation_history_prompt="Custom history instructions\n",
+ )
+ hint = AssistantMsg(
+ name="assistant",
+ content=[HintBlock(hint="Continue")],
+ )
+ hint_output = [{"role": "user", "content": "Continue"}]
+ tool = AssistantMsg(
+ name="assistant",
+ content=[
+ ToolCallBlock(id="c1", name="inspect", input="{}"),
+ ToolResultBlock(
+ id="c1",
+ name="inspect",
+ output="Done",
+ state=ToolResultState.SUCCESS,
+ ),
+ ],
+ )
+ tool_output: list[dict] = [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"function": {"name": "inspect", "arguments": {}}},
+ ],
+ },
+ {"role": "tool", "tool_name": "inspect", "content": "Done"},
+ ]
+ separators: list[tuple[Msg, list[dict]]] = [
+ (hint, hint_output),
+ (tool, tool_output),
+ ]
+ for name in ("", "viewer"):
+ for separator, expected_separator in separators:
+ with self.subTest(name=name, separator=separator.content):
+ msgs = [
+ UserMsg(
+ name=name,
+ content=[
+ DataBlock(
+ source=Base64Source(
+ data=self.image_b64,
+ media_type="image/png",
+ ),
+ ),
+ ],
+ ),
+ separator,
+ UserMsg(name="user", content="Describe the image"),
+ hint,
+ UserMsg(name="user", content="More details"),
+ ]
+ first_content = (
+ f"{fmt.conversation_history_prompt}"
+ f"<history>\n{name}:\n</history>"
+ if name
+ else ""
+ )
+ later_prompt = (
+ "" if name else fmt.conversation_history_prompt
+ )
+ self.assertListEqual(
+ [
+ {
+ "role": "user",
+ "content": first_content,
+ "images": [self.image_b64],
+ },
+ *expected_separator,
+ {
+ "role": "user",
+ "content": (
+ f"{later_prompt}<history>\n"
+ "user:\nDescribe the image\n</history>"
+ ),
+ },
+ *hint_output,
+ {
+ "role": "user",
+ "content": (
+ "<history>\nuser:\n"
+ "More details\n</history>"
+ ),
+ },
+ ],
+ await fmt.format(msgs),
+ )
+
async def test_chat_formatter_complex_multi_step(self) -> None:
"""Complex multi-step sequence with interleaved thinking, text,
tool calls, and tool results."""