Skip to content

A programming framework for agentic AI

FEAT: adding multiple_system_message on model_info (#6327)

EeSb13264ac60ef
diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_society_of_mind_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_society_of_mind_agent.py
index fb88ac29c..5bcf83651 100644
--- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_society_of_mind_agent.py
+++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_society_of_mind_agent.py
@@ -5,7 +5,7 @@ from autogen_core.model_context import (
ChatCompletionContext,
UnboundedChatCompletionContext,
)
-from autogen_core.models import ChatCompletionClient, LLMMessage, SystemMessage
+from autogen_core.models import ChatCompletionClient, LLMMessage, SystemMessage, UserMessage
from pydantic import BaseModel
from typing_extensions import Self
@@ -214,12 +214,26 @@ class SocietyOfMindAgent(BaseChatAgent, Component[SocietyOfMindAgentConfig]):
# Response's inner_messages should be empty. Cause that mean is response to outer world.
)
else:
+ llm_messages: List[LLMMessage] = []
+
+ if self._model_client.model_info.get("multiple_system_messages", False):
+ # The model client supports multiple system messages, so we
+ llm_messages.append(SystemMessage(content=self._instruction))
+ else:
+ # The model client does not support multiple system messages, so we
+ llm_messages.append(UserMessage(content=self._instruction, source="user"))
+
# Generate a response using the model client.
- llm_messages: List[LLMMessage] = [SystemMessage(content=self._instruction)]
for message in inner_messages:
if isinstance(message, BaseChatMessage):
llm_messages.append(message.to_model_message())
- llm_messages.append(SystemMessage(content=self._response_prompt))
+
+ if self._model_client.model_info.get("multiple_system_messages", False):
+ # The model client supports multiple system messages, so we
+ llm_messages.append(SystemMessage(content=self._response_prompt))
+ else:
+ # The model client does not support multiple system messages, so we
+ llm_messages.append(UserMessage(content=self._response_prompt, source="user"))
completion = await self._model_client.create(messages=llm_messages, cancellation_token=cancellation_token)
assert isinstance(completion.content, str)
yield Response(
diff --git a/python/packages/autogen-agentchat/tests/test_society_of_mind_agent.py b/python/packages/autogen-agentchat/tests/test_society_of_mind_agent.py
index c7b25e93f..5bc9a010c 100644
--- a/python/packages/autogen-agentchat/tests/test_society_of_mind_agent.py
+++ b/python/packages/autogen-agentchat/tests/test_society_of_mind_agent.py
@@ -1,4 +1,5 @@
-from typing import AsyncGenerator
+from types import MethodType
+from typing import Any, AsyncGenerator, Sequence
import pytest
import pytest_asyncio
@@ -6,6 +7,7 @@ from autogen_agentchat.agents import AssistantAgent, SocietyOfMindAgent
from autogen_agentchat.conditions import MaxMessageTermination
from autogen_agentchat.teams import RoundRobinGroupChat
from autogen_core import AgentRuntime, SingleThreadedAgentRuntime
+from autogen_core.models import CreateResult, LLMMessage, SystemMessage
from autogen_ext.models.replay import ReplayChatCompletionClient
@@ -113,3 +115,83 @@ async def test_society_of_mind_agent_multiple_rounds(runtime: AgentRuntime | Non
response = await society_of_mind_agent.run()
assert len(response.messages) == 1
assert response.messages[0].source == "society_of_mind"
+
+
+async def test_society_of_mind_agent_no_multiple_system_messages(
+ monkeypatch: pytest.MonkeyPatch, runtime: AgentRuntime | None
+) -> None:
+ model_client = ReplayChatCompletionClient(["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"])
+
+ model_client_soma = ReplayChatCompletionClient(
+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"],
+ model_info={
+ "vision": False,
+ "function_calling": False,
+ "json_output": False,
+ "family": "unknown",
+ "structured_output": False,
+ "multiple_system_messages": False,
+ },
+ )
+
+ original_create = model_client_soma.create
+
+ # mock method with bound self
+ async def _mock_create(
+ self: ReplayChatCompletionClient, messages: Sequence[LLMMessage], *args: Any, **kwargs: Any
+ ) -> CreateResult:
+ for message in messages:
+ assert not isinstance(message, SystemMessage)
+ kwargs["messages"] = messages
+ return await original_create(*args, **kwargs)
+
+ # bind it
+ monkeypatch.setattr(model_client_soma, "create", MethodType(_mock_create, model_client_soma))
+
+ agent1 = AssistantAgent("assistant1", model_client=model_client, system_message="You are a helpful assistant.")
+ agent2 = AssistantAgent("assistant2", model_client=model_client, system_message="You are a helpful assistant.")
+ inner_termination = MaxMessageTermination(3)
+ inner_team = RoundRobinGroupChat([agent1, agent2], termination_condition=inner_termination, runtime=runtime)
+ society_of_mind_agent = SocietyOfMindAgent("society_of_mind", team=inner_team, model_client=model_client_soma)
+ await society_of_mind_agent.run(task="Count to 10.")
+
+
+async def test_society_of_mind_agent_yes_multiple_system_messages(
+ monkeypatch: pytest.MonkeyPatch, runtime: AgentRuntime | None
+) -> None:
+ model_client = ReplayChatCompletionClient(["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"])
+
+ model_client_soma = ReplayChatCompletionClient(
+ ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"],
+ model_info={
+ "vision": False,
+ "function_calling": False,
+ "json_output": False,
+ "family": "unknown",
+ "structured_output": False,
+ "multiple_system_messages": True,
+ },
+ )
+
+ original_create = model_client_soma.create
+
+ # mock method with bound self
+ async def _mock_create(
+ self: ReplayChatCompletionClient, messages: Sequence[LLMMessage], *args: Any, **kwargs: Any
+ ) -> CreateResult:
+ assert isinstance(messages[0], SystemMessage)
+ assert isinstance(messages[-1], SystemMessage)
+ kwargs["messages"] = messages
+ return await original_create(*args, **kwargs)
+
+ # bind it
+ monkeypatch.setattr(model_client_soma, "create", MethodType(_mock_create, model_client_soma))
+
+ agent1 = AssistantAgent("assistant1", model_client=model_client, system_message="You are a helpful assistant.")
+ agent2 = AssistantAgent("assistant2", model_client=model_client, system_message="You are a helpful assistant.")
+ inner_termination = MaxMessageTermination(3)
+ inner_team = RoundRobinGroupChat([agent1, agent2], termination_condition=inner_termination, runtime=runtime)
+ society_of_mind_agent = SocietyOfMindAgent("society_of_mind", team=inner_team, model_client=model_client_soma)
+ await society_of_mind_agent.run(task="Count to 10.")
diff --git a/python/packages/autogen-core/src/autogen_core/models/_model_client.py b/python/packages/autogen-core/src/autogen_core/models/_model_client.py
index 9250625cd..b65a19853 100644
--- a/python/packages/autogen-core/src/autogen_core/models/_model_client.py
+++ b/python/packages/autogen-core/src/autogen_core/models/_model_client.py
@@ -123,6 +123,8 @@ class ModelInfo(TypedDict, total=False):
"""Model family should be one of the constants from :py:class:`ModelFamily` or a string representing an unknown model family."""
structured_output: Required[bool]
"""True if the model supports structured output, otherwise False. This is different to json_output."""
+ multiple_system_messages: Optional[bool]
+ """True if the model supports multiple, non-consecutive system messages, otherwise False."""
def validate_model_info(model_info: ModelInfo) -> None:
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_model_info.py b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_model_info.py
index 059af5097..27b6dd68a 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_model_info.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/anthropic/_model_info.py
@@ -13,6 +13,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_7_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude 3.7 Sonnet latest alias
"claude-3-7-sonnet-latest": {
@@ -21,6 +22,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_7_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude 3 Opus (most powerful)
"claude-3-opus-20240229": {
@@ -29,6 +31,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude 3 Sonnet (balanced)
"claude-3-sonnet-20240229": {
@@ -37,6 +40,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude 3 Haiku (fastest)
"claude-3-haiku-20240307": {
@@ -45,6 +49,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude 3.5 Sonnet
"claude-3-5-sonnet-20240620": {
@@ -53,6 +58,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude Instant v1 (legacy)
"claude-instant-1.2": {
@@ -61,6 +67,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude 2 (legacy)
"claude-2.0": {
@@ -69,6 +76,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
# Claude 2.1 (legacy)
"claude-2.1": {
@@ -77,6 +85,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": False,
},
}
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py
index 600fac41a..47b3fedf7 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_model_info.py
@@ -45,6 +45,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.O4,
"structured_output": True,
+ "multiple_system_messages": True,
},
"o3-2025-04-16": {
"vision": True,
@@ -52,6 +53,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.O3,
"structured_output": True,
+ "multiple_system_messages": True,
},
"o3-mini-2025-01-31": {
"vision": False,
@@ -59,6 +61,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.O3,
"structured_output": True,
+ "multiple_system_messages": True,
},
"o1-2024-12-17": {
"vision": False,
@@ -66,6 +69,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False,
"family": ModelFamily.O1,
"structured_output": True,
+ "multiple_system_messages": True,
},
"o1-preview-2024-09-12": {
"vision": False,
@@ -73,6 +77,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False,
"family": ModelFamily.O1,
"structured_output": True,
+ "multiple_system_messages": True,
},
"o1-mini-2024-09-12": {
"vision": False,
@@ -80,6 +85,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False,
"family": ModelFamily.O1,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-4.1-2025-04-14": {
"vision": True,
@@ -87,6 +93,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_41,
"structured_output": True,
+ "multiple_system_messages": True,
},
"gpt-4.1-mini-2025-04-14": {
"vision": True,
@@ -94,6 +101,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_41,
"structured_output": True,
+ "multiple_system_messages": True,
},
"gpt-4.1-nano-2025-04-14": {
"vision": True,
@@ -101,6 +109,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_41,
"structured_output": True,
+ "multiple_system_messages": True,
},
"gpt-4.5-preview-2025-02-27": {
"vision": True,
@@ -108,6 +117,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_45,
"structured_output": True,
+ "multiple_system_messages": True,
},
"gpt-4o-2024-11-20": {
"vision": True,
@@ -115,6 +125,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4O,
"structured_output": True,
+ "multiple_system_messages": True,
},
"gpt-4o-2024-08-06": {
"vision": True,
@@ -122,6 +133,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4O,
"structured_output": True,
+ "multiple_system_messages": True,
},
"gpt-4o-2024-05-13": {
"vision": True,
@@ -129,6 +141,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4O,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-4o-mini-2024-07-18": {
"vision": True,
@@ -136,6 +149,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4O,
"structured_output": True,
+ "multiple_system_messages": True,
},
"gpt-4-turbo-2024-04-09": {
"vision": True,
@@ -143,6 +157,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-4-0125-preview": {
"vision": False,
@@ -150,6 +165,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-4-1106-preview": {
"vision": False,
@@ -157,6 +173,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-4-1106-vision-preview": {
"vision": True,
@@ -164,6 +181,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False,
"family": ModelFamily.GPT_4,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-4-0613": {
"vision": False,
@@ -171,6 +189,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-4-32k-0613": {
"vision": False,
@@ -178,6 +197,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_4,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-3.5-turbo-0125": {
"vision": False,
@@ -185,6 +205,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_35,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-3.5-turbo-1106": {
"vision": False,
@@ -192,6 +213,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_35,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-3.5-turbo-instruct": {
"vision": False,
@@ -199,6 +221,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_35,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-3.5-turbo-0613": {
"vision": False,
@@ -206,6 +229,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_35,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gpt-3.5-turbo-16k-0613": {
"vision": False,
@@ -213,6 +237,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GPT_35,
"structured_output": False,
+ "multiple_system_messages": True,
},
"gemini-1.5-flash": {
"vision": True,
@@ -220,6 +245,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GEMINI_1_5_FLASH,
"structured_output": True,
+ "multiple_system_messages": False,
},
"gemini-1.5-flash-8b": {
"vision": True,
@@ -227,6 +253,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GEMINI_1_5_FLASH,
"structured_output": True,
+ "multiple_system_messages": False,
},
"gemini-1.5-pro": {
"vision": True,
@@ -234,6 +261,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GEMINI_1_5_PRO,
"structured_output": True,
+ "multiple_system_messages": False,
},
"gemini-2.0-flash": {
"vision": True,
@@ -241,6 +269,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GEMINI_2_0_FLASH,
"structured_output": True,
+ "multiple_system_messages": False,
},
"gemini-2.0-flash-lite-preview-02-05": {
"vision": True,
@@ -248,6 +277,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GEMINI_2_0_FLASH,
"structured_output": True,
+ "multiple_system_messages": False,
},
"gemini-2.5-pro-preview-03-25": {
"vision": True,
@@ -255,6 +285,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": True,
"family": ModelFamily.GEMINI_2_5_PRO,
"structured_output": True,
+ "multiple_system_messages": False,
},
"claude-3-haiku-20240307": {
"vision": True,
@@ -262,6 +293,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False, # Update this when Anthropic supports structured output
"family": ModelFamily.CLAUDE_3_HAIKU,
"structured_output": False,
+ "multiple_system_messages": True,
},
"claude-3-sonnet-20240229": {
"vision": True,
@@ -269,6 +301,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False, # Update this when Anthropic supports structured output
"family": ModelFamily.CLAUDE_3_SONNET,
"structured_output": False,
+ "multiple_system_messages": True,
},
"claude-3-opus-20240229": {
"vision": True,
@@ -276,6 +309,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False, # Update this when Anthropic supports structured output
"family": ModelFamily.CLAUDE_3_OPUS,
"structured_output": False,
+ "multiple_system_messages": True,
},
"claude-3-5-haiku-20241022": {
"vision": True,
@@ -283,6 +317,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False, # Update this when Anthropic supports structured output
"family": ModelFamily.CLAUDE_3_5_HAIKU,
"structured_output": False,
+ "multiple_system_messages": True,
},
"claude-3-5-sonnet-20241022": {
"vision": True,
@@ -290,6 +325,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False, # Update this when Anthropic supports structured output
"family": ModelFamily.CLAUDE_3_5_SONNET,
"structured_output": False,
+ "multiple_system_messages": True,
},
"claude-3-7-sonnet-20250219": {
"vision": True,
@@ -297,6 +333,7 @@ _MODEL_INFO: Dict[str, ModelInfo] = {
"json_output": False, # Update this when Anthropic supports structured output
"family": ModelFamily.CLAUDE_3_7_SONNET,
"structured_output": False,
+ "multiple_system_messages": True,
},
}