A programming framework for agentic AI
Add text-only model support to M1 (#5344)
afourneycf6fa7727371
diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.pyindex 067efd25e..b328ba308 100644--- a/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py+++ b/python/packages/autogen-agentchat/src/autogen_agentchat/teams/_group_chat/_magentic_one/_magentic_one_orchestrator.py@@ -2,7 +2,7 @@ import jsonimport loggingfrom typing import Any, Dict, List, Mapping-from autogen_core import AgentId, CancellationToken, DefaultTopicId, Image, MessageContext, event, rpc+from autogen_core import AgentId, CancellationToken, DefaultTopicId, MessageContext, event, rpcfrom autogen_core.models import (AssistantMessage,ChatCompletionClient,@@ -24,6 +24,7 @@ from ....messages import (ToolCallSummaryMessage,)from ....state import MagenticOneOrchestratorState+from ....utils import content_to_str, remove_imagesfrom .._base_group_chat_manager import BaseGroupChatManagerfrom .._events import (GroupChatAgentResponse,@@ -138,7 +139,7 @@ class MagenticOneOrchestrator(BaseGroupChatManager):# Create the initial task ledger################################## Combine all message contents for task- self._task = " ".join([self._content_to_str(msg.content) for msg in message.messages])+ self._task = " ".join([content_to_str(msg.content) for msg in message.messages])planning_conversation: List[LLMMessage] = []# 1. GATHER FACTS@@ -146,7 +147,9 @@ class MagenticOneOrchestrator(BaseGroupChatManager):planning_conversation.append(UserMessage(content=self._get_task_ledger_facts_prompt(self._task), source=self._name))- response = await self._model_client.create(planning_conversation, cancellation_token=ctx.cancellation_token)+ response = await self._model_client.create(+ self._get_compatible_context(planning_conversation), cancellation_token=ctx.cancellation_token+ )assert isinstance(response.content, str)self._facts = response.content@@ -157,7 +160,9 @@ class MagenticOneOrchestrator(BaseGroupChatManager):planning_conversation.append(UserMessage(content=self._get_task_ledger_plan_prompt(self._team_description), source=self._name))- response = await self._model_client.create(planning_conversation, cancellation_token=ctx.cancellation_token)+ response = await self._model_client.create(+ self._get_compatible_context(planning_conversation), cancellation_token=ctx.cancellation_token+ )assert isinstance(response.content, str)self._plan = response.content@@ -281,7 +286,7 @@ class MagenticOneOrchestrator(BaseGroupChatManager):assert self._max_json_retries > 0key_error: bool = Falsefor _ in range(self._max_json_retries):- response = await self._model_client.create(context, json_output=True)+ response = await self._model_client.create(self._get_compatible_context(context), json_output=True)ledger_str = response.contenttry:assert isinstance(ledger_str, str)@@ -397,7 +402,9 @@ class MagenticOneOrchestrator(BaseGroupChatManager):update_facts_prompt = self._get_task_ledger_facts_update_prompt(self._task, self._facts)context.append(UserMessage(content=update_facts_prompt, source=self._name))- response = await self._model_client.create(context, cancellation_token=cancellation_token)+ response = await self._model_client.create(+ self._get_compatible_context(context), cancellation_token=cancellation_token+ )assert isinstance(response.content, str)self._facts = response.content@@ -407,7 +414,9 @@ class MagenticOneOrchestrator(BaseGroupChatManager):update_plan_prompt = self._get_task_ledger_plan_update_prompt(self._team_description)context.append(UserMessage(content=update_plan_prompt, source=self._name))- response = await self._model_client.create(context, cancellation_token=cancellation_token)+ response = await self._model_client.create(+ self._get_compatible_context(context), cancellation_token=cancellation_token+ )assert isinstance(response.content, str)self._plan = response.content@@ -420,7 +429,9 @@ class MagenticOneOrchestrator(BaseGroupChatManager):final_answer_prompt = self._get_final_answer_prompt(self._task)context.append(UserMessage(content=final_answer_prompt, source=self._name))- response = await self._model_client.create(context, cancellation_token=cancellation_token)+ response = await self._model_client.create(+ self._get_compatible_context(context), cancellation_token=cancellation_token+ )assert isinstance(response.content, str)message = TextMessage(content=response.content, source=self._name)@@ -464,15 +475,9 @@ class MagenticOneOrchestrator(BaseGroupChatManager):context.append(UserMessage(content=m.content, source=m.source))return context- def _content_to_str(self, content: str | List[str | Image]) -> str:- """Convert the content to a string."""- if isinstance(content, str):- return content+ def _get_compatible_context(self, messages: List[LLMMessage]) -> List[LLMMessage]:+ """Ensure that the messages are compatible with the underlying client, by removing images if needed."""+ if self._model_client.model_info["vision"]:+ return messageselse:- result: List[str] = []- for c in content:- if isinstance(c, str):- result.append(c)- else:- result.append("<image>")- return "\n".join(result)+ return remove_images(messages)diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/utils/__init__.py b/python/packages/autogen-agentchat/src/autogen_agentchat/utils/__init__.pyindex 7fada74af..44de85b3c 100644--- a/python/packages/autogen-agentchat/src/autogen_agentchat/utils/__init__.py+++ b/python/packages/autogen-agentchat/src/autogen_agentchat/utils/__init__.py@@ -2,6 +2,6 @@This module implements various utilities common to AgentChat agents and teams."""-from ._utils import remove_images+from ._utils import content_to_str, remove_images-__all__ = ["remove_images"]+__all__ = ["content_to_str", "remove_images"]diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/utils/_utils.py b/python/packages/autogen-agentchat/src/autogen_agentchat/utils/_utils.pyindex c70652770..6de117864 100644--- a/python/packages/autogen-agentchat/src/autogen_agentchat/utils/_utils.py+++ b/python/packages/autogen-agentchat/src/autogen_agentchat/utils/_utils.py@@ -1,11 +1,17 @@-from typing import List+from typing import List, Union-from autogen_core import Image-from autogen_core.models import LLMMessage, UserMessage+from autogen_core import FunctionCall, Image+from autogen_core.models import FunctionExecutionResult, LLMMessage, UserMessage+# Type aliases for convenience+_UserContent = Union[str, List[Union[str, Image]]]+_AssistantContent = Union[str, List[FunctionCall]]+_FunctionExecutionContent = List[FunctionExecutionResult]+_SystemContent = str-def _image_content_to_str(content: str | List[str | Image]) -> str:- """Convert the content of an LLMMessageto a string."""++def content_to_str(content: _UserContent | _AssistantContent | _FunctionExecutionContent | _SystemContent) -> str:+ """Convert the content of an LLMMessage to a string."""if isinstance(content, str):return contentelse:@@ -16,7 +22,7 @@ def _image_content_to_str(content: str | List[str | Image]) -> str:elif isinstance(c, Image):result.append("<image>")else:- raise AssertionError("Received unexpected content type.")+ result.append(str(c))return "\n".join(result)@@ -26,7 +32,7 @@ def remove_images(messages: List[LLMMessage]) -> List[LLMMessage]:str_messages: List[LLMMessage] = []for message in messages:if isinstance(message, UserMessage) and isinstance(message.content, list):- str_messages.append(UserMessage(content=_image_content_to_str(message.content), source=message.source))+ str_messages.append(UserMessage(content=content_to_str(message.content), source=message.source))else:str_messages.append(message)return str_messagesdiff --git a/python/packages/autogen-ext/src/autogen_ext/agents/file_surfer/_file_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/file_surfer/_file_surfer.pyindex 0f3893130..742bddc65 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/file_surfer/_file_surfer.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/file_surfer/_file_surfer.py@@ -9,6 +9,7 @@ from autogen_agentchat.messages import (MultiModalMessage,TextMessage,)+from autogen_agentchat.utils import remove_imagesfrom autogen_core import CancellationToken, FunctionCallfrom autogen_core.models import (AssistantMessage,@@ -126,7 +127,7 @@ class FileSurfer(BaseChatAgent):)create_result = await self._model_client.create(- messages=history + [context_message, task_message],+ messages=self._get_compatible_context(history + [context_message, task_message]),tools=[TOOL_OPEN_PATH,TOOL_PAGE_DOWN,@@ -172,3 +173,10 @@ class FileSurfer(BaseChatAgent):final_response = "TERMINATE"return False, final_response++ def _get_compatible_context(self, messages: List[LLMMessage]) -> List[LLMMessage]:+ """Ensure that the messages are compatible with the underlying client, by removing images if needed."""+ if self._model_client.model_info["vision"]:+ return messages+ else:+ return remove_images(messages)diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.pyindex 748f61e42..c83bf98d2 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py@@ -24,6 +24,7 @@ import PIL.Imagefrom autogen_agentchat.agents import BaseChatAgentfrom autogen_agentchat.base import Responsefrom autogen_agentchat.messages import AgentEvent, ChatMessage, MultiModalMessage, TextMessage+from autogen_agentchat.utils import content_to_str, remove_imagesfrom autogen_core import EVENT_LOGGER_NAME, CancellationToken, Component, ComponentModel, FunctionCallfrom autogen_core import Image as AGImagefrom autogen_core.models import (@@ -40,7 +41,13 @@ from pydantic import BaseModelfrom typing_extensions import Selffrom ._events import WebSurferEvent-from ._prompts import WEB_SURFER_OCR_PROMPT, WEB_SURFER_QA_PROMPT, WEB_SURFER_QA_SYSTEM_MESSAGE, WEB_SURFER_TOOL_PROMPT+from ._prompts import (+ WEB_SURFER_OCR_PROMPT,+ WEB_SURFER_QA_PROMPT,+ WEB_SURFER_QA_SYSTEM_MESSAGE,+ WEB_SURFER_TOOL_PROMPT_MM,+ WEB_SURFER_TOOL_PROMPT_TEXT,+)from ._set_of_mark import add_set_of_markfrom ._tool_definitions import (TOOL_CLICK,@@ -56,7 +63,6 @@ from ._tool_definitions import (TOOL_WEB_SEARCH,)from ._types import InteractiveRegion, UserContent-from ._utils import message_content_to_strfrom .playwright_controller import PlaywrightController@@ -215,8 +221,7 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):raise ValueError("The model does not support function calling. MultimodalWebSurfer requires a model that supports function calling.")- if model_client.model_info["vision"] is False:- raise ValueError("The model is not multimodal. MultimodalWebSurfer requires a multimodal model.")+self._model_client = model_clientself.headless = headlessself.browser_channel = browser_channel@@ -404,7 +409,7 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):self.model_usage: List[RequestUsage] = []try:content = await self._generate_reply(cancellation_token=cancellation_token)- self._chat_history.append(AssistantMessage(content=message_content_to_str(content), source=self.name))+ self._chat_history.append(AssistantMessage(content=content_to_str(content), source=self.name))final_usage = RequestUsage(prompt_tokens=sum([u.prompt_tokens for u in self.model_usage]),completion_tokens=sum([u.completion_tokens for u in self.model_usage]),@@ -434,22 +439,8 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):assert self._page is not None- # Clone the messages to give context, removing old screenshots- history: List[LLMMessage] = []- for m in self._chat_history:- assert isinstance(m, UserMessage | AssistantMessage | SystemMessage)- assert isinstance(m.content, str | list)-- if isinstance(m.content, str):- history.append(m)- else:- content = message_content_to_str(m.content)- if isinstance(m, UserMessage):- history.append(UserMessage(content=content, source=m.source))- elif isinstance(m, AssistantMessage):- history.append(AssistantMessage(content=content, source=m.source))- elif isinstance(m, SystemMessage):- history.append(SystemMessage(content=content))+ # Clone the messages, removing old screenshots+ history: List[LLMMessage] = remove_images(self._chat_history)# Ask the page for interactive elements, then prepare the state-of-mark screenshotrects = await self._playwright_controller.get_interactive_rects(self._page)@@ -512,22 +503,37 @@ class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]):tool_names = "\n".join([t["name"] for t in tools])- text_prompt = WEB_SURFER_TOOL_PROMPT.format(- url=self._page.url,- visible_targets=visible_targets,- other_targets_str=other_targets_str,- focused_hint=focused_hint,- tool_names=tool_names,- ).strip()-- # Scale the screenshot for the MLM, and close the original- scaled_screenshot = som_screenshot.resize((self.MLM_WIDTH, self.MLM_HEIGHT))- som_screenshot.close()- if self.to_save_screenshots:- scaled_screenshot.save(os.path.join(self.debug_dir, "screenshot_scaled.png")) # type: ignore+ if self._model_client.model_info["vision"]:+ text_prompt = WEB_SURFER_TOOL_PROMPT_MM.format(+ url=self._page.url,+ visible_targets=visible_targets,+ other_targets_str=other_targets_str,+ focused_hint=focused_hint,+ tool_names=tool_names,+ ).strip()++ # Scale the screenshot for the MLM, and close the original+ scaled_screenshot = som_screenshot.resize((self.MLM_WIDTH, self.MLM_HEIGHT))+ som_screenshot.close()+ if self.to_save_screenshots:+ scaled_screenshot.save(os.path.join(self.debug_dir, "screenshot_scaled.png")) # type: ignore++ # Add the message+ history.append(UserMessage(content=[text_prompt, AGImage.from_pil(scaled_screenshot)], source=self.name))+ else:+ visible_text = await self._playwright_controller.get_visible_text(self._page)- # Add the multimodal message and make the request- history.append(UserMessage(content=[text_prompt, AGImage.from_pil(scaled_screenshot)], source=self.name))+ text_prompt = WEB_SURFER_TOOL_PROMPT_TEXT.format(+ url=self._page.url,+ visible_targets=visible_targets,+ other_targets_str=other_targets_str,+ focused_hint=focused_hint,+ tool_names=tool_names,+ visible_text=visible_text.strip(),+ ).strip()++ # Add the message+ history.append(UserMessage(content=text_prompt, source=self.name))response = await self._model_client.create(history, tools=tools, extra_create_args={"tool_choice": "auto"}, cancellation_token=cancellation_tokendiff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.pyindex 2292a1fba..050b267e0 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_prompts.py@@ -1,4 +1,4 @@-WEB_SURFER_TOOL_PROMPT = """+WEB_SURFER_TOOL_PROMPT_MM = """Consider the following screenshot of a web browser, which is open to the page '{url}'. In this screenshot, interactive elements are outlined in bounding boxes of different colors. Each bounding box has a numeric ID label in the same color. Additional information about each visible label is listed below:{visible_targets}{other_targets_str}{focused_hint}@@ -13,6 +13,27 @@ When deciding between tools, consider if the request can be best addressed by:- on some other website entirely (in which case actions like performing a new web search might be the best option)"""+WEB_SURFER_TOOL_PROMPT_TEXT = """+Your web browser is open to the page '{url}'. The following text is visible in the viewport:++```+{visible_text}+```++You have also identified the following interactive components:++{visible_targets}{other_targets_str}{focused_hint}++You are to respond to the most recent request by selecting an appropriate tool from the following set, or by answering the question directly if possible without tools:++{tool_names}++When deciding between tools, consider if the request can be best addressed by:+ - the contents of the current viewport (in which case actions like clicking links, clicking buttons, inputting text might be most appropriate, or hovering over element)+ - contents found elsewhere on the full webpage (in which case actions like scrolling, summarization, or full-page Q&A might be most appropriate)+ - on some other website entirely (in which case actions like performing a new web search might be the best option)+"""+WEB_SURFER_OCR_PROMPT = """Please transcribe all visible text on this page, including both main content and the labels of UI elements."""diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_utils.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_utils.pydeleted file mode 100644index c492135b4..000000000--- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_utils.py+++ /dev/null@@ -1,25 +0,0 @@-from typing import List--from autogen_core import Image--from ._types import AssistantContent, FunctionExecutionContent, SystemContent, UserContent---# Convert UserContent to a string-def message_content_to_str(- message_content: UserContent | AssistantContent | SystemContent | FunctionExecutionContent,-) -> str:- if isinstance(message_content, str):- return message_content- elif isinstance(message_content, List):- converted: List[str] = list()- for item in message_content:- if isinstance(item, str):- converted.append(item.rstrip())- elif isinstance(item, Image):- converted.append("<Image>")- else:- converted.append(str(item).rstrip())- return "\n".join(converted)- else:- raise AssertionError("Unexpected response type.")diff --git a/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.py b/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.pyindex fc2e4f6b9..0b4bf8aea 100644--- a/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.py+++ b/python/packages/autogen-ext/src/autogen_ext/teams/magentic_one.py@@ -142,7 +142,7 @@ class MagenticOne(MagenticOneGroupChat):def _validate_client_capabilities(self, client: ChatCompletionClient) -> None:capabilities = client.model_info- required_capabilities = ["vision", "function_calling", "json_output"]+ required_capabilities = ["function_calling", "json_output"]if not all(capabilities.get(cap) for cap in required_capabilities):warnings.warn(