A programming framework for agentic AI
Message "content" now supports both `str` and `List` in Agents (#713)
Beibin Lic19f234149f6
diff --git a/.github/workflows/contrib-lmm.yml b/.github/workflows/contrib-lmm.ymldeleted file mode 100644index c032d5ea4..000000000--- a/.github/workflows/contrib-lmm.yml+++ /dev/null@@ -1,60 +0,0 @@-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions--name: ContribTests--on:- pull_request:- branches: ['main', 'dev/v0.2']- paths:- - 'autogen/img_utils.py'- - 'autogen/agentchat/contrib/multimodal_conversable_agent.py'- - 'autogen/agentchat/contrib/llava_agent.py'- - 'test/test_img_utils.py'- - 'test/agentchat/contrib/test_lmm.py'- - 'test/agentchat/contrib/test_llava.py'- - '.github/workflows/lmm-test.yml'- - 'setup.py'--concurrency:- group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}- cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}--jobs:- LMMTest:-- runs-on: ${{ matrix.os }}- strategy:- fail-fast: false- matrix:- os: [ubuntu-latest, macos-latest, windows-2019]- python-version: ["3.8", "3.9", "3.10", "3.11"]- steps:- - uses: actions/checkout@v3- - name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v4- with:- python-version: ${{ matrix.python-version }}- - name: Install packages and dependencies for all tests- run: |- python -m pip install --upgrade pip wheel- pip install pytest- - name: Install packages and dependencies for LMM- run: |- pip install -e .[lmm]- pip uninstall -y openai- - name: Test LMM and LLaVA- run: |- pytest test/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py- - name: Coverage- if: matrix.python-version == '3.10'- run: |- pip install coverage>=5.3- coverage run -a -m pytest test/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py- coverage xml- - name: Upload coverage to Codecov- if: matrix.python-version == '3.10'- uses: codecov/codecov-action@v3- with:- file: ./coverage.xml- flags: unittestsdiff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.ymlindex cbe594b6e..d0b40b85b 100644--- a/.github/workflows/contrib-tests.yml+++ b/.github/workflows/contrib-tests.yml@@ -136,3 +136,40 @@ jobs:- name: Test TeachableAgentrun: |pytest test/agentchat/contrib/test_teachable_agent.py++ LMMTest:+ runs-on: ${{ matrix.os }}+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, macos-latest, windows-2019]+ python-version: ["3.8", "3.9", "3.10", "3.11"]+ steps:+ - uses: actions/checkout@v3+ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v4+ with:+ python-version: ${{ matrix.python-version }}+ - name: Install packages and dependencies for all tests+ run: |+ python -m pip install --upgrade pip wheel+ pip install pytest+ - name: Install packages and dependencies for LMM+ run: |+ pip install -e .[lmm]+ pip uninstall -y openai+ - name: Test LMM and LLaVA+ run: |+ pytest test/agentchat/contrib/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py+ - name: Coverage+ if: matrix.python-version == '3.10'+ run: |+ pip install coverage>=5.3+ coverage run -a -m pytest test/agentchat/contrib/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py+ coverage xml+ - name: Upload coverage to Codecov+ if: matrix.python-version == '3.10'+ uses: codecov/codecov-action@v3+ with:+ file: ./coverage.xml+ flags: unittestsdiff --git a/.gitignore b/.gitignoreindex 622fc88ff..836acf2a8 100644--- a/.gitignore+++ b/.gitignore@@ -167,6 +167,7 @@ wolfram.txt# DB on disk for TeachableAgenttmp/+test/my_tmp/*# Storage for the AgentEval outputtest/test_files/agenteval-in-out/out/diff --git a/autogen/agentchat/assistant_agent.py b/autogen/agentchat/assistant_agent.pyindex b39b5f75f..55230708f 100644--- a/autogen/agentchat/assistant_agent.py+++ b/autogen/agentchat/assistant_agent.py@@ -1,6 +1,7 @@-from .conversable_agent import ConversableAgentfrom typing import Callable, Dict, Literal, Optional, Union+from .conversable_agent import ConversableAgent+class AssistantAgent(ConversableAgent):"""(In preview) Assistant agent, designed to solve a task with LLM.diff --git a/autogen/img_utils.py b/autogen/agentchat/contrib/img_utils.pysimilarity index 100%rename from autogen/img_utils.pyrename to autogen/agentchat/contrib/img_utils.pydiff --git a/autogen/agentchat/contrib/llava_agent.py b/autogen/agentchat/contrib/llava_agent.pyindex 39c4b2987..e4cb9c4e2 100644--- a/autogen/agentchat/contrib/llava_agent.py+++ b/autogen/agentchat/contrib/llava_agent.py@@ -10,9 +10,9 @@ import requestsfrom regex import Rfrom autogen.agentchat.agent import Agent+from autogen.agentchat.contrib.img_utils import get_image_data, llava_formaterfrom autogen.agentchat.contrib.multimodal_conversable_agent import MultimodalConversableAgentfrom autogen.code_utils import content_str-from autogen.img_utils import get_image_data, llava_formatertry:from termcolor import coloreddiff --git a/autogen/agentchat/contrib/multimodal_conversable_agent.py b/autogen/agentchat/contrib/multimodal_conversable_agent.pyindex 088861bb6..9a8c36591 100644--- a/autogen/agentchat/contrib/multimodal_conversable_agent.py+++ b/autogen/agentchat/contrib/multimodal_conversable_agent.py@@ -1,8 +1,9 @@+import copyfrom typing import Any, Callable, Dict, List, Optional, Tuple, Unionfrom autogen import OpenAIWrapperfrom autogen.agentchat import Agent, ConversableAgent-from autogen.img_utils import gpt4v_formatter+from autogen.agentchat.contrib.img_utils import gpt4v_formattertry:from termcolor import colored@@ -41,19 +42,14 @@ class MultimodalConversableAgent(ConversableAgent):*args,**kwargs,)-+ # call the setter to handle special format.self.update_system_message(system_message)self._is_termination_msg = (is_termination_msgif is_termination_msg is not None- else (lambda x: any([item["text"] == "TERMINATE" for item in x.get("content") if item["type"] == "text"]))+ else (lambda x: content_str(x.get("content")) == "TERMINATE"))- @property- def system_message(self) -> List:- """Return the system message."""- return self._oai_system_message[0]["content"]-def update_system_message(self, system_message: Union[Dict, List, str]):"""Update the system message.@@ -64,44 +60,29 @@ class MultimodalConversableAgent(ConversableAgent):self._oai_system_message[0]["role"] = "system"@staticmethod- def _message_to_dict(message: Union[Dict, List, str]):- """Convert a message to a dictionary.+ def _message_to_dict(message: Union[Dict, List, str]) -> Dict:+ """Convert a message to a dictionary. This implementation+ handles the GPT-4V formatting for easier prompts.- The message can be a string or a dictionary. The string will be put in the "content" field of the new dictionary.+ The message can be a string, a dictionary, or a list of dictionaries:+ - If it's a string, it will be cast into a list and placed in the 'content' field.+ - If it's a list, it will be directly placed in the 'content' field.+ - If it's a dictionary, it is already in message dict format. The 'content' field of this dictionary+ will be processed using the gpt4v_formatter."""if isinstance(message, str):return {"content": gpt4v_formatter(message)}if isinstance(message, list):return {"content": message}- else:+ if isinstance(message, dict):+ assert "content" in message, "The message dict must have a `content` field"+ if isinstance(message["content"], str):+ message = copy.deepcopy(message)+ message["content"] = gpt4v_formatter(message["content"])+ try:+ content_str(message["content"])+ except (TypeError, ValueError) as e:+ print("The `content` field should be compatible with the content_str function!")+ raise ereturn message-- def _print_received_message(self, message: Union[Dict, str], sender: Agent):- # print the message received- print(colored(sender.name, "yellow"), "(to", f"{self.name}):\n", flush=True)- if message.get("role") == "function":- func_print = f"***** Response from calling function \"{message['name']}\" *****"- print(colored(func_print, "green"), flush=True)- print(content_str(message["content"]), flush=True)- print(colored("*" * len(func_print), "green"), flush=True)- else:- content = message.get("content")- if content is not None:- if "context" in message:- content = OpenAIWrapper.instantiate(- content,- message["context"],- self.llm_config and self.llm_config.get("allow_format_str_template", False),- )- print(content_str(content), flush=True)- if "function_call" in message:- func_print = f"***** Suggested function Call: {message['function_call'].get('name', '(No function name found)')} *****"- print(colored(func_print, "green"), flush=True)- print(- "Arguments: \n",- message["function_call"].get("arguments", "(No arguments found)"),- flush=True,- sep="",- )- print(colored("*" * len(func_print), "green"), flush=True)- print("\n", "-" * 80, flush=True, sep="")+ raise ValueError(f"Unsupported message type: {type(message)}")diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.pyindex eb252b22b..492fc251c 100644--- a/autogen/agentchat/conversable_agent.py+++ b/autogen/agentchat/conversable_agent.py@@ -1,18 +1,14 @@import asyncio-from collections import defaultdictimport copyimport jsonimport logging+from collections import defaultdictfrom typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union+from autogen import OpenAIWrapper+from autogen.code_utils import DEFAULT_MODEL, UNKNOWN, content_str, execute_code, extract_code, infer_lang+from .agent import Agent-from autogen.code_utils import (- DEFAULT_MODEL,- UNKNOWN,- execute_code,- extract_code,- infer_lang,-)try:from termcolor import colored@@ -50,7 +46,7 @@ class ConversableAgent(Agent):def __init__(self,name: str,- system_message: Optional[str] = "You are a helpful AI Assistant.",+ system_message: Optional[Union[str, List]] = "You are a helpful AI Assistant.",is_termination_msg: Optional[Callable[[Dict], bool]] = None,max_consecutive_auto_reply: Optional[int] = None,human_input_mode: Optional[str] = "TERMINATE",@@ -62,7 +58,7 @@ class ConversableAgent(Agent):"""Args:name (str): name of the agent.- system_message (str): system message for the ChatCompletion inference.+ system_message (str or list): system message for the ChatCompletion inference.is_termination_msg (function): a function that takes a message in the form of a dictionaryand returns a boolean value indicating if this received message is a termination message.The dict can contain the following keys: "content", "role", "name", "function_call".@@ -105,8 +101,11 @@ class ConversableAgent(Agent):self._oai_messages = defaultdict(list)self._oai_system_message = [{"content": system_message, "role": "system"}]self._is_termination_msg = (- is_termination_msg if is_termination_msg is not None else (lambda x: x.get("content") == "TERMINATE")+ is_termination_msg+ if is_termination_msg is not None+ else (lambda x: content_str(x.get("content")) == "TERMINATE"))+if llm_config is False:self.llm_config = Falseself.client = None@@ -190,15 +189,15 @@ class ConversableAgent(Agent):)@property- def system_message(self):+ def system_message(self) -> Union[str, List]:"""Return the system message."""return self._oai_system_message[0]["content"]- def update_system_message(self, system_message: str):+ def update_system_message(self, system_message: Union[str, List]):"""Update the system message.Args:- system_message (str): system message for the ChatCompletion inference.+ system_message (str or List): system message for the ChatCompletion inference."""self._oai_system_message[0]["content"] = system_message@@ -258,7 +257,7 @@ class ConversableAgent(Agent):return None if self._code_execution_config is False else self._code_execution_config.get("use_docker")@staticmethod- def _message_to_dict(message: Union[Dict, str]):+ def _message_to_dict(message: Union[Dict, str]) -> Dict:"""Convert a message to a dictionary.The message can be a string or a dictionary. The string will be put in the "content" field of the new dictionary.@@ -314,7 +313,7 @@ class ConversableAgent(Agent):Args:message (dict or str): message to be sent.The message could contain the following fields:- - content (str): Required, the content of the message. (Can be None)+ - content (str or List): Required, the content of the message. (Can be None)- function_call (str): the name of the function to be called.- name (str): the name of the function to be called.- role (str): the role of the message, any role that is not "function"@@ -363,7 +362,7 @@ class ConversableAgent(Agent):Args:message (dict or str): message to be sent.The message could contain the following fields:- - content (str): Required, the content of the message. (Can be None)+ - content (str or List): Required, the content of the message. (Can be None)- function_call (str): the name of the function to be called.- name (str): the name of the function to be called.- role (str): the role of the message, any role that is not "function"@@ -419,7 +418,7 @@ class ConversableAgent(Agent):message["context"],self.llm_config and self.llm_config.get("allow_format_str_template", False),)- print(content, flush=True)+ print(content_str(content), flush=True)if "function_call" in message:function_call = dict(message["function_call"])func_print = (@@ -435,7 +434,7 @@ class ConversableAgent(Agent):print(colored("*" * len(func_print), "green"), flush=True)print("\n", "-" * 80, flush=True, sep="")- def _process_received_message(self, message, sender, silent):+ def _process_received_message(self, message: Union[Dict, str], sender: Agent, silent: bool):message = self._message_to_dict(message)# When the agent receives a message, the role of the message is "user". (If 'role' exists and is 'function', it will remain unchanged.)valid = self._append_oai_message(message, "user", sender)@@ -681,7 +680,7 @@ class ConversableAgent(Agent):messages: Optional[List[Dict]] = None,sender: Optional[Agent] = None,config: Optional[Any] = None,- ):+ ) -> Tuple[bool, Union[Dict, None]]:"""Generate a reply using function call."""if config is None:config = self@@ -698,7 +697,7 @@ class ConversableAgent(Agent):messages: Optional[List[Dict]] = None,sender: Optional[Agent] = None,config: Optional[Any] = None,- ):+ ) -> Tuple[bool, Union[Dict, None]]:"""Generate a reply using async function call."""if config is None:config = self@@ -720,8 +719,26 @@ class ConversableAgent(Agent):messages: Optional[List[Dict]] = None,sender: Optional[Agent] = None,config: Optional[Any] = None,- ) -> Tuple[bool, Union[str, Dict, None]]:- """Check if the conversation should be terminated, and if human reply is provided."""+ ) -> Tuple[bool, Union[str, None]]:+ """Check if the conversation should be terminated, and if human reply is provided.++ This method checks for conditions that require the conversation to be terminated, such as reaching+ a maximum number of consecutive auto-replies or encountering a termination message. Additionally,+ it prompts for and processes human input based on the configured human input mode, which can be+ 'ALWAYS', 'NEVER', or 'TERMINATE'. The method also manages the consecutive auto-reply counter+ for the conversation and prints relevant messages based on the human input received.++ Args:+ - messages (Optional[List[Dict]]): A list of message dictionaries, representing the conversation history.+ - sender (Optional[Agent]): The agent object representing the sender of the message.+ - config (Optional[Any]): Configuration object, defaults to the current instance if not provided.++ Returns:+ - Tuple[bool, Union[str, Dict, None]]: A tuple containing a boolean indicating if the conversation+ should be terminated, and a human reply which can be a string, a dictionary, or None.+ """+ # Function implementation...+if config is None:config = selfif messages is None:@@ -791,8 +808,24 @@ class ConversableAgent(Agent):messages: Optional[List[Dict]] = None,sender: Optional[Agent] = None,config: Optional[Any] = None,- ) -> Tuple[bool, Union[str, Dict, None]]:- """(async) Check if the conversation should be terminated, and if human reply is provided."""+ ) -> Tuple[bool, Union[str, None]]:+ """(async) Check if the conversation should be terminated, and if human reply is provided.++ This method checks for conditions that require the conversation to be terminated, such as reaching+ a maximum number of consecutive auto-replies or encountering a termination message. Additionally,+ it prompts for and processes human input based on the configured human input mode, which can be+ 'ALWAYS', 'NEVER', or 'TERMINATE'. The method also manages the consecutive auto-reply counter+ for the conversation and prints relevant messages based on the human input received.++ Args:+ - messages (Optional[List[Dict]]): A list of message dictionaries, representing the conversation history.+ - sender (Optional[Agent]): The agent object representing the sender of the message.+ - config (Optional[Any]): Configuration object, defaults to the current instance if not provided.++ Returns:+ - Tuple[bool, Union[str, Dict, None]]: A tuple containing a boolean indicating if the conversation+ should be terminated, and a human reply which can be a string, a dictionary, or None.+ """if config is None:config = selfif messages is None:@@ -962,8 +995,20 @@ class ConversableAgent(Agent):return replyreturn self._default_auto_reply- def _match_trigger(self, trigger, sender):- """Check if the sender matches the trigger."""+ def _match_trigger(self, trigger: Union[None, str, type, Agent, Callable, List], sender: Agent) -> bool:+ """Check if the sender matches the trigger.++ Args:+ - trigger (Union[None, str, type, Agent, Callable, List]): The condition to match against the sender.+ Can be `None`, string, type, `Agent` instance, callable, or a list of these.+ - sender (Agent): The sender object or type to be matched against the trigger.++ Returns:+ - bool: Returns `True` if the sender matches the trigger, otherwise `False`.++ Raises:+ - ValueError: If the trigger type is unsupported.+ """if trigger is None:return sender is Noneelif isinstance(trigger, str):@@ -971,9 +1016,12 @@ class ConversableAgent(Agent):elif isinstance(trigger, type):return isinstance(sender, trigger)elif isinstance(trigger, Agent):+ # return True if the sender is the same type (class) as the triggerreturn trigger == senderelif isinstance(trigger, Callable):- return trigger(sender)+ rst = trigger(sender)+ assert rst in [True, False], f"trigger {trigger} must return a boolean value."+ return rstelif isinstance(trigger, list):return any(self._match_trigger(t, sender) for t in trigger)else:@@ -1095,7 +1143,7 @@ class ConversableAgent(Agent):result.append(char)return "".join(result)- def execute_function(self, func_call):+ def execute_function(self, func_call) -> Tuple[bool, Dict[str, str]]:"""Execute a function call and return the result.Override this function to modify the way to execute a function call.@@ -1195,7 +1243,10 @@ class ConversableAgent(Agent):"""Generate the initial message for the agent.Override this function to customize the initial message based on user's request.- If not overridden, "message" needs to be provided in the context.+ If not overriden, "message" needs to be provided in the context.++ Args:+ **context: any context information, and "message" parameter needs to be provided."""return context["message"]diff --git a/autogen/agentchat/groupchat.py b/autogen/agentchat/groupchat.pyindex 7689246a0..deaca3036 100644--- a/autogen/agentchat/groupchat.py+++ b/autogen/agentchat/groupchat.py@@ -1,9 +1,11 @@import logging-import sysimport random+import re+import sysfrom dataclasses import dataclassfrom typing import Dict, List, Optional, Union-import re++from ..code_utils import content_strfrom .agent import Agentfrom .conversable_agent import ConversableAgent@@ -50,6 +52,14 @@ class GroupChat:"""Reset the group chat."""self.messages.clear()+ def append(self, message: Dict):+ """Append a message to the group chat.+ We cast the content to str here so that it can be managed by text-based+ model.+ """+ message["content"] = content_str(message["content"])+ self.messages.append(message)+def agent_by_name(self, name: str) -> Agent:"""Returns the agent with a given name."""return self.agents[self.agent_names.index(name)]@@ -64,7 +74,7 @@ class GroupChat:if self.agents[(offset + i) % len(self.agents)] in agents:return self.agents[(offset + i) % len(self.agents)]- def select_speaker_msg(self, agents: List[Agent]):+ def select_speaker_msg(self, agents: List[Agent]) -> str:"""Return the message for selecting the next speaker."""return f"""You are in a role play game. The following roles are available:{self._participant_roles(agents)}.@@ -72,7 +82,7 @@ class GroupChat:Read the following conversation.Then select the next role from {[agent.name for agent in agents]} to play. Only return the role."""- def manual_select_speaker(self, agents: List[Agent]) -> Agent:+ def manual_select_speaker(self, agents: List[Agent]) -> Union[Agent, None]:"""Manually select the next speaker."""print("Please select the next speaker from the following list:")@@ -190,19 +200,26 @@ Then select the next role from {[agent.name for agent in agents]} to play. Onlyroles = []for agent in agents:- if agent.system_message.strip() == "":+ if content_str(agent.system_message).strip() == "":logger.warning(f"The agent '{agent.name}' has an empty system_message, and may not work well with GroupChat.")roles.append(f"{agent.name}: {agent.system_message}")return "\n".join(roles)- def _mentioned_agents(self, message_content: str, agents: List[Agent]) -> Dict:- """- Finds and counts agent mentions in the string message_content, taking word boundaries into account.+ def _mentioned_agents(self, message_content: Union[str, List], agents: List[Agent]) -> Dict:+ """Counts the number of times each agent is mentioned in the provided message content.++ Args:+ message_content (Union[str, List]): The content of the message, either as a single string or a list of strings.+ agents (List[Agent]): A list of Agent objects, each having a 'name' attribute to be searched in the message content.- Returns: A dictionary mapping agent names to mention counts (to be included, at least one mention must occur)+ Returns:+ Dict: a counter for mentioned agents."""+ # Cast message content to str+ message_content = content_str(message_content)+mentions = dict()for agent in agents:regex = (@@ -224,7 +241,7 @@ class GroupChatManager(ConversableAgent):# unlimited consecutive auto reply by defaultmax_consecutive_auto_reply: Optional[int] = sys.maxsize,human_input_mode: Optional[str] = "NEVER",- system_message: Optional[str] = "Group chat manager.",+ system_message: Optional[Union[str, List]] = "Group chat manager.",**kwargs,):super().__init__(@@ -256,12 +273,12 @@ class GroupChatManager(ConversableAgent):# set the name to speaker's name if the role is not functionif message["role"] != "function":message["name"] = speaker.name- groupchat.messages.append(message)++ groupchat.append(message)if self._is_termination_msg(message):# The conversation is overbreak-# broadcast the message to all agents except the speakerfor agent in groupchat.agents:if agent != speaker:@@ -306,7 +323,8 @@ class GroupChatManager(ConversableAgent):# set the name to speaker's name if the role is not functionif message["role"] != "function":message["name"] = speaker.name- groupchat.messages.append(message)++ groupchat.append(message)if self._is_termination_msg(message):# The conversation is overdiff --git a/autogen/agentchat/user_proxy_agent.py b/autogen/agentchat/user_proxy_agent.pyindex 8a218b1fe..b658aed4b 100644--- a/autogen/agentchat/user_proxy_agent.py+++ b/autogen/agentchat/user_proxy_agent.py@@ -1,5 +1,6 @@+from typing import Callable, Dict, List, Literal, Optional, Union+from .conversable_agent import ConversableAgent-from typing import Callable, Dict, Literal, Optional, Unionclass UserProxyAgent(ConversableAgent):@@ -25,7 +26,7 @@ class UserProxyAgent(ConversableAgent):code_execution_config: Optional[Union[Dict, Literal[False]]] = None,default_auto_reply: Optional[Union[str, Dict, None]] = "",llm_config: Optional[Union[Dict, Literal[False]]] = False,- system_message: Optional[str] = "",+ system_message: Optional[Union[str, List]] = "",):"""Args:@@ -66,7 +67,7 @@ class UserProxyAgent(ConversableAgent):Please refer to [OpenAIWrapper.create](/docs/reference/oai/client#create)for available options.Default to false, which disables llm-based auto reply.- system_message (str): system message for ChatCompletion inference.+ system_message (str or List): system message for ChatCompletion inference.Only used when llm_config is not False. Use it to reprogram the agent."""super().__init__(diff --git a/autogen/code_utils.py b/autogen/code_utils.pyindex f03094c8b..dbb863716 100644--- a/autogen/code_utils.py+++ b/autogen/code_utils.py@@ -38,16 +38,44 @@ PATH_SEPARATOR = WIN32 and "\\" or "/"logger = logging.getLogger(__name__)-def content_str(content: Union[str, List]) -> str:- if type(content) is str:+def content_str(content: Union[str, List, None]) -> str:+ """Converts `content` into a string format.++ This function processes content that may be a string, a list of mixed text and image URLs, or None,+ and converts it into a string. Text is directly appended to the result string, while image URLs are+ represented by a placeholder image token. If the content is None, an empty string is returned.++ Args:+ - content (Union[str, List, None]): The content to be processed. Can be a string, a list of dictionaries+ representing text and image URLs, or None.++ Returns:+ str: A string representation of the input content. Image URLs are replaced with an image token.++ Note:+ - The function expects each dictionary in the list to have a "type" key that is either "text" or "image_url".+ For "text" type, the "text" key's value is appended to the result. For "image_url", an image token is appended.+ - This function is useful for handling content that may include both text and image references, especially+ in contexts where images need to be represented as placeholders.+ """+ if content is None:+ return ""+ if isinstance(content, str):return content+ if not isinstance(content, list):+ raise TypeError(f"content must be None, str, or list, but got {type(content)}")+rst = ""for item in content:+ if not isinstance(item, dict):+ raise TypeError("Wrong content format: every element should be dict if the content is a list.")+ assert "type" in item, "Wrong content format. Missing 'type' key in content's dict."if item["type"] == "text":rst += item["text"]- else:- assert isinstance(item, dict) and item["type"] == "image_url", "Wrong content format."+ elif item["type"] == "image_url":rst += "<image>"+ else:+ raise ValueError(f"Wrong content format: unknown type {item['type']} within the content")return rstdiff --git a/notebook/agentchat_dalle_and_gpt4v.ipynb b/notebook/agentchat_dalle_and_gpt4v.ipynbindex b5590862e..8d7577b14 100644--- a/notebook/agentchat_dalle_and_gpt4v.ipynb+++ b/notebook/agentchat_dalle_and_gpt4v.ipynb@@ -39,7 +39,7 @@"import autogen\n","from autogen import AssistantAgent, Agent, UserProxyAgent, ConversableAgent\n","\n",- "from autogen.img_utils import get_image_data, _to_pil\n",+ "from autogen.agentchat.contrib.img_utils import get_image_data, _to_pil\n","from termcolor import colored\n","import random"]diff --git a/notebook/agentchat_lmm_gpt-4v.ipynb b/notebook/agentchat_lmm_gpt-4v.ipynbindex d3ece5c66..260a22d35 100644--- a/notebook/agentchat_lmm_gpt-4v.ipynb+++ b/notebook/agentchat_lmm_gpt-4v.ipynb@@ -91,38 +91,6 @@{"cell_type": "code","execution_count": 3,- "id": "57462351",- "metadata": {},- "outputs": [- {- "data": {- "text/plain": [- "['openai']"- ]- },- "execution_count": 3,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "# Remove the `api_type` param as it is not needed for 4V\n",- "[config.pop(\"api_type\", None) for config in config_list_4v]"- ]- },- {- "cell_type": "code",- "execution_count": 4,- "id": "e23df0dd",- "metadata": {},- "outputs": [],- "source": [- "# image_agent._oai_messages[user_proxy]"- ]- },- {- "cell_type": "code",- "execution_count": 5,"id": "67157629","metadata": {"scrolled": false@@ -180,7 +148,7 @@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": 4,"id": "73a2b234","metadata": {"scrolled": false@@ -236,7 +204,7 @@},{"cell_type": "code",- "execution_count": 7,+ "execution_count": 5,"id": "e8eca993","metadata": {},"outputs": [],@@ -339,7 +307,7 @@},{"cell_type": "code",- "execution_count": 8,+ "execution_count": 6,"id": "977b9017","metadata": {"scrolled": false@@ -724,7 +692,7 @@},{"cell_type": "code",- "execution_count": 9,+ "execution_count": 7,"id": "f0a58827","metadata": {},"outputs": [],@@ -736,10 +704,100 @@{"cell_type": "code","execution_count": null,- "id": "b95bf449",+ "id": "c6206648","metadata": {},"outputs": [],"source": []+ },+ {+ "cell_type": "markdown",+ "id": "a95d87c2",+ "metadata": {},+ "source": [+ "## Group Chat Example with Multimodal Agent"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "id": "56bd5742",+ "metadata": {+ "scrolled": false+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n",+ "\n",+ "Describe the image:\n",+ " <img https://th.bing.com/th/id/R.422068ce8af4e15b0634fe2540adea7a?rik=y4OcXBE%2fqutDOw&pid=ImgRaw&r=0>.\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001b[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",+ "\u001b[33mimage-explainer-1\u001b[0m (to chat_manager):\n",+ "\n",+ "In a soft-focus world, a caramel-colored puppy with a coat of curly fur sits serenely, its innocent eyes gazing into the distance. Adorned with a collar that hosts a vibrant, multicolored bandana and a shiny tag engraved with the name \"Webster,\" the pup exudes a sense of youthful curiosity and gentle charm. Behind this bundle of joy, the muted backdrop of a home's interior whispers tales of comfort and domesticity, with a pair of black boots resting by the door, hinting at the comings and goings of human life amidst which this little creature finds its love and belonging.\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001b[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",+ "\u001b[33mimage-explainer-2\u001b[0m (to chat_manager):\n",+ "\n",+ "The image shows a young, caramel-colored puppy with curly fur sitting on the floor. The puppy is wearing a blue collar with a colorful bandana and a tag that appears to have the name \"Webster\" engraved on it. In the background, there are black boots near a white door, suggesting an indoor, home setting. The focus is on the puppy, making the background appear softly blurred. The puppy's expression is gentle, with a hint of curiosity in its eyes.\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001b[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",+ "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n",+ "\n",+ "\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001b[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",+ "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n",+ "\n",+ "\n",+ "\n",+ "--------------------------------------------------------------------------------\n"+ ]+ }+ ],+ "source": [+ "agent1 = MultimodalConversableAgent(\n",+ " name=\"image-explainer-1\",\n",+ " max_consecutive_auto_reply=10,\n",+ " llm_config={\"config_list\": config_list_4v, \"temperature\": 0.5, \"max_tokens\": 300},\n",+ " system_message=\"Your image description is poetic and engaging.\",\n",+ ")\n",+ "agent2 = MultimodalConversableAgent(\n",+ " name=\"image-explainer-2\",\n",+ " max_consecutive_auto_reply=10,\n",+ " llm_config={\"config_list\": config_list_4v, \"temperature\": 0.5, \"max_tokens\": 300},\n",+ " system_message=\"Your image description is factual and to the point.\",\n",+ ")\n",+ "\n",+ "user_proxy = autogen.UserProxyAgent(\n",+ " name=\"User_proxy\",\n",+ " system_message=\"Ask both image explainer 1 and 2 for their description.\",\n",+ " human_input_mode=\"TERMINATE\", # Try between ALWAYS, NEVER, and TERMINATE\n",+ " max_consecutive_auto_reply=10,\n",+ ")\n",+ "\n",+ "# We set max_round to 5\n",+ "groupchat = autogen.GroupChat(agents=[agent1, agent2, user_proxy], \n",+ " messages=[], \n",+ " max_round=5)\n",+ "group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, \n",+ " llm_config=gpt4_llm_config)\n",+ "\n",+ "user_proxy.initiate_chat(group_chat_manager,\n",+ " message=f\"\"\"Describe the image:\n",+ " <img https://th.bing.com/th/id/R.422068ce8af4e15b0634fe2540adea7a?rik=y4OcXBE%2fqutDOw&pid=ImgRaw&r=0>.\"\"\")"+ ]}],"metadata": {diff --git a/test/test_img_utils.py b/test/agentchat/contrib/test_img_utils.pysimilarity index 94%rename from test/test_img_utils.pyrename to test/agentchat/contrib/test_img_utils.pyindex fc0053a65..3081c5d55 100644--- a/test/test_img_utils.py+++ b/test/agentchat/contrib/test_img_utils.py@@ -10,7 +10,7 @@ import requeststry:from PIL import Image- from autogen.img_utils import extract_img_paths, get_image_data, gpt4v_formatter, llava_formater+ from autogen.agentchat.contrib.img_utils import extract_img_paths, get_image_data, gpt4v_formatter, llava_formaterexcept ImportError:skip = Trueelse:@@ -71,7 +71,7 @@ class TestLlavaFormater(unittest.TestCase):result = llava_formater(prompt)self.assertEqual(result, expected_output)- @patch("autogen.img_utils.get_image_data")+ @patch("autogen.agentchat.contrib.img_utils.get_image_data")def test_with_images(self, mock_get_image_data):"""Test the llava_formater function with a prompt containing images.@@ -84,7 +84,7 @@ class TestLlavaFormater(unittest.TestCase):result = llava_formater(prompt)self.assertEqual(result, expected_output)- @patch("autogen.img_utils.get_image_data")+ @patch("autogen.agentchat.contrib.img_utils.get_image_data")def test_with_ordered_images(self, mock_get_image_data):"""Test the llava_formater function with ordered image tokens.@@ -109,7 +109,7 @@ class TestGpt4vFormatter(unittest.TestCase):result = gpt4v_formatter(prompt)self.assertEqual(result, expected_output)- @patch("autogen.img_utils.get_image_data")+ @patch("autogen.agentchat.contrib.img_utils.get_image_data")def test_with_images(self, mock_get_image_data):"""Test the gpt4v_formatter function with a prompt containing images.@@ -126,7 +126,7 @@ class TestGpt4vFormatter(unittest.TestCase):result = gpt4v_formatter(prompt)self.assertEqual(result, expected_output)- @patch("autogen.img_utils.get_image_data")+ @patch("autogen.agentchat.contrib.img_utils.get_image_data")def test_multiple_images(self, mock_get_image_data):"""Test the gpt4v_formatter function with a prompt containing multiple images.diff --git a/test/agentchat/contrib/test_lmm.py b/test/agentchat/contrib/test_lmm.pyindex 89ef62ccc..415133971 100644--- a/test/agentchat/contrib/test_lmm.py+++ b/test/agentchat/contrib/test_lmm.py@@ -79,5 +79,53 @@ class TestMultimodalConversableAgent(unittest.TestCase):self.agent._print_received_message.assert_called_with(message_str, sender)[email protected](skip, reason="Dependency not installed")+def test_group_chat_with_lmm():+ """+ Tests the group chat functionality with two MultimodalConversable Agents.+ Verifies that the chat is correctly limited by the max_round parameter.+ Each agent is set to describe an image in a unique style, but the chat should not exceed the specified max_rounds.+ """++ # Configuration parameters+ max_round = 5+ max_consecutive_auto_reply = 10+ llm_config = False++ # Creating two MultimodalConversable Agents with different descriptive styles+ agent1 = MultimodalConversableAgent(+ name="image-explainer-1",+ max_consecutive_auto_reply=max_consecutive_auto_reply,+ llm_config=llm_config,+ system_message="Your image description is poetic and engaging.",+ )+ agent2 = MultimodalConversableAgent(+ name="image-explainer-2",+ max_consecutive_auto_reply=max_consecutive_auto_reply,+ llm_config=llm_config,+ system_message="Your image description is factual and to the point.",+ )++ # Creating a user proxy agent for initiating the group chat+ user_proxy = autogen.UserProxyAgent(+ name="User_proxy",+ system_message="Ask both image explainer 1 and 2 for their description.",+ human_input_mode="NEVER", # Options: 'ALWAYS' or 'NEVER'+ max_consecutive_auto_reply=max_consecutive_auto_reply,+ )++ # Setting up the group chat+ groupchat = autogen.GroupChat(agents=[agent1, agent2, user_proxy], messages=[], max_round=max_round)+ group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config)++ # Initiating the group chat and observing the number of rounds+ user_proxy.initiate_chat(group_chat_manager, message=f"What do you see? <img {base64_encoded_image}>")++ # Assertions to check if the number of rounds does not exceed max_round+ assert all(len(arr) <= max_round for arr in agent1._oai_messages.values()), "Agent 1 exceeded max rounds"+ assert all(len(arr) <= max_round for arr in agent2._oai_messages.values()), "Agent 2 exceeded max rounds"+ assert all(len(arr) <= max_round for arr in user_proxy._oai_messages.values()), "User proxy exceeded max rounds"++if __name__ == "__main__":unittest.main()diff --git a/test/test_code.py b/test/test_code.pyindex a00ef072f..ec8ada2c9 100644--- a/test/test_code.py+++ b/test/test_code.py@@ -403,7 +403,7 @@ class TestContentStr(unittest.TestCase):def test_invalid_content(self):content = [{"type": "text", "text": "hello"}, {"type": "wrong_type", "url": "http://example.com/image.png"}]- with self.assertRaises(AssertionError) as context:+ with self.assertRaises(ValueError) as context:content_str(content)self.assertIn("Wrong content format", str(context.exception))