š« CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Support multi-modal input and multi-modal output in one agent Dalle support (#698)
Will Shang06f87e6049e7
diff --git a/camel/messages/base.py b/camel/messages/base.pyindex 126c27171..f58895250 100644--- a/camel/messages/base.py+++ b/camel/messages/base.py@@ -60,6 +60,7 @@ class BaseMessage:role_type: RoleTypemeta_dict: Optional[Dict[str, str]]content: str+video_bytes: Optional[bytes] = Noneimage_list: Optional[List[Image.Image]] = Noneimage_detail: Literal["auto", "low", "high"] = "auto"diff --git a/camel/prompts/__init__.py b/camel/prompts/__init__.pyindex b3754d8e4..5bf39c7f1 100644--- a/camel/prompts/__init__.py+++ b/camel/prompts/__init__.py@@ -18,7 +18,11 @@ from .evaluation import EvaluationPromptTemplateDictfrom .generate_text_embedding_data import (GenerateTextEmbeddingDataPromptTemplateDict,)+from .image_craft import ImageCraftPromptTemplateDictfrom .misalignment import MisalignmentPromptTemplateDict+from .multi_condition_image_craft import (+ MultiConditionImageCraftPromptTemplateDict,+)from .object_recognition import ObjectRecognitionPromptTemplateDictfrom .prompt_templates import PromptTemplateGeneratorfrom .role_description_prompt_template import RoleDescriptionPromptTemplateDict@@ -42,5 +46,8 @@ __all__ = ['SolutionExtractionPromptTemplateDict','GenerateTextEmbeddingDataPromptTemplateDict','ObjectRecognitionPromptTemplateDict',+ 'ImageCraftPromptTemplateDict',+ 'MultiConditionImageCraftPromptTemplateDict',+ 'DescriptionVideoPromptTemplateDict','VideoDescriptionPromptTemplateDict',]diff --git a/camel/prompts/image_craft.py b/camel/prompts/image_craft.pynew file mode 100644index 000000000..0c7fb3deb--- /dev/null+++ b/camel/prompts/image_craft.py@@ -0,0 +1,34 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from typing import Any++from camel.prompts import TextPrompt, TextPromptDict+from camel.types import RoleType+++class ImageCraftPromptTemplateDict(TextPromptDict):+ ASSISTANT_PROMPT = TextPrompt(+ """You are tasked with creating an original image based on+ the provided descriptive captions. Use your imagination+ and artistic skills to visualize and draw the images and+ explain your thought process."""+ )++ def __init__(self, *args: Any, **kwargs: Any) -> None:+ super().__init__(*args, **kwargs)+ self.update(+ {+ RoleType.ASSISTANT: self.ASSISTANT_PROMPT,+ }+ )diff --git a/camel/prompts/multi_condition_image_craft.py b/camel/prompts/multi_condition_image_craft.pynew file mode 100644index 000000000..7b017b01b--- /dev/null+++ b/camel/prompts/multi_condition_image_craft.py@@ -0,0 +1,34 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from typing import Any++from camel.prompts import TextPrompt, TextPromptDict+from camel.types import RoleType+++class MultiConditionImageCraftPromptTemplateDict(TextPromptDict):+ ASSISTANT_PROMPT = TextPrompt(+ """You are tasked with creating an image based on+ the provided text and images conditions. Please use your+ imagination and artistic capabilities to visualize and+ draw the images and explain what you are thinking about."""+ )++ def __init__(self, *args: Any, **kwargs: Any) -> None:+ super().__init__(*args, **kwargs)+ self.update(+ {+ RoleType.ASSISTANT: self.ASSISTANT_PROMPT,+ }+ )diff --git a/camel/prompts/task_prompt_template.py b/camel/prompts/task_prompt_template.pyindex 4e7df06dd..6662697bb 100644--- a/camel/prompts/task_prompt_template.py+++ b/camel/prompts/task_prompt_template.py@@ -24,7 +24,11 @@ from camel.prompts.evaluation import (from camel.prompts.generate_text_embedding_data import (GenerateTextEmbeddingDataPromptTemplateDict,)+from camel.prompts.image_craft import ImageCraftPromptTemplateDictfrom camel.prompts.misalignment import MisalignmentPromptTemplateDict+from camel.prompts.multi_condition_image_craft import (+ MultiConditionImageCraftPromptTemplateDict,+)from camel.prompts.object_recognition import (ObjectRecognitionPromptTemplateDict,)@@ -64,6 +68,8 @@ class TaskPromptTemplateDict(Dict[Any, TextPromptDict]):TaskType.ROLE_DESCRIPTION: RoleDescriptionPromptTemplateDict(),TaskType.OBJECT_RECOGNITION: ObjectRecognitionPromptTemplateDict(), # noqa: E501TaskType.GENERATE_TEXT_EMBEDDING_DATA: GenerateTextEmbeddingDataPromptTemplateDict(), # noqa: E501+ TaskType.IMAGE_CRAFT: ImageCraftPromptTemplateDict(),+ TaskType.MULTI_CONDITION_IMAGE_CRAFT: MultiConditionImageCraftPromptTemplateDict(), # noqa: E501TaskType.VIDEO_DESCRIPTION: VideoDescriptionPromptTemplateDict(), # noqa: E501})diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.pyindex 6eb50d2e2..72432a7f6 100644--- a/camel/toolkits/__init__.py+++ b/camel/toolkits/__init__.py@@ -27,6 +27,7 @@ from .search_toolkit import SEARCH_FUNCS, SearchToolkitfrom .twitter_toolkit import TWITTER_FUNCS, TwitterToolkitfrom .weather_toolkit import WEATHER_FUNCS, WeatherToolkitfrom .slack_toolkit import SLACK_FUNCS, SlackToolkit+from .dalle_toolkit import DALLE_FUNCS, DalleToolkitfrom .base import BaseToolkitfrom .code_execution import CodeExecutionToolkit@@ -45,12 +46,14 @@ __all__ = ['TWITTER_FUNCS','WEATHER_FUNCS','SLACK_FUNCS',+ 'DALLE_FUNCS','BaseToolkit','GithubToolkit','MathToolkit','GoogleMapsToolkit','SearchToolkit','SlackToolkit',+ 'DalleToolkit','TwitterToolkit','WeatherToolkit','RetrievalToolkit',diff --git a/camel/toolkits/dalle_toolkit.py b/camel/toolkits/dalle_toolkit.pynew file mode 100644index 000000000..4898f0d69--- /dev/null+++ b/camel/toolkits/dalle_toolkit.py@@ -0,0 +1,146 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import base64+import os+import uuid+from io import BytesIO+from typing import List, Optional++from openai import OpenAI+from PIL import Image++from camel.toolkits import OpenAIFunction+from camel.toolkits.base import BaseToolkit+++class DalleToolkit(BaseToolkit):+ r"""A class representing a toolkit for image generation using OpenAI's.++ This class provides methods handle image generation using OpenAI's DALL-E.+ """++ def base64_to_image(self, base64_string: str) -> Optional[Image.Image]:+ r"""Converts a base64 encoded string into a PIL Image object.++ Args:+ base64_string (str): The base64 encoded string of the image.++ Returns:+ Optional[Image.Image]: The PIL Image object or None if conversion+ fails.+ """+ try:+ # Decode the base64 string to get the image data+ image_data = base64.b64decode(base64_string)+ # Create a memory buffer for the image data+ image_buffer = BytesIO(image_data)+ # Open the image using the PIL library+ image = Image.open(image_buffer)+ return image+ except Exception as e:+ print(f"An error occurred while converting base64 to image: {e}")+ return None++ def image_path_to_base64(self, image_path: str) -> str:+ r"""Converts the file path of an image to a Base64 encoded string.++ Args:+ image_path (str): The path to the image file.++ Returns:+ str: A Base64 encoded string representing the content of the image+ file.+ """+ try:+ with open(image_path, "rb") as image_file:+ return base64.b64encode(image_file.read()).decode('utf-8')+ except Exception as e:+ print(+ f"An error occurred while converting image path to base64: {e}"+ )+ return ""++ def image_to_base64(self, image: Image.Image) -> str:+ r"""Converts an image into a base64-encoded string.++ This function takes an image object as input, encodes the image into a+ PNG format base64 string, and returns it.+ If the encoding process encounters an error, it prints the error+ message and returns None.++ Args:+ image: The image object to be encoded, supports any image format+ that can be saved in PNG format.++ Returns:+ str: A base64-encoded string of the image.+ """+ try:+ with BytesIO() as buffered_image:+ image.save(buffered_image, format="PNG")+ buffered_image.seek(0)+ image_bytes = buffered_image.read()+ base64_str = base64.b64encode(image_bytes).decode('utf-8')+ return base64_str+ except Exception as e:+ print(f"An error occurred: {e}")+ return ""++ def get_dalle_img(self, prompt: str, image_dir: str = "img") -> str:+ r"""Generate an image using OpenAI's DALL-E model.+ The generated image is saved to the specified directory.++ Args:+ prompt (str): The text prompt based on which the image is+ generated.+ image_dir (str): The directory to save the generated image.+ Defaults to 'img'.++ Returns:+ str: The path to the saved image.+ """++ dalle_client = OpenAI()+ response = dalle_client.images.generate(+ model="dall-e-3",+ prompt=prompt,+ size="1024x1792",+ quality="standard",+ n=1, # NOTE: now dall-e-3 only supports n=1+ response_format="b64_json",+ )+ image_b64 = response.data[0].b64_json+ image = self.base64_to_image(image_b64) # type: ignore[arg-type]++ if image is None:+ raise ValueError("Failed to convert base64 string to image.")++ os.makedirs(image_dir, exist_ok=True)+ image_path = os.path.join(image_dir, f"{uuid.uuid4()}.png")+ image.save(image_path)++ return image_path++ def get_tools(self) -> List[OpenAIFunction]:+ r"""Returns a list of OpenAIFunction objects representing the+ functions in the toolkit.++ Returns:+ List[OpenAIFunction]: A list of OpenAIFunction objects+ representing the functions in the toolkit.+ """+ return [OpenAIFunction(self.get_dalle_img)]+++DALLE_FUNCS: List[OpenAIFunction] = DalleToolkit().get_tools()diff --git a/camel/types/enums.py b/camel/types/enums.pyindex 840eff0cc..0cbdef041 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -361,6 +361,8 @@ class TaskType(Enum):ROLE_DESCRIPTION = "role_description"GENERATE_TEXT_EMBEDDING_DATA = "generate_text_embedding_data"OBJECT_RECOGNITION = "object_recognition"+ IMAGE_CRAFT = "image_craft"+ MULTI_CONDITION_IMAGE_CRAFT = "multi_condition_image_craft"DEFAULT = "default"VIDEO_DESCRIPTION = "video_description"diff --git a/examples/vision/image_crafting.py b/examples/vision/image_crafting.pynew file mode 100644index 000000000..c6c66e8c9--- /dev/null+++ b/examples/vision/image_crafting.py@@ -0,0 +1,88 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from camel.agents.chat_agent import ChatAgent+from camel.configs import ChatGPTConfig+from camel.messages.base import BaseMessage+from camel.models import ModelFactory+from camel.prompts import PromptTemplateGenerator+from camel.toolkits import DALLE_FUNCS+from camel.types import (+ ModelPlatformType,+ ModelType,+ RoleType,+ TaskType,+)+++def main():+ sys_msg = PromptTemplateGenerator().get_prompt_from_key(+ TaskType.IMAGE_CRAFT, RoleType.ASSISTANT+ )+ print("=" * 20 + " SYS MSG " + "=" * 20)+ print(sys_msg)+ print("=" * 49)++ assistant_sys_msg = BaseMessage.make_assistant_message(+ role_name="Artist",+ content=sys_msg,+ )++ user_msg = BaseMessage.make_user_message(+ role_name="User",+ content="Draw a picture of a camel.",+ )++ model_config = ChatGPTConfig(tools=[*DALLE_FUNCS])++ model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.GPT_4O_MINI,+ model_config_dict=model_config.__dict__,+ )++ dalle_agent = ChatAgent(+ system_message=assistant_sys_msg,+ model=model,+ tools=DALLE_FUNCS,+ )++ response = dalle_agent.step(user_msg)++ print("=" * 20 + " RESULT " + "=" * 20)+ print(response.msg.content)+ print("=" * 48)+++if __name__ == "__main__":+ main()++"""+===============================================================================+==================== SYS MSG ====================+You are tasked with creating an original image based on+ the provided descriptive captions. Use your imagination+ and artistic skills to visualize and draw the images and+ explain your thought process.+=================================================+==================== RESULT ====================+I have created an image of a camel standing in a desert oasis under the shade+of a palm tree. You can see the realistic and detailed drawing of the camel in+the image below.++++The scene captures the essence of the desert environment with the camel+peacefully resting in the oasis.+===============================================================================+"""diff --git a/examples/vision/multi_condition_image_crafting.py b/examples/vision/multi_condition_image_crafting.pynew file mode 100644index 000000000..4eca5fdfa--- /dev/null+++ b/examples/vision/multi_condition_image_crafting.py@@ -0,0 +1,93 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from PIL import Image++from camel.agents.chat_agent import ChatAgent+from camel.configs import ChatGPTConfig+from camel.generators import PromptTemplateGenerator+from camel.messages.base import BaseMessage+from camel.models import ModelFactory+from camel.toolkits import DALLE_FUNCS+from camel.types import (+ ModelPlatformType,+ ModelType,+ RoleType,+ TaskType,+)+++def main(image_paths: list[str]) -> list[str]:+ sys_msg = PromptTemplateGenerator().get_prompt_from_key(+ TaskType.MULTI_CONDITION_IMAGE_CRAFT, RoleType.ASSISTANT+ )+ print("=" * 20 + " SYS MSG " + "=" * 20)+ print(sys_msg)+ print("=" * 49)++ assistant_sys_msg = BaseMessage.make_assistant_message(+ role_name="Artist",+ content=sys_msg,+ )++ model_config = ChatGPTConfig(tools=[*DALLE_FUNCS])++ model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.GPT_4O_MINI,+ model_config_dict=model_config.__dict__,+ )++ dalle_agent = ChatAgent(+ system_message=assistant_sys_msg,+ model=model,+ tools=DALLE_FUNCS,+ )++ image_list = [Image.open(image_path) for image_path in image_paths]++ user_msg = BaseMessage.make_user_message(+ role_name="User",+ content='''Please generate an image based on the provided images and+ text, make the backgroup of this image is in the morning''',+ image_list=image_list,+ )++ response = dalle_agent.step(user_msg)++ print("=" * 20 + " RESULT " + "=" * 20)+ print(response.msg.content)+ print("=" * 48)+++if __name__ == "__main__":+ main()++"""+===============================================================================+==================== SYS MSG ====================+You are tasked with creating an image based on+ the provided text and images conditions. Please use your+ imagination and artistic capabilities to visualize and+ draw the images and explain what you are thinking about.+=================================================+==================== RESULT ====================+Here is the generated image of a serene desert scene in the morning:++++The scene features a camel standing on a sand dune, palm trees, and an oasis+in the background. The sun is rising, casting a soft golden light over the+landscape with clear skies and a few scattered clouds.+===============================================================================+"""diff --git a/examples/vision/multi_turn_image_refining.py b/examples/vision/multi_turn_image_refining.pynew file mode 100644index 000000000..e6a380ebb--- /dev/null+++ b/examples/vision/multi_turn_image_refining.py@@ -0,0 +1,235 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import re+from copy import deepcopy++from colorama import Fore+from PIL import Image++from camel.agents import ChatAgent+from camel.configs import ChatGPTConfig+from camel.messages import BaseMessage+from camel.models import ModelFactory+from camel.prompts import PromptTemplateGenerator+from camel.responses import ChatAgentResponse+from camel.toolkits import DALLE_FUNCS+from camel.types import (+ ModelPlatformType,+ ModelType,+ RoleType,+ TaskType,+)+from camel.utils import print_text_animated+++class MMChat:+ r"""The class of multimodal chat session.++ NOTE: Currently this example doesn't work properly, since the generated+ image is not included in the response message. Need to add support to+ include Image in response message.+ """++ def __init__(+ self,+ ) -> None:+ self.critic = None+ self.artist = None+ critic_sys = """You need to describe what you see in the figure+and improve the prompt of it.+Reply with the following format:++CRITICS: the image needs to improve...+PROMPT: here is the updated prompt!+ """+ self.critic_sys_msg = BaseMessage.make_assistant_message(+ role_name='critic', content=critic_sys+ )++ self.artist_sys_msg = BaseMessage.make_assistant_message(+ role_name="Artist",+ content=PromptTemplateGenerator().get_prompt_from_key(+ TaskType.MULTI_CONDITION_IMAGE_CRAFT, RoleType.ASSISTANT+ ),+ )++ self.init_agents()++ def init_agents(self):+ r"""Initialize artist and critic agents with their system messages."""++ model_config = ChatGPTConfig(tools=[*DALLE_FUNCS])++ model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.GPT_4O_MINI,+ model_config_dict=model_config.__dict__,+ )++ self.artist = ChatAgent(+ system_message=self.artist_sys_msg,+ model=model,+ tools=DALLE_FUNCS,+ )++ self.artist.reset()++ self.critic = ChatAgent(+ system_message=self.critic_sys_msg, model=model+ )+ self.critic.reset()++ def step(self, initialPrompt: str, iter_num=2) -> ChatAgentResponse:+ r"""Process of the drawing and criticising.++ Returns:+ ChatAgentResponse: it contains the response message of+ the artist agent in the last iteration.++ """++ artist_user_msg = BaseMessage.make_user_message(+ role_name="User", content=initialPrompt+ )+ print(+ Fore.MAGENTA+ + "=" * 10+ + "ARTIST SYS"+ + "=" * 10+ + "\n"+ + self.artist_sys_msg.content+ )+ print(+ Fore.YELLOW+ + "=" * 10+ + "ARTIST USR"+ + "=" * 10+ + "\n"+ + artist_user_msg.content+ )++ pattern = r'''\(.*?/([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-+ [0-9a-fA-F]{4}-[0-9a-fA-F]{12})(\.jpg|\.png)\)'''+ response = self.artist.step(artist_user_msg)+ matches = re.findall(pattern, response.msg.content)++ image_paths = [f"./img/{uuid}{ext}" for uuid, ext in matches]+ tmp_paths = deepcopy(image_paths)+ response_msg = re.sub(+ pattern,+ lambda x: "(" + image_paths.pop(0) + ")",+ response.msg.content,+ )+ image_paths = deepcopy(tmp_paths)++ print_text_animated(+ Fore.BLUE+ + "=" * 10+ + "ARTIST RES"+ + "=" * 10+ + "\n"+ + response_msg+ )+ print(response_msg)++ i = 0+ while i < iter_num:+ i += 1+ # Resize the image to 128x128+ resized_imgs = [+ Image.open(image_path).resize(+ (128, 128), Image.Resampling.LANCZOS+ )+ for image_path in image_paths+ ]+ # Save for maintaining the image format+ [+ img.save(f"tmp_{i}.png", "PNG")+ for i, img in enumerate(resized_imgs)+ ]+ saved = [f"tmp_{i}.png" for i in range(len(resized_imgs))]+ image_list = [Image.open(image) for image in saved]++ critic_user_msg = BaseMessage.make_user_message(+ role_name="User",+ content="image:",+ image_list=image_list,+ image_detail="low",+ )+ print(+ Fore.GREEN+ + "=" * 10+ + "CRITIC SYS"+ + "=" * 10+ + "\n"+ + self.critic_sys_msg.content+ )+ print(+ Fore.RED+ + "=" * 10+ + "CRITIC USR"+ + "=" * 10+ + "\n"+ + critic_user_msg.content+ )+ prompt = self.critic.step(critic_user_msg).msg.content+ print_text_animated(+ Fore.CYAN+ + "=" * 10+ + "CRITIC RES"+ + "=" * 10+ + "\n"+ + prompt+ + Fore.RESET+ )++ artist_user_msg = BaseMessage.make_user_message(+ role_name="User",+ content='''Please generate a image based on+ the following prompt: \n'''+ + prompt,+ )+ response = self.artist.step(artist_user_msg)++ matches = re.findall(pattern, response.msg.content)+ image_paths = [f"./img/{uuid}{ext}" for uuid, ext in matches]+ tmp_paths = deepcopy(image_paths)+ response_msg = re.sub(+ pattern,+ lambda x, image_paths=image_paths: "("+ + image_paths.pop(0)+ + ")",+ response.msg.content,+ )+ image_paths = deepcopy(tmp_paths)+ print_text_animated(+ Fore.BLUE+ + "=" * 10+ + "ARTIST RES"+ + "=" * 10+ + "\n"+ + response_msg+ )+ print(response_msg)++ return response+++if __name__ == "__main__":+ session = MMChat()+ res = session.step(+ initialPrompt='''Create an image with pink background,+ a dog is showing a sign with 'I Love Camel'.''',+ iter_num=1,+ )diff --git a/examples/vision/object_recognition.py b/examples/vision/object_recognition.pyindex ce6832ceb..73676a038 100644--- a/examples/vision/object_recognition.py+++ b/examples/vision/object_recognition.py@@ -20,18 +20,26 @@ from camel.configs.openai_config import ChatGPTConfigfrom camel.generators import PromptTemplateGeneratorfrom camel.messages import BaseMessagefrom camel.models import ModelFactory-from camel.types import ModelPlatformType, ModelType, RoleType, TaskType+from camel.types import (+ ModelPlatformType,+ ModelType,+ RoleType,+ TaskType,+)parser = argparse.ArgumentParser(description="Arguments for object detection.")parser.add_argument(- "--image_path",+ "--image_paths",+ metavar='N',type=str,- help="Path to the image for object detection.",+ nargs='+',+ help="Path to the images for object detection.",+ default=None,required=True,)-def detect_image_obj(image_path: str) -> None:+def detect_image_obj(image_paths: str) -> None:sys_msg = PromptTemplateGenerator().get_prompt_from_key(TaskType.OBJECT_RECOGNITION, RoleType.ASSISTANT)@@ -52,7 +60,8 @@ def detect_image_obj(image_path: str) -> None:assistant_sys_msg,model=model,)- image_list = [Image.open(image_path)]+ image_list = [Image.open(image_path) for image_path in image_paths]+user_msg = BaseMessage.make_user_message(role_name="User",content="Please start the object detection for following image!",@@ -66,7 +75,7 @@ def detect_image_obj(image_path: str) -> None:def main(args: argparse.Namespace) -> None:- detect_image_obj(args.image_path)+ detect_image_obj(args.image_paths)if __name__ == "__main__":diff --git a/test/toolkits/test_dalle_functions.py b/test/toolkits/test_dalle_functions.pynew file mode 100644index 000000000..1e18973bd--- /dev/null+++ b/test/toolkits/test_dalle_functions.py@@ -0,0 +1,94 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the āLicenseā);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an āAS ISā BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import base64+import os++import pytest+from PIL import Image++from camel.toolkits import DalleToolkit+++def dalle_toolkit():+ return DalleToolkit()+++def test_image_path():+ # Prepare the path for the test image+ test_image_path = 'test_image.png'+ # Create a test image file (or ensure it exists for the test)+ with open(test_image_path, 'wb') as f:+ f.write(b'This is a test image placeholder content.')++ # Provide the path to the temp file for the test to use+ yield test_image_path++ # Teardown: Clean up the temporary file+ try:+ os.remove(test_image_path)+ except OSError as e:+ print(f"Error: {e.strerror}")+++def test_base64_to_image_valid(dalle_toolkit):+ valid_base64_string = "iVBORw0KGgoAAAANSUhEUgAAAAUA\+ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO\+ 9TXL0Y4OHwAAAABJRU5ErkJggg=="+ if valid_base64_string.startswith('data:image/png;base64,'):+ valid_base64_string = valid_base64_string[22:]++ image = dalle_toolkit.base64_to_image(valid_base64_string)++ assert isinstance(+ image, Image.Image+ ), "The function should return a PIL Image object"+++def test_base64_to_image_invalid(dalle_toolkit):+ invalid_base64_string = "invalid_base64_string"++ image = dalle_toolkit.base64_to_image(invalid_base64_string)++ # Check response is None+ assert (+ image is None+ ), "The function should return None for an invalid base64 string"+++def test_image_path_to_base64(test_image_path, dalle_toolkit):+ # Obtain the Base64 encoded string of the test image+ base64_encoded_string = dalle_toolkit.image_path_to_base64(test_image_path)+ # Decode the Base64 string back to binary data for validation+ decoded_binary_data = base64.b64decode(base64_encoded_string)+ # Verify if the decoded data matches the original binary content of+ # the test image+ with open(test_image_path, 'rb') as f:+ original_binary_data = f.read()++ assert (+ decoded_binary_data == original_binary_data+ ), "The Base64 encoded string does not match the original image content."+++def test_image_to_base64_with_invalid_input(dalle_toolkit):+ # Intentionally pass a non-image object to simulate an error condition+ invalid_input = "This is not an image"++ # Expect the function to return None when an error occurs during encoding+ result = dalle_toolkit.image_to_base64(invalid_input)++ # Assert that the function correctly returns None for invalid inputs+ assert result == ""