Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Support multi-modal input and multi-modal output in one agent Dalle support (#698)

Will Shang06f87e6049e7
diff --git a/camel/messages/base.py b/camel/messages/base.py
index 126c27171..f58895250 100644
--- a/camel/messages/base.py
+++ b/camel/messages/base.py
@@ -60,6 +60,7 @@ class BaseMessage:
role_type: RoleType
meta_dict: Optional[Dict[str, str]]
content: str
+
video_bytes: Optional[bytes] = None
image_list: Optional[List[Image.Image]] = None
image_detail: Literal["auto", "low", "high"] = "auto"
diff --git a/camel/prompts/__init__.py b/camel/prompts/__init__.py
index b3754d8e4..5bf39c7f1 100644
--- a/camel/prompts/__init__.py
+++ b/camel/prompts/__init__.py
@@ -18,7 +18,11 @@ from .evaluation import EvaluationPromptTemplateDict
from .generate_text_embedding_data import (
GenerateTextEmbeddingDataPromptTemplateDict,
)
+from .image_craft import ImageCraftPromptTemplateDict
from .misalignment import MisalignmentPromptTemplateDict
+from .multi_condition_image_craft import (
+ MultiConditionImageCraftPromptTemplateDict,
+)
from .object_recognition import ObjectRecognitionPromptTemplateDict
from .prompt_templates import PromptTemplateGenerator
from .role_description_prompt_template import RoleDescriptionPromptTemplateDict
@@ -42,5 +46,8 @@ __all__ = [
'SolutionExtractionPromptTemplateDict',
'GenerateTextEmbeddingDataPromptTemplateDict',
'ObjectRecognitionPromptTemplateDict',
+ 'ImageCraftPromptTemplateDict',
+ 'MultiConditionImageCraftPromptTemplateDict',
+ 'DescriptionVideoPromptTemplateDict',
'VideoDescriptionPromptTemplateDict',
]
diff --git a/camel/prompts/image_craft.py b/camel/prompts/image_craft.py
new file mode 100644
index 000000000..0c7fb3deb
--- /dev/null
+++ b/camel/prompts/image_craft.py
@@ -0,0 +1,34 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import Any
+
+from camel.prompts import TextPrompt, TextPromptDict
+from camel.types import RoleType
+
+
+class ImageCraftPromptTemplateDict(TextPromptDict):
+ ASSISTANT_PROMPT = TextPrompt(
+ """You are tasked with creating an original image based on
+ the provided descriptive captions. Use your imagination
+ and artistic skills to visualize and draw the images and
+ explain your thought process."""
+ )
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self.update(
+ {
+ RoleType.ASSISTANT: self.ASSISTANT_PROMPT,
+ }
+ )
diff --git a/camel/prompts/multi_condition_image_craft.py b/camel/prompts/multi_condition_image_craft.py
new file mode 100644
index 000000000..7b017b01b
--- /dev/null
+++ b/camel/prompts/multi_condition_image_craft.py
@@ -0,0 +1,34 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import Any
+
+from camel.prompts import TextPrompt, TextPromptDict
+from camel.types import RoleType
+
+
+class MultiConditionImageCraftPromptTemplateDict(TextPromptDict):
+ ASSISTANT_PROMPT = TextPrompt(
+ """You are tasked with creating an image based on
+ the provided text and images conditions. Please use your
+ imagination and artistic capabilities to visualize and
+ draw the images and explain what you are thinking about."""
+ )
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self.update(
+ {
+ RoleType.ASSISTANT: self.ASSISTANT_PROMPT,
+ }
+ )
diff --git a/camel/prompts/task_prompt_template.py b/camel/prompts/task_prompt_template.py
index 4e7df06dd..6662697bb 100644
--- a/camel/prompts/task_prompt_template.py
+++ b/camel/prompts/task_prompt_template.py
@@ -24,7 +24,11 @@ from camel.prompts.evaluation import (
from camel.prompts.generate_text_embedding_data import (
GenerateTextEmbeddingDataPromptTemplateDict,
)
+from camel.prompts.image_craft import ImageCraftPromptTemplateDict
from camel.prompts.misalignment import MisalignmentPromptTemplateDict
+from camel.prompts.multi_condition_image_craft import (
+ MultiConditionImageCraftPromptTemplateDict,
+)
from camel.prompts.object_recognition import (
ObjectRecognitionPromptTemplateDict,
)
@@ -64,6 +68,8 @@ class TaskPromptTemplateDict(Dict[Any, TextPromptDict]):
TaskType.ROLE_DESCRIPTION: RoleDescriptionPromptTemplateDict(),
TaskType.OBJECT_RECOGNITION: ObjectRecognitionPromptTemplateDict(), # noqa: E501
TaskType.GENERATE_TEXT_EMBEDDING_DATA: GenerateTextEmbeddingDataPromptTemplateDict(), # noqa: E501
+ TaskType.IMAGE_CRAFT: ImageCraftPromptTemplateDict(),
+ TaskType.MULTI_CONDITION_IMAGE_CRAFT: MultiConditionImageCraftPromptTemplateDict(), # noqa: E501
TaskType.VIDEO_DESCRIPTION: VideoDescriptionPromptTemplateDict(), # noqa: E501
}
)
diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.py
index 6eb50d2e2..72432a7f6 100644
--- a/camel/toolkits/__init__.py
+++ b/camel/toolkits/__init__.py
@@ -27,6 +27,7 @@ from .search_toolkit import SEARCH_FUNCS, SearchToolkit
from .twitter_toolkit import TWITTER_FUNCS, TwitterToolkit
from .weather_toolkit import WEATHER_FUNCS, WeatherToolkit
from .slack_toolkit import SLACK_FUNCS, SlackToolkit
+from .dalle_toolkit import DALLE_FUNCS, DalleToolkit
from .base import BaseToolkit
from .code_execution import CodeExecutionToolkit
@@ -45,12 +46,14 @@ __all__ = [
'TWITTER_FUNCS',
'WEATHER_FUNCS',
'SLACK_FUNCS',
+ 'DALLE_FUNCS',
'BaseToolkit',
'GithubToolkit',
'MathToolkit',
'GoogleMapsToolkit',
'SearchToolkit',
'SlackToolkit',
+ 'DalleToolkit',
'TwitterToolkit',
'WeatherToolkit',
'RetrievalToolkit',
diff --git a/camel/toolkits/dalle_toolkit.py b/camel/toolkits/dalle_toolkit.py
new file mode 100644
index 000000000..4898f0d69
--- /dev/null
+++ b/camel/toolkits/dalle_toolkit.py
@@ -0,0 +1,146 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import base64
+import os
+import uuid
+from io import BytesIO
+from typing import List, Optional
+
+from openai import OpenAI
+from PIL import Image
+
+from camel.toolkits import OpenAIFunction
+from camel.toolkits.base import BaseToolkit
+
+
+class DalleToolkit(BaseToolkit):
+ r"""A class representing a toolkit for image generation using OpenAI's.
+
+ This class provides methods handle image generation using OpenAI's DALL-E.
+ """
+
+ def base64_to_image(self, base64_string: str) -> Optional[Image.Image]:
+ r"""Converts a base64 encoded string into a PIL Image object.
+
+ Args:
+ base64_string (str): The base64 encoded string of the image.
+
+ Returns:
+ Optional[Image.Image]: The PIL Image object or None if conversion
+ fails.
+ """
+ try:
+ # Decode the base64 string to get the image data
+ image_data = base64.b64decode(base64_string)
+ # Create a memory buffer for the image data
+ image_buffer = BytesIO(image_data)
+ # Open the image using the PIL library
+ image = Image.open(image_buffer)
+ return image
+ except Exception as e:
+ print(f"An error occurred while converting base64 to image: {e}")
+ return None
+
+ def image_path_to_base64(self, image_path: str) -> str:
+ r"""Converts the file path of an image to a Base64 encoded string.
+
+ Args:
+ image_path (str): The path to the image file.
+
+ Returns:
+ str: A Base64 encoded string representing the content of the image
+ file.
+ """
+ try:
+ with open(image_path, "rb") as image_file:
+ return base64.b64encode(image_file.read()).decode('utf-8')
+ except Exception as e:
+ print(
+ f"An error occurred while converting image path to base64: {e}"
+ )
+ return ""
+
+ def image_to_base64(self, image: Image.Image) -> str:
+ r"""Converts an image into a base64-encoded string.
+
+ This function takes an image object as input, encodes the image into a
+ PNG format base64 string, and returns it.
+ If the encoding process encounters an error, it prints the error
+ message and returns None.
+
+ Args:
+ image: The image object to be encoded, supports any image format
+ that can be saved in PNG format.
+
+ Returns:
+ str: A base64-encoded string of the image.
+ """
+ try:
+ with BytesIO() as buffered_image:
+ image.save(buffered_image, format="PNG")
+ buffered_image.seek(0)
+ image_bytes = buffered_image.read()
+ base64_str = base64.b64encode(image_bytes).decode('utf-8')
+ return base64_str
+ except Exception as e:
+ print(f"An error occurred: {e}")
+ return ""
+
+ def get_dalle_img(self, prompt: str, image_dir: str = "img") -> str:
+ r"""Generate an image using OpenAI's DALL-E model.
+ The generated image is saved to the specified directory.
+
+ Args:
+ prompt (str): The text prompt based on which the image is
+ generated.
+ image_dir (str): The directory to save the generated image.
+ Defaults to 'img'.
+
+ Returns:
+ str: The path to the saved image.
+ """
+
+ dalle_client = OpenAI()
+ response = dalle_client.images.generate(
+ model="dall-e-3",
+ prompt=prompt,
+ size="1024x1792",
+ quality="standard",
+ n=1, # NOTE: now dall-e-3 only supports n=1
+ response_format="b64_json",
+ )
+ image_b64 = response.data[0].b64_json
+ image = self.base64_to_image(image_b64) # type: ignore[arg-type]
+
+ if image is None:
+ raise ValueError("Failed to convert base64 string to image.")
+
+ os.makedirs(image_dir, exist_ok=True)
+ image_path = os.path.join(image_dir, f"{uuid.uuid4()}.png")
+ image.save(image_path)
+
+ return image_path
+
+ def get_tools(self) -> List[OpenAIFunction]:
+ r"""Returns a list of OpenAIFunction objects representing the
+ functions in the toolkit.
+
+ Returns:
+ List[OpenAIFunction]: A list of OpenAIFunction objects
+ representing the functions in the toolkit.
+ """
+ return [OpenAIFunction(self.get_dalle_img)]
+
+
+DALLE_FUNCS: List[OpenAIFunction] = DalleToolkit().get_tools()
diff --git a/camel/types/enums.py b/camel/types/enums.py
index 840eff0cc..0cbdef041 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -361,6 +361,8 @@ class TaskType(Enum):
ROLE_DESCRIPTION = "role_description"
GENERATE_TEXT_EMBEDDING_DATA = "generate_text_embedding_data"
OBJECT_RECOGNITION = "object_recognition"
+ IMAGE_CRAFT = "image_craft"
+ MULTI_CONDITION_IMAGE_CRAFT = "multi_condition_image_craft"
DEFAULT = "default"
VIDEO_DESCRIPTION = "video_description"
diff --git a/examples/vision/image_crafting.py b/examples/vision/image_crafting.py
new file mode 100644
index 000000000..c6c66e8c9
--- /dev/null
+++ b/examples/vision/image_crafting.py
@@ -0,0 +1,88 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from camel.agents.chat_agent import ChatAgent
+from camel.configs import ChatGPTConfig
+from camel.messages.base import BaseMessage
+from camel.models import ModelFactory
+from camel.prompts import PromptTemplateGenerator
+from camel.toolkits import DALLE_FUNCS
+from camel.types import (
+ ModelPlatformType,
+ ModelType,
+ RoleType,
+ TaskType,
+)
+
+
+def main():
+ sys_msg = PromptTemplateGenerator().get_prompt_from_key(
+ TaskType.IMAGE_CRAFT, RoleType.ASSISTANT
+ )
+ print("=" * 20 + " SYS MSG " + "=" * 20)
+ print(sys_msg)
+ print("=" * 49)
+
+ assistant_sys_msg = BaseMessage.make_assistant_message(
+ role_name="Artist",
+ content=sys_msg,
+ )
+
+ user_msg = BaseMessage.make_user_message(
+ role_name="User",
+ content="Draw a picture of a camel.",
+ )
+
+ model_config = ChatGPTConfig(tools=[*DALLE_FUNCS])
+
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ model_config_dict=model_config.__dict__,
+ )
+
+ dalle_agent = ChatAgent(
+ system_message=assistant_sys_msg,
+ model=model,
+ tools=DALLE_FUNCS,
+ )
+
+ response = dalle_agent.step(user_msg)
+
+ print("=" * 20 + " RESULT " + "=" * 20)
+ print(response.msg.content)
+ print("=" * 48)
+
+
+if __name__ == "__main__":
+ main()
+
+"""
+===============================================================================
+==================== SYS MSG ====================
+You are tasked with creating an original image based on
+ the provided descriptive captions. Use your imagination
+ and artistic skills to visualize and draw the images and
+ explain your thought process.
+=================================================
+==================== RESULT ====================
+I have created an image of a camel standing in a desert oasis under the shade
+of a palm tree. You can see the realistic and detailed drawing of the camel in
+the image below.
+
+![Camel in a Desert Oasis](img/58a2a3fa-1e7e-407c-8cd6-4b99448b6a90.png)
+
+The scene captures the essence of the desert environment with the camel
+peacefully resting in the oasis.
+===============================================================================
+"""
diff --git a/examples/vision/multi_condition_image_crafting.py b/examples/vision/multi_condition_image_crafting.py
new file mode 100644
index 000000000..4eca5fdfa
--- /dev/null
+++ b/examples/vision/multi_condition_image_crafting.py
@@ -0,0 +1,93 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from PIL import Image
+
+from camel.agents.chat_agent import ChatAgent
+from camel.configs import ChatGPTConfig
+from camel.generators import PromptTemplateGenerator
+from camel.messages.base import BaseMessage
+from camel.models import ModelFactory
+from camel.toolkits import DALLE_FUNCS
+from camel.types import (
+ ModelPlatformType,
+ ModelType,
+ RoleType,
+ TaskType,
+)
+
+
+def main(image_paths: list[str]) -> list[str]:
+ sys_msg = PromptTemplateGenerator().get_prompt_from_key(
+ TaskType.MULTI_CONDITION_IMAGE_CRAFT, RoleType.ASSISTANT
+ )
+ print("=" * 20 + " SYS MSG " + "=" * 20)
+ print(sys_msg)
+ print("=" * 49)
+
+ assistant_sys_msg = BaseMessage.make_assistant_message(
+ role_name="Artist",
+ content=sys_msg,
+ )
+
+ model_config = ChatGPTConfig(tools=[*DALLE_FUNCS])
+
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ model_config_dict=model_config.__dict__,
+ )
+
+ dalle_agent = ChatAgent(
+ system_message=assistant_sys_msg,
+ model=model,
+ tools=DALLE_FUNCS,
+ )
+
+ image_list = [Image.open(image_path) for image_path in image_paths]
+
+ user_msg = BaseMessage.make_user_message(
+ role_name="User",
+ content='''Please generate an image based on the provided images and
+ text, make the backgroup of this image is in the morning''',
+ image_list=image_list,
+ )
+
+ response = dalle_agent.step(user_msg)
+
+ print("=" * 20 + " RESULT " + "=" * 20)
+ print(response.msg.content)
+ print("=" * 48)
+
+
+if __name__ == "__main__":
+ main()
+
+"""
+===============================================================================
+==================== SYS MSG ====================
+You are tasked with creating an image based on
+ the provided text and images conditions. Please use your
+ imagination and artistic capabilities to visualize and
+ draw the images and explain what you are thinking about.
+=================================================
+==================== RESULT ====================
+Here is the generated image of a serene desert scene in the morning:
+
+![Morning Desert Scene](img/3d8310e8-9f14-48be-94db-c66dd0461cd0.png)
+
+The scene features a camel standing on a sand dune, palm trees, and an oasis
+in the background. The sun is rising, casting a soft golden light over the
+landscape with clear skies and a few scattered clouds.
+===============================================================================
+"""
diff --git a/examples/vision/multi_turn_image_refining.py b/examples/vision/multi_turn_image_refining.py
new file mode 100644
index 000000000..e6a380ebb
--- /dev/null
+++ b/examples/vision/multi_turn_image_refining.py
@@ -0,0 +1,235 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import re
+from copy import deepcopy
+
+from colorama import Fore
+from PIL import Image
+
+from camel.agents import ChatAgent
+from camel.configs import ChatGPTConfig
+from camel.messages import BaseMessage
+from camel.models import ModelFactory
+from camel.prompts import PromptTemplateGenerator
+from camel.responses import ChatAgentResponse
+from camel.toolkits import DALLE_FUNCS
+from camel.types import (
+ ModelPlatformType,
+ ModelType,
+ RoleType,
+ TaskType,
+)
+from camel.utils import print_text_animated
+
+
+class MMChat:
+ r"""The class of multimodal chat session.
+
+ NOTE: Currently this example doesn't work properly, since the generated
+ image is not included in the response message. Need to add support to
+ include Image in response message.
+ """
+
+ def __init__(
+ self,
+ ) -> None:
+ self.critic = None
+ self.artist = None
+ critic_sys = """You need to describe what you see in the figure
+and improve the prompt of it.
+Reply with the following format:
+
+CRITICS: the image needs to improve...
+PROMPT: here is the updated prompt!
+ """
+ self.critic_sys_msg = BaseMessage.make_assistant_message(
+ role_name='critic', content=critic_sys
+ )
+
+ self.artist_sys_msg = BaseMessage.make_assistant_message(
+ role_name="Artist",
+ content=PromptTemplateGenerator().get_prompt_from_key(
+ TaskType.MULTI_CONDITION_IMAGE_CRAFT, RoleType.ASSISTANT
+ ),
+ )
+
+ self.init_agents()
+
+ def init_agents(self):
+ r"""Initialize artist and critic agents with their system messages."""
+
+ model_config = ChatGPTConfig(tools=[*DALLE_FUNCS])
+
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ model_config_dict=model_config.__dict__,
+ )
+
+ self.artist = ChatAgent(
+ system_message=self.artist_sys_msg,
+ model=model,
+ tools=DALLE_FUNCS,
+ )
+
+ self.artist.reset()
+
+ self.critic = ChatAgent(
+ system_message=self.critic_sys_msg, model=model
+ )
+ self.critic.reset()
+
+ def step(self, initialPrompt: str, iter_num=2) -> ChatAgentResponse:
+ r"""Process of the drawing and criticising.
+
+ Returns:
+ ChatAgentResponse: it contains the response message of
+ the artist agent in the last iteration.
+
+ """
+
+ artist_user_msg = BaseMessage.make_user_message(
+ role_name="User", content=initialPrompt
+ )
+ print(
+ Fore.MAGENTA
+ + "=" * 10
+ + "ARTIST SYS"
+ + "=" * 10
+ + "\n"
+ + self.artist_sys_msg.content
+ )
+ print(
+ Fore.YELLOW
+ + "=" * 10
+ + "ARTIST USR"
+ + "=" * 10
+ + "\n"
+ + artist_user_msg.content
+ )
+
+ pattern = r'''\(.*?/([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-
+ [0-9a-fA-F]{4}-[0-9a-fA-F]{12})(\.jpg|\.png)\)'''
+ response = self.artist.step(artist_user_msg)
+ matches = re.findall(pattern, response.msg.content)
+
+ image_paths = [f"./img/{uuid}{ext}" for uuid, ext in matches]
+ tmp_paths = deepcopy(image_paths)
+ response_msg = re.sub(
+ pattern,
+ lambda x: "(" + image_paths.pop(0) + ")",
+ response.msg.content,
+ )
+ image_paths = deepcopy(tmp_paths)
+
+ print_text_animated(
+ Fore.BLUE
+ + "=" * 10
+ + "ARTIST RES"
+ + "=" * 10
+ + "\n"
+ + response_msg
+ )
+ print(response_msg)
+
+ i = 0
+ while i < iter_num:
+ i += 1
+ # Resize the image to 128x128
+ resized_imgs = [
+ Image.open(image_path).resize(
+ (128, 128), Image.Resampling.LANCZOS
+ )
+ for image_path in image_paths
+ ]
+ # Save for maintaining the image format
+ [
+ img.save(f"tmp_{i}.png", "PNG")
+ for i, img in enumerate(resized_imgs)
+ ]
+ saved = [f"tmp_{i}.png" for i in range(len(resized_imgs))]
+ image_list = [Image.open(image) for image in saved]
+
+ critic_user_msg = BaseMessage.make_user_message(
+ role_name="User",
+ content="image:",
+ image_list=image_list,
+ image_detail="low",
+ )
+ print(
+ Fore.GREEN
+ + "=" * 10
+ + "CRITIC SYS"
+ + "=" * 10
+ + "\n"
+ + self.critic_sys_msg.content
+ )
+ print(
+ Fore.RED
+ + "=" * 10
+ + "CRITIC USR"
+ + "=" * 10
+ + "\n"
+ + critic_user_msg.content
+ )
+ prompt = self.critic.step(critic_user_msg).msg.content
+ print_text_animated(
+ Fore.CYAN
+ + "=" * 10
+ + "CRITIC RES"
+ + "=" * 10
+ + "\n"
+ + prompt
+ + Fore.RESET
+ )
+
+ artist_user_msg = BaseMessage.make_user_message(
+ role_name="User",
+ content='''Please generate a image based on
+ the following prompt: \n'''
+ + prompt,
+ )
+ response = self.artist.step(artist_user_msg)
+
+ matches = re.findall(pattern, response.msg.content)
+ image_paths = [f"./img/{uuid}{ext}" for uuid, ext in matches]
+ tmp_paths = deepcopy(image_paths)
+ response_msg = re.sub(
+ pattern,
+ lambda x, image_paths=image_paths: "("
+ + image_paths.pop(0)
+ + ")",
+ response.msg.content,
+ )
+ image_paths = deepcopy(tmp_paths)
+ print_text_animated(
+ Fore.BLUE
+ + "=" * 10
+ + "ARTIST RES"
+ + "=" * 10
+ + "\n"
+ + response_msg
+ )
+ print(response_msg)
+
+ return response
+
+
+if __name__ == "__main__":
+ session = MMChat()
+ res = session.step(
+ initialPrompt='''Create an image with pink background,
+ a dog is showing a sign with 'I Love Camel'.''',
+ iter_num=1,
+ )
diff --git a/examples/vision/object_recognition.py b/examples/vision/object_recognition.py
index ce6832ceb..73676a038 100644
--- a/examples/vision/object_recognition.py
+++ b/examples/vision/object_recognition.py
@@ -20,18 +20,26 @@ from camel.configs.openai_config import ChatGPTConfig
from camel.generators import PromptTemplateGenerator
from camel.messages import BaseMessage
from camel.models import ModelFactory
-from camel.types import ModelPlatformType, ModelType, RoleType, TaskType
+from camel.types import (
+ ModelPlatformType,
+ ModelType,
+ RoleType,
+ TaskType,
+)
parser = argparse.ArgumentParser(description="Arguments for object detection.")
parser.add_argument(
- "--image_path",
+ "--image_paths",
+ metavar='N',
type=str,
- help="Path to the image for object detection.",
+ nargs='+',
+ help="Path to the images for object detection.",
+ default=None,
required=True,
)
-def detect_image_obj(image_path: str) -> None:
+def detect_image_obj(image_paths: str) -> None:
sys_msg = PromptTemplateGenerator().get_prompt_from_key(
TaskType.OBJECT_RECOGNITION, RoleType.ASSISTANT
)
@@ -52,7 +60,8 @@ def detect_image_obj(image_path: str) -> None:
assistant_sys_msg,
model=model,
)
- image_list = [Image.open(image_path)]
+ image_list = [Image.open(image_path) for image_path in image_paths]
+
user_msg = BaseMessage.make_user_message(
role_name="User",
content="Please start the object detection for following image!",
@@ -66,7 +75,7 @@ def detect_image_obj(image_path: str) -> None:
def main(args: argparse.Namespace) -> None:
- detect_image_obj(args.image_path)
+ detect_image_obj(args.image_paths)
if __name__ == "__main__":
diff --git a/test/toolkits/test_dalle_functions.py b/test/toolkits/test_dalle_functions.py
new file mode 100644
index 000000000..1e18973bd
--- /dev/null
+++ b/test/toolkits/test_dalle_functions.py
@@ -0,0 +1,94 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import base64
+import os
+
+import pytest
+from PIL import Image
+
+from camel.toolkits import DalleToolkit
+
+
+def dalle_toolkit():
+ return DalleToolkit()
+
+
+def test_image_path():
+ # Prepare the path for the test image
+ test_image_path = 'test_image.png'
+ # Create a test image file (or ensure it exists for the test)
+ with open(test_image_path, 'wb') as f:
+ f.write(b'This is a test image placeholder content.')
+
+ # Provide the path to the temp file for the test to use
+ yield test_image_path
+
+ # Teardown: Clean up the temporary file
+ try:
+ os.remove(test_image_path)
+ except OSError as e:
+ print(f"Error: {e.strerror}")
+
+
+def test_base64_to_image_valid(dalle_toolkit):
+ valid_base64_string = "iVBORw0KGgoAAAANSUhEUgAAAAUA\
+ AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO\
+ 9TXL0Y4OHwAAAABJRU5ErkJggg=="
+ if valid_base64_string.startswith('data:image/png;base64,'):
+ valid_base64_string = valid_base64_string[22:]
+
+ image = dalle_toolkit.base64_to_image(valid_base64_string)
+
+ assert isinstance(
+ image, Image.Image
+ ), "The function should return a PIL Image object"
+
+
+def test_base64_to_image_invalid(dalle_toolkit):
+ invalid_base64_string = "invalid_base64_string"
+
+ image = dalle_toolkit.base64_to_image(invalid_base64_string)
+
+ # Check response is None
+ assert (
+ image is None
+ ), "The function should return None for an invalid base64 string"
+
+
+def test_image_path_to_base64(test_image_path, dalle_toolkit):
+ # Obtain the Base64 encoded string of the test image
+ base64_encoded_string = dalle_toolkit.image_path_to_base64(test_image_path)
+ # Decode the Base64 string back to binary data for validation
+ decoded_binary_data = base64.b64decode(base64_encoded_string)
+ # Verify if the decoded data matches the original binary content of
+ # the test image
+ with open(test_image_path, 'rb') as f:
+ original_binary_data = f.read()
+
+ assert (
+ decoded_binary_data == original_binary_data
+ ), "The Base64 encoded string does not match the original image content."
+
+
+def test_image_to_base64_with_invalid_input(dalle_toolkit):
+ # Intentionally pass a non-image object to simulate an error condition
+ invalid_input = "This is not an image"
+
+ # Expect the function to return None when an error occurs during encoding
+ result = dalle_toolkit.image_to_base64(invalid_input)
+
+ # Assert that the function correctly returns None for invalid inputs
+ assert result == ""