Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: enable image modality for `ChatAgent` (#473)

Zecheng Zhang57c700ada0de
diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.py
index 96409b76d..792679776 100644
--- a/camel/agents/chat_agent.py
+++ b/camel/agents/chat_agent.py
@@ -19,7 +19,7 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
from camel.agents import BaseAgent
-from camel.configs import ChatGPTConfig
+from camel.configs import ChatGPTConfig, ChatGPTVisionConfig
from camel.memories import (
AgentMemory,
ChatHistoryMemory,
@@ -128,7 +128,22 @@ class ChatAgent(BaseAgent):
if function_list is not None:
for func in function_list:
self.func_dict[func.get_function_name()] = func.func
- self.model_config = model_config or ChatGPTConfig()
+
+ self.model_config: BaseConfig
+ if self.model_type == ModelType.GPT_4_TURBO_VISION:
+ if model_config is not None and \
+ not isinstance(model_config, ChatGPTVisionConfig):
+ raise ValueError("Please use `ChatGPTVisionConfig` as "
+ "the `model_config` when `model_type` "
+ "is `GPT_4_TURBO_VISION`")
+ self.model_config = model_config or ChatGPTVisionConfig()
+ else:
+ if model_config is not None and \
+ isinstance(model_config, ChatGPTVisionConfig):
+ raise ValueError("Please don't use `ChatGPTVisionConfig` as "
+ "the `model_config` when `model_type` "
+ "is not `GPT_4_TURBO_VISION`")
+ self.model_config = model_config or ChatGPTConfig()
self.model_backend: BaseModelBackend = ModelFactory.create(
self.model_type, self.model_config.__dict__)
diff --git a/camel/configs.py b/camel/configs.py
index b40ebb20c..39665c9fa 100644
--- a/camel/configs.py
+++ b/camel/configs.py
@@ -87,6 +87,55 @@ class ChatGPTConfig(BaseConfig):
user: str = ""
+@dataclass(frozen=True)
+class ChatGPTVisionConfig(BaseConfig):
+ r"""Defines the parameters for generating chat completions with
+ vision model using the OpenAI API. The vision config here is the
+ subset of the :class:`ChatGPTConfig`.
+
+ Args:
+ temperature (float, optional): Sampling temperature to use, between
+ :obj:`0` and :obj:`2`. Higher values make the output more random,
+ while lower values make it more focused and deterministic.
+ (default: :obj:`0.2`)
+ top_p (float, optional): An alternative to sampling with temperature,
+ called nucleus sampling, where the model considers the results of
+ the tokens with top_p probability mass. So :obj:`0.1` means only
+ the tokens comprising the top 10% probability mass are considered.
+ (default: :obj:`1.0`)
+ n (int, optional): How many chat completion choices to generate for
+ each input message. (default: :obj:`1`)
+ stream (bool, optional): If True, partial message deltas will be sent
+ as data-only server-sent events as they become available.
+ (default: :obj:`False`)
+ max_tokens (int, optional): The maximum number of tokens to generate
+ in the chat completion. The total length of input tokens and
+ generated tokens is limited by the model's context length.
+ (default: :obj:`4096`)
+ presence_penalty (float, optional): Number between :obj:`-2.0` and
+ :obj:`2.0`. Positive values penalize new tokens based on whether
+ they appear in the text so far, increasing the model's likelihood
+ to talk about new topics. See more information about frequency and
+ presence penalties. (default: :obj:`0.0`)
+ frequency_penalty (float, optional): Number between :obj:`-2.0` and
+ :obj:`2.0`. Positive values penalize new tokens based on their
+ existing frequency in the text so far, decreasing the model's
+ likelihood to repeat the same line verbatim. See more information
+ about frequency and presence penalties. (default: :obj:`0.0`)
+ user (str, optional): A unique identifier representing your end-user,
+ which can help OpenAI to monitor and detect abuse.
+ (default: :obj:`""`)
+ """
+ temperature: float = 0.2 # openai default: 1.0
+ top_p: float = 1.0
+ n: int = 1
+ stream: bool = False
+ max_tokens: int = 4096
+ presence_penalty: float = 0.0
+ frequency_penalty: float = 0.0
+ user: str = ""
+
+
@dataclass(frozen=True)
class FunctionCallingConfig(ChatGPTConfig):
r"""Defines the parameters for generating chat completions using the
diff --git a/camel/memories/context_creators/score_based.py b/camel/memories/context_creators/score_based.py
index b892311ed..f18323efe 100644
--- a/camel/memories/context_creators/score_based.py
+++ b/camel/memories/context_creators/score_based.py
@@ -103,7 +103,7 @@ class ScoreBasedContextCreator(BaseContextCreator):
context_units = sorted(context_units,
key=lambda unit: unit.record.score)
- # Remove least score messages until total token number is smaller
+ # Remove the least score messages until total token number is smaller
# than token limit
truncate_idx = None
for i, unit in enumerate(context_units):
diff --git a/camel/messages/base.py b/camel/messages/base.py
index df199217c..0f22142d6 100644
--- a/camel/messages/base.py
+++ b/camel/messages/base.py
@@ -11,8 +11,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import base64
+import io
from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from PIL import Image
from camel.messages import (
OpenAIAssistantMessage,
@@ -21,7 +25,12 @@ from camel.messages import (
OpenAIUserMessage,
)
from camel.prompts import CodePrompt, TextPrompt
-from camel.types import OpenAIBackendRole, RoleType
+from camel.types import (
+ OpenAIBackendRole,
+ OpenAIImageDetailType,
+ OpenAIImageType,
+ RoleType,
+)
@dataclass
@@ -40,18 +49,28 @@ class BaseMessage:
role_type: RoleType
meta_dict: Optional[Dict[str, str]]
content: str
+ image: Optional[Image.Image] = None
+ image_detail: Literal["auto", "low", "high"] = "auto"
@classmethod
def make_user_message(
- cls, role_name: str, content: str,
- meta_dict: Optional[Dict[str, str]] = None) -> 'BaseMessage':
- return cls(role_name, RoleType.USER, meta_dict, content)
+ cls, role_name: str, content: str,
+ meta_dict: Optional[Dict[str, str]] = None,
+ image: Optional[Image.Image] = None,
+ image_detail: Union[OpenAIImageDetailType,
+ str] = "auto") -> 'BaseMessage':
+ return cls(role_name, RoleType.USER, meta_dict, content, image,
+ OpenAIImageDetailType(image_detail).value)
@classmethod
def make_assistant_message(
- cls, role_name: str, content: str,
- meta_dict: Optional[Dict[str, str]] = None) -> 'BaseMessage':
- return cls(role_name, RoleType.ASSISTANT, meta_dict, content)
+ cls, role_name: str, content: str,
+ meta_dict: Optional[Dict[str, str]] = None,
+ image: Optional[Image.Image] = None,
+ image_detail: Union[OpenAIImageDetailType,
+ str] = "auto") -> 'BaseMessage':
+ return cls(role_name, RoleType.ASSISTANT, meta_dict, content, image,
+ OpenAIImageDetailType(image_detail).value)
def create_new_instance(self, content: str) -> "BaseMessage":
r"""Create a new instance of the :obj:`BaseMessage` with updated
@@ -198,7 +217,40 @@ class BaseMessage:
Returns:
OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
"""
- return {"role": "user", "content": self.content}
+ if self.image is None:
+ return {"role": "user", "content": self.content}
+ else:
+ #
+ if self.image.format is None:
+ raise ValueError(f"Image's `format` is `None`, please "
+ f"transform the `PIL.Image.Image` to one of "
+ f"following supported formats, such as "
+ f"{list(OpenAIImageType)}")
+
+ image_type: str = self.image.format.lower()
+ if image_type not in OpenAIImageType:
+ raise ValueError(f"Image type {self.image.format} "
+ f"is not supported by OpenAI vision model")
+ with io.BytesIO() as buffer:
+ self.image.save(fp=buffer, format=self.image.format)
+ encoded_image = base64.b64encode(
+ buffer.getvalue()).decode("utf-8")
+ image_prefix = f"data:image/{image_type};base64,"
+
+ return {
+ "role":
+ "user",
+ "content": [{
+ "type": "text",
+ "text": self.content,
+ }, {
+ "type": "image_url",
+ "image_url": {
+ "url": f"{image_prefix}{encoded_image}",
+ "detail": self.image_detail,
+ }
+ }],
+ }
def to_openai_assistant_message(self) -> OpenAIAssistantMessage:
r"""Converts the message to an :obj:`OpenAIAssistantMessage` object.
diff --git a/camel/prompts/__init__.py b/camel/prompts/__init__.py
index 49f9d6f35..8a75646f7 100644
--- a/camel/prompts/__init__.py
+++ b/camel/prompts/__init__.py
@@ -16,6 +16,7 @@ from .ai_society import AISocietyPromptTemplateDict
from .code import CodePromptTemplateDict
from .misalignment import MisalignmentPromptTemplateDict
from .translation import TranslationPromptTemplateDict
+from .object_recognition import ObjectRecognitionPromptTemplateDict
from .solution_extraction import SolutionExtractionPromptTemplateDict
from .evaluation import EvaluationPromptTemplateDict
from .role_description_prompt_template import RoleDescriptionPromptTemplateDict
@@ -35,4 +36,5 @@ __all__ = [
'TaskPromptTemplateDict',
'PromptTemplateGenerator',
'SolutionExtractionPromptTemplateDict',
+ 'ObjectRecognitionPromptTemplateDict',
]
diff --git a/camel/prompts/object_recognition.py b/camel/prompts/object_recognition.py
new file mode 100644
index 000000000..de8d91cbf
--- /dev/null
+++ b/camel/prompts/object_recognition.py
@@ -0,0 +1,32 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import Any
+
+from camel.prompts import TextPrompt, TextPromptDict
+from camel.types import RoleType
+
+
+# flake8: noqa :E501
+class ObjectRecognitionPromptTemplateDict(TextPromptDict):
+ ASSISTANT_PROMPT = TextPrompt(
+ """You have been assigned an object recognition task.
+Your mission is to list all detected objects in following image.
+Your output should always be a list of strings starting with `1.`, `2.` etc.
+Do not explain yourself or output anything else.""")
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self.update({
+ RoleType.ASSISTANT: self.ASSISTANT_PROMPT,
+ })
\ No newline at end of file
diff --git a/camel/prompts/task_prompt_template.py b/camel/prompts/task_prompt_template.py
index 0e603e666..3ab338bae 100644
--- a/camel/prompts/task_prompt_template.py
+++ b/camel/prompts/task_prompt_template.py
@@ -18,6 +18,7 @@ from camel.prompts import (
CodePromptTemplateDict,
EvaluationPromptTemplateDict,
MisalignmentPromptTemplateDict,
+ ObjectRecognitionPromptTemplateDict,
RoleDescriptionPromptTemplateDict,
SolutionExtractionPromptTemplateDict,
TextPromptDict,
@@ -53,4 +54,6 @@ class TaskPromptTemplateDict(Dict[Any, TextPromptDict]):
SolutionExtractionPromptTemplateDict(),
TaskType.ROLE_DESCRIPTION:
RoleDescriptionPromptTemplateDict(),
+ TaskType.OBJECT_RECOGNITION:
+ ObjectRecognitionPromptTemplateDict(),
})
diff --git a/camel/types/__init__.py b/camel/types/__init__.py
index 45eb2c44e..92ec2515b 100644
--- a/camel/types/__init__.py
+++ b/camel/types/__init__.py
@@ -19,6 +19,8 @@ from .enums import (
OpenAIBackendRole,
EmbeddingModelType,
VectorDistance,
+ OpenAIImageType,
+ OpenAIImageDetailType,
StorageType,
)
from .openai_types import (
@@ -53,4 +55,6 @@ __all__ = [
'ChatCompletionAssistantMessageParam',
'ChatCompletionFunctionMessageParam',
'CompletionUsage',
+ 'OpenAIImageType',
+ 'OpenAIImageDetailType',
]
diff --git a/camel/types/enums.py b/camel/types/enums.py
index b905f478d..e9aaf0b93 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -12,7 +12,7 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import re
-from enum import Enum
+from enum import Enum, EnumMeta
class RoleType(Enum):
@@ -155,6 +155,7 @@ class TaskType(Enum):
EVALUATION = "evaluation"
SOLUTION_EXTRACTION = "solution_extraction"
ROLE_DESCRIPTION = "role_description"
+ OBJECT_RECOGNITION = "object_recognition"
DEFAULT = "default"
@@ -183,6 +184,32 @@ class TerminationMode(Enum):
ALL = "all"
+class OpenAIImageTypeMeta(EnumMeta):
+
+ def __contains__(cls, image_type: object) -> bool:
+ try:
+ cls(image_type)
+ except ValueError:
+ return False
+ return True
+
+
+class OpenAIImageType(Enum, metaclass=OpenAIImageTypeMeta):
+ r"""Image types supported by OpenAI vision model."""
+ # https://platform.openai.com/docs/guides/vision
+ PNG = "png"
+ JPEG = "jpeg"
+ JPG = "jpg"
+ WEBP = "webp"
+ GIF = "gif"
+
+
+class OpenAIImageDetailType(Enum):
+ AUTO = "auto"
+ LOW = "low"
+ HIGH = "high"
+
+
class StorageType(Enum):
MILVUS = "milvus"
QDRANT = "qdrant"
diff --git a/camel/utils/token_counting.py b/camel/utils/token_counting.py
index 92266cd1b..db1f4c212 100644
--- a/camel/utils/token_counting.py
+++ b/camel/utils/token_counting.py
@@ -11,11 +11,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import base64
from abc import ABC, abstractmethod
-from typing import List
+from io import BytesIO
+from math import ceil
+from typing import List, Optional
+
+from PIL import Image
from camel.messages import OpenAIMessage
-from camel.types import ModelType
+from camel.types import ModelType, OpenAIImageDetailType, OpenAIImageType
+
+LOW_DETAIL_TOKENS = 85
+FIT_SQUARE_PIXELS = 2048
+SHORTEST_SIDE_PIXELS = 768
+SQUARE_PIXELS = 512
+SQUARE_TOKENS = 170
+EXTRA_TOKENS = 85
def messages_to_prompt(messages: List[OpenAIMessage], model: ModelType) -> str:
@@ -167,8 +179,7 @@ class OpenAITokenCounter(BaseTokenCounter):
r"""Constructor for the token counter for OpenAI models.
Args:
- model_type (ModelType): Model type for which tokens will be
- counted.
+ model (ModelType): Model type for which tokens will be counted.
"""
self.model: str = model.value_for_tiktoken
@@ -211,10 +222,73 @@ class OpenAITokenCounter(BaseTokenCounter):
for message in messages:
num_tokens += self.tokens_per_message
for key, value in message.items():
- num_tokens += len(self.encoding.encode(str(value)))
+ if not isinstance(value, list):
+ num_tokens += len(self.encoding.encode(str(value)))
+ else:
+ for item in value:
+ if item["type"] == "text":
+ num_tokens += len(
+ self.encoding.encode(str(item["text"])))
+ elif item["type"] == "image_url":
+ image_str: str = item["image_url"]["url"]
+ detail = item["image_url"]["detail"]
+ image_prefix_format = "data:image/{};base64,"
+ image_prefix: Optional[str] = None
+ for image_type in list(OpenAIImageType):
+ # Find the correct image format
+ image_prefix = image_prefix_format.format(
+ image_type.value)
+ if image_prefix in image_str:
+ break
+ assert isinstance(image_prefix, str)
+ encoded_image = image_str.split(image_prefix)[1]
+ image_bytes = BytesIO(
+ base64.b64decode(encoded_image))
+ image = Image.open(image_bytes)
+ num_tokens += count_tokens_from_image(
+ image, OpenAIImageDetailType(detail))
if key == "name":
num_tokens += self.tokens_per_name
# every reply is primed with <|start|>assistant<|message|>
num_tokens += 3
return num_tokens
+
+
+def count_tokens_from_image(image: Image.Image,
+ detail: OpenAIImageDetailType) -> int:
+ r"""Count image tokens for OpenAI vision model. An :obj:`"auto"`
+ resolution model will be treated as :obj:`"high"`. All images with
+ :obj:`"low"` detail cost 85 tokens each. Images with :obj:`"high"` detail
+ are first scaled to fit within a 2048 x 2048 square, maintaining their
+ aspect ratio. Then, they are scaled such that the shortest side of the
+ image is 768px long. Finally, we count how many 512px squares the image
+ consists of. Each of those squares costs 170 tokens. Another 85 tokens are
+ always added to the final total. For more details please refer to `OpenAI
+ vision docs <https://platform.openai.com/docs/guides/vision>`_
+
+ Args:
+ image (PIL.Image.Image): Image to count number of tokens.
+ detail (OpenAIImageDetailType): Image detail type to count
+ number of tokens.
+
+ Returns:
+ int: Number of tokens for the image given a detail type.
+ """
+ if detail == OpenAIImageDetailType.LOW:
+ return LOW_DETAIL_TOKENS
+
+ width, height = image.size
+ if width > FIT_SQUARE_PIXELS or height > FIT_SQUARE_PIXELS:
+ scaling_factor = max(width, height) / FIT_SQUARE_PIXELS
+ width = int(width / scaling_factor)
+ height = int(height / scaling_factor)
+
+ scaling_factor = min(width, height) / SHORTEST_SIDE_PIXELS
+ scaled_width = int(width / scaling_factor)
+ scaled_height = int(height / scaling_factor)
+
+ h = ceil(scaled_height / SQUARE_PIXELS)
+ w = ceil(scaled_width / SQUARE_PIXELS)
+ total = EXTRA_TOKENS + SQUARE_TOKENS * h * w
+ return total
diff --git a/examples/vision/object_recognition.py b/examples/vision/object_recognition.py
new file mode 100644
index 000000000..d9ce440d1
--- /dev/null
+++ b/examples/vision/object_recognition.py
@@ -0,0 +1,64 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import argparse
+
+from PIL import Image
+
+from camel.agents import ChatAgent
+from camel.generators import PromptTemplateGenerator
+from camel.messages import BaseMessage
+from camel.types import ModelType, RoleType, TaskType
+
+parser = argparse.ArgumentParser(description="Arguments for object detection.")
+parser.add_argument(
+ "--image_path",
+ type=str,
+ help="Path to the image for object detection.",
+)
+
+
+def detect_image_obj(image_path: str) -> None:
+ sys_msg = PromptTemplateGenerator().get_prompt_from_key(
+ TaskType.OBJECT_RECOGNITION, RoleType.ASSISTANT)
+ print("=" * 20 + " SYS MSG " + "=" * 20)
+ print(sys_msg)
+ print("=" * 49)
+
+ assistant_sys_msg = BaseMessage.make_assistant_message(
+ role_name="Assistant",
+ content=sys_msg,
+ )
+ agent = ChatAgent(
+ assistant_sys_msg,
+ model_type=ModelType.GPT_4_TURBO_VISION,
+ )
+ user_msg = BaseMessage.make_user_message(
+ role_name="User",
+ content="Please start the object detection for following image!",
+ image=Image.open(image_path),
+ image_detail="high",
+ )
+ assistant_response = agent.step(user_msg)
+ print("=" * 20 + " RESULT " + "=" * 20)
+ print(assistant_response.msgs[0].content)
+ print("=" * 48)
+
+
+def main(args: argparse.Namespace) -> None:
+ detect_image_obj(args.image_path)
+
+
+if __name__ == "__main__":
+ args = parser.parse_args()
+ main(args=args)
diff --git a/test/agents/test_chat_agent.py b/test/agents/test_chat_agent.py
index 45dfde732..dd65f1324 100644
--- a/test/agents/test_chat_agent.py
+++ b/test/agents/test_chat_agent.py
@@ -11,19 +11,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from io import BytesIO
from typing import List
+from unittest.mock import Mock
import pytest
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.completion_usage import CompletionUsage
+from PIL import Image
from camel.agents import ChatAgent
from camel.agents.chat_agent import FunctionCallingRecord
-from camel.configs import ChatGPTConfig, FunctionCallingConfig
+from camel.configs import (
+ ChatGPTConfig,
+ ChatGPTVisionConfig,
+ FunctionCallingConfig,
+)
from camel.functions import MATH_FUNCS
from camel.generators import SystemMessageGenerator
from camel.memories import MemoryRecord
from camel.messages import BaseMessage
from camel.terminators import ResponseWordsTerminator
-from camel.types import ModelType, OpenAIBackendRole, RoleType, TaskType
+from camel.types import (
+ ChatCompletion,
+ ModelType,
+ OpenAIBackendRole,
+ RoleType,
+ TaskType,
+)
parametrize = pytest.mark.parametrize('model', [
ModelType.STUB,
@@ -313,3 +329,51 @@ def test_response_words_termination():
assert agent.terminated
assert agent_response.terminated
assert "goodbye" in agent_response.info['termination_reasons'][0]
+
+
+def test_chat_agent_vision():
+ system_message = BaseMessage(role_name="assistant",
+ role_type=RoleType.ASSISTANT, meta_dict=None,
+ content="You are a help assistant.")
+ model_config = ChatGPTVisionConfig(temperature=0, max_tokens=200)
+ agent = ChatAgent(
+ system_message=system_message,
+ model_type=ModelType.GPT_4_TURBO_VISION,
+ model_config=model_config,
+ )
+
+ # Create an all blue PNG image:
+ image = Image.new("RGB", (100, 100), "blue")
+ img_byte_arr = BytesIO()
+ image.save(img_byte_arr, format='PNG')
+ image = Image.open(img_byte_arr)
+
+ user_msg = BaseMessage(
+ role_name="User",
+ role_type=RoleType.USER,
+ meta_dict=dict(),
+ content="Is this image blue? Just answer yes or no.",
+ image=image,
+ image_detail="low",
+ )
+ # Mock the OpenAI model return value:
+ agent.model_backend = Mock()
+ agent.model_backend.run.return_value = ChatCompletion(
+ id="mock_vision_id",
+ choices=[
+ Choice(
+ finish_reason='stop', index=0, logprobs=None,
+ message=ChatCompletionMessage(content='Yes.', role='assistant',
+ function_call=None,
+ tool_calls=None))
+ ],
+ created=123456,
+ model='gpt-4-1106-vision-preview',
+ object='chat.completion',
+ system_fingerprint=None,
+ usage=CompletionUsage(completion_tokens=2, prompt_tokens=113,
+ total_tokens=115),
+ )
+
+ agent_response = agent.step(user_msg)
+ assert agent_response.msgs[0].content == "Yes."
diff --git a/test/utils/test_token_counting.py b/test/utils/test_token_counting.py
new file mode 100644
index 000000000..a1421a337
--- /dev/null
+++ b/test/utils/test_token_counting.py
@@ -0,0 +1,31 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the ā€œLicenseā€);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ā€œAS ISā€ BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import pytest
+from PIL import Image
+
+from camel.types import OpenAIImageDetailType
+from camel.utils.token_counting import count_tokens_from_image
+
+
[email protected]("width,height,detail,token_cost", [
+ (1024, 1024, OpenAIImageDetailType.HIGH, 765),
+ (1024, 1024, OpenAIImageDetailType.AUTO, 765),
+ (2048, 4096, OpenAIImageDetailType.HIGH, 1105),
+ (2048, 4096, OpenAIImageDetailType.LOW, 85),
+])
+def test_openai_count_token_from_image(width: int, height: int,
+ detail: OpenAIImageDetailType,
+ token_cost: int):
+ image = Image.new("RGB", (width, height), "black")
+ assert count_tokens_from_image(image, detail) == token_cost