🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Integration of VLM embedding model (#446)
Yunmeng52938a71cc2b
diff --git a/camel/embeddings/__init__.py b/camel/embeddings/__init__.pyindex 0e4d5b136..f4b947dd3 100644--- a/camel/embeddings/__init__.py+++ b/camel/embeddings/__init__.py@@ -14,9 +14,11 @@from .base import BaseEmbeddingfrom .openai_embedding import OpenAIEmbeddingfrom .sentence_transformers_embeddings import SentenceTransformerEncoder+from .vlm_embedding import VisionLanguageEmbedding__all__ = ["BaseEmbedding","OpenAIEmbedding","SentenceTransformerEncoder",+ "VisionLanguageEmbedding",]diff --git a/camel/embeddings/vlm_embedding.py b/camel/embeddings/vlm_embedding.pynew file mode 100644index 000000000..4bc391ffa--- /dev/null+++ b/camel/embeddings/vlm_embedding.py@@ -0,0 +1,143 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the “License”);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an “AS IS” BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from typing import Any, List, Optional, Union++from PIL import Image++from camel.embeddings import BaseEmbedding+++class VisionLanguageEmbedding(BaseEmbedding[Union[str, Image.Image]]):+ r"""Provides image embedding functionalities using multimodal model.++ Args:+ model_name : The model type to be used for generating embeddings.+ And the default value is: obj:`openai/clip-vit-base-patch32`.++ Raises:+ RuntimeError: If an unsupported model type is specified.+ """++ def __init__(+ self, model_name: str = "openai/clip-vit-base-patch32"+ ) -> None:+ r"""Initializes the: obj: `VisionLanguageEmbedding` class with a+ specified model and return the dimension of embeddings.++ Args:+ model_name (str, optional): The version name of the model to use.+ (default: :obj:`openai/clip-vit-base-patch32`)+ """+ from transformers import AutoModel, AutoProcessor++ try:+ self.model = AutoModel.from_pretrained(model_name)+ self.processor = AutoProcessor.from_pretrained(model_name)+ except Exception as e:+ raise RuntimeError(f"Failed to load model '{model_name}': {e}")++ self.valid_processor_kwargs = []+ self.valid_model_kwargs = []++ try:+ self.valid_processor_kwargs = (+ self.processor.image_processor._valid_processor_keys+ )+ self.valid_model_kwargs = [+ "pixel_values",+ "return_dict",+ "interpolate_pos_encoding",+ ]+ except Exception:+ print("Warning: not typically processor and model structure")+ pass+ self.dim: Optional[int] = None++ def embed_list(+ self, objs: List[Union[Image.Image, str]], **kwargs: Any+ ) -> List[List[float]]:+ """Generates embeddings for the given images or texts.++ Args:+ objs (List[Image.Image|str]): The list of images or texts for+ which to generate the embeddings.+ image_processor_kwargs: Extra kwargs passed to the image processor.+ tokenizer_kwargs: Extra kwargs passed to the text tokenizer(processor).+ model_kwargs: Extra kwargs passed to the main model.++ Returns:+ List[List[float]]: A list that represents the generated embedding+ as a list of floating-point numbers.++ Raises:+ ValueError: If the input type is not `Image.Image` or `str`.+ """+ if not objs:+ raise ValueError("Input objs list is empty.")++ image_processor_kwargs: Optional[dict] = kwargs.get(+ 'image_processor_kwargs', {}+ )+ tokenizer_kwargs: Optional[dict] = kwargs.get('tokenizer_kwargs', {})+ model_kwargs: Optional[dict] = kwargs.get('model_kwargs', {})++ result_list = []+ for obj in objs:+ if isinstance(obj, Image.Image):+ image_input = self.processor(+ images=obj,+ return_tensors="pt",+ padding=True,+ **image_processor_kwargs,+ )+ image_feature = (+ self.model.get_image_features(**image_input, **model_kwargs)+ .squeeze(dim=0)+ .tolist()+ )+ result_list.append(image_feature)+ elif isinstance(obj, str):+ text_input = self.processor(+ text=obj,+ return_tensors="pt",+ padding=True,+ **tokenizer_kwargs,+ )+ text_feature = (+ self.model.get_text_features(**text_input, **model_kwargs)+ .squeeze(dim=0)+ .tolist()+ )+ result_list.append(text_feature)+ else:+ raise ValueError("Input type is not image nor text.")++ self.dim = len(result_list[0])++ if any(len(result) != self.dim for result in result_list):+ raise ValueError("Dimensionality is not consistent.")++ return result_list++ def get_output_dim(self) -> int:+ r"""Returns the output dimension of the embeddings.++ Returns:+ int: The dimensionality of the embedding for the current model.+ """+ if self.dim is None:+ text = 'dimension'+ inputs = self.processor(text=[text], return_tensors="pt")+ self.dim = self.model.get_text_features(**inputs).shape[1]+ return self.dimdiff --git a/licenses/update_license.py b/licenses/update_license.pyindex d937bc699..0afbf3b9a 100644--- a/licenses/update_license.py+++ b/licenses/update_license.py@@ -39,10 +39,12 @@ def update_license_in_file(start_line_start_with: str,end_line_start_with: str,) -> bool:- with open(file_path, 'r') as f:+ with open(+ file_path, 'r', encoding='utf-8'+ ) as f: # for windows compatibilitycontent = f.read()- with open(license_template_path, 'r') as f:+ with open(license_template_path, 'r', encoding='utf-8') as f:new_license = f.read().strip()maybe_existing_licenses = re.findall(diff --git a/poetry.lock b/poetry.lockindex e0b3ebd7f..0c4304f0e 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.[[package]]name = "accelerate"@@ -2354,9 +2354,13 @@ files = [{file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"},{file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"},{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"},+ {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"},{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"},+ {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"},{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"},+ {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"},{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"},+ {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"},{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"},{file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"},{file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"},@@ -7076,16 +7080,16 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linktesting = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"][extras]-all = ["PyMuPDF", "accelerate", "beautifulsoup4", "cohere", "datasets", "diffusers", "docx2txt", "googlemaps", "neo4j", "openapi-spec-validator", "opencv-python", "prance", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "slack-sdk", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]+all = ["PyMuPDF", "accelerate", "beautifulsoup4", "cohere", "datasets", "diffusers", "docx2txt", "googlemaps", "neo4j", "openapi-spec-validator", "opencv-python", "pillow", "prance", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "slack-sdk", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]encoders = ["sentence-transformers"]graph-storages = ["neo4j"]huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "transformers"]retrievers = ["cohere", "rank-bm25"]test = ["mock", "pytest"]-tools = ["PyMuPDF", "beautifulsoup4", "docx2txt", "googlemaps", "openapi-spec-validator", "prance", "pydub", "pygithub", "pyowm", "requests_oauthlib", "slack-sdk", "unstructured", "wikipedia", "wolframalpha"]+tools = ["PyMuPDF", "beautifulsoup4", "docx2txt", "googlemaps", "openapi-spec-validator", "pillow", "prance", "pydub", "pygithub", "pyowm", "requests_oauthlib", "slack-sdk", "unstructured", "wikipedia", "wolframalpha"]vector-databases = ["pymilvus", "qdrant-client"][metadata]lock-version = "2.0"python-versions = ">=3.8.1,<3.12"-content-hash = "4e22773023f79ee56f30597046ad95cc5b87d3e1a27b3e7b99ba79e7622f0723"+content-hash = "2c2023104b8ef3b2eade2c43c979b047d3dc07a9bbb69795808ee4a96032dd33"diff --git a/pyproject.toml b/pyproject.tomlindex 251ee8d87..2fa482281 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -61,6 +61,7 @@ requests_oauthlib = { version = "^1.3.1", optional = true }prance = { version = "^23.6.21.0", optional = true }openapi-spec-validator = { version = "^0.7.1", optional = true }unstructured = { extras = ["all-docs"], version = "^0.10.30", optional = true }+pillow = { version = "^10.2.0", optional = true }slack-sdk = { version = "^3.27.2", optional = true }pydub = { version = "^0.25.1", optional = true }pygithub = { version = "^2.3.0", optional = true }@@ -114,6 +115,7 @@ tools = ["prance","openapi-spec-validator","unstructured",+ "pillow","slack-sdk","pydub","pygithub",@@ -169,6 +171,7 @@ all = ["neo4j",# retrievers"rank-bm25",+ "pillow",][tool.poetry.group.dev]@@ -276,8 +279,10 @@ module = ["cohere","sentence_transformers.*","pymilvus",+ "pillow","slack-sdk","pydub","pygithub"+]ignore_missing_imports = truediff --git a/test/embeddings/test_vlm_embeddings.py b/test/embeddings/test_vlm_embeddings.pynew file mode 100644index 000000000..5e7179fa0--- /dev/null+++ b/test/embeddings/test_vlm_embeddings.py@@ -0,0 +1,78 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the “License”);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an “AS IS” BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import pytest+import requests+from PIL import Image+from transformers import CLIPModel, CLIPProcessor++from camel.embeddings import VisionLanguageEmbedding+++def VLM_instance() -> VisionLanguageEmbedding:+ return VisionLanguageEmbedding()+++def test_CLIPEmbedding_initialization(VLM_instance):+ assert VLM_instance is not None+ assert isinstance(VLM_instance.model, CLIPModel)+ assert isinstance(VLM_instance.processor, CLIPProcessor)+++def test_image_embed_list_with_valid_input(VLM_instance):+ url = "http://images.cocodataset.org/val2017/000000039769.jpg"+ image = Image.open(requests.get(url, stream=True).raw)+ test_images = [image, image]+ embeddings = VLM_instance.embed_list(test_images)+ assert isinstance(embeddings, list)+ assert len(embeddings) == 2+ for e in embeddings:+ assert len(e) == VLM_instance.get_output_dim()+++def test_image_embed_list_with_empty_input(VLM_instance):+ with pytest.raises(ValueError):+ VLM_instance.embed_list([])+++def test_text_embed_list_with_valid_input(VLM_instance):+ test_texts = ['Hello world', 'Testing sentence embeddings']+ embeddings = VLM_instance.embed_list(test_texts)+ assert isinstance(embeddings, list)+ assert len(embeddings) == 2+ for e in embeddings:+ assert len(e) == VLM_instance.get_output_dim()+++def test_text_embed_list_with_empty_input(VLM_instance):+ with pytest.raises(ValueError):+ VLM_instance.embed_list([])+++def test_mixed_embed_list_with_valid_input(VLM_instance):+ test_list = ['Hello world', 'Testing sentence embeddings']+ url = "http://images.cocodataset.org/val2017/000000039769.jpg"+ image = Image.open(requests.get(url, stream=True).raw)+ test_list.append(image)+ embeddings = VLM_instance.embed_list(test_list)+ assert isinstance(embeddings, list)+ assert len(embeddings) == 3+ for e in embeddings:+ assert len(e) == VLM_instance.get_output_dim()+++def test_get_output_dim(VLM_instance):+ output_dim = VLM_instance.get_output_dim()+ assert isinstance(output_dim, int)+ assert output_dim > 0