🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
Revert "feat: Add support for open-sourced multi-modal model (#787)" (#876)
Wendong-Fanb21279fc29bf
diff --git a/camel/messages/base.py b/camel/messages/base.pyindex f0c428a2f..f58895250 100644--- a/camel/messages/base.py+++ b/camel/messages/base.py@@ -27,7 +27,6 @@ from camel.messages import ()from camel.prompts import CodePrompt, TextPromptfrom camel.types import (- MultiModalModelType,OpenAIBackendRole,OpenAIImageType,OpenAIVisionDetailType,@@ -47,16 +46,14 @@ class BaseMessage:meta_dict (Optional[Dict[str, str]]): Additional metadata dictionaryfor the message.content (str): The content of the message.- video_bytes (Optional[Union[bytes, List[bytes]]): Optional bytes- object or list of bytes objects associated with the message.+ video_bytes (Optional[bytes]): Optional bytes of a video associated+ with the message. Default is None.image_list (Optional[List[Image.Image]]): Optional list of PIL Imageobjects associated with the message. Default is None.image_detail (Literal["auto", "low", "high"]): Detail level of theimages associated with the message. Default is "auto".video_detail (Literal["auto", "low", "high"]): Detail level of thevideos associated with the message. Default is "low".- video_type (Optional[str]): Optional type of the video associated- with the message. Default is None."""role_name: str@@ -64,11 +61,10 @@ class BaseMessage:meta_dict: Optional[Dict[str, str]]content: str- video_bytes: Optional[Union[bytes, List[bytes]]] = None+ video_bytes: Optional[bytes] = Noneimage_list: Optional[List[Image.Image]] = Noneimage_detail: Literal["auto", "low", "high"] = "auto"video_detail: Literal["auto", "low", "high"] = "low"- video_type: Optional[str] = None@classmethoddef make_user_message(@@ -76,7 +72,7 @@ class BaseMessage:role_name: str,content: str,meta_dict: Optional[Dict[str, str]] = None,- video_bytes: Optional[Union[bytes, List[bytes]]] = None,+ video_bytes: Optional[bytes] = None,image_list: Optional[List[Image.Image]] = None,image_detail: Union[OpenAIVisionDetailType, str@@ -84,7 +80,6 @@ class BaseMessage:video_detail: Union[OpenAIVisionDetailType, str] = OpenAIVisionDetailType.LOW,- video_type: Optional[str] = None,) -> "BaseMessage":return cls(role_name,@@ -95,7 +90,6 @@ class BaseMessage:image_list,OpenAIVisionDetailType(image_detail).value,OpenAIVisionDetailType(video_detail).value,- video_type,)@classmethod@@ -104,7 +98,7 @@ class BaseMessage:role_name: str,content: str,meta_dict: Optional[Dict[str, str]] = None,- video_bytes: Optional[Union[bytes, List[bytes]]] = None,+ video_bytes: Optional[bytes] = None,image_list: Optional[List[Image.Image]] = None,image_detail: Union[OpenAIVisionDetailType, str@@ -112,18 +106,16 @@ class BaseMessage:video_detail: Union[OpenAIVisionDetailType, str] = OpenAIVisionDetailType.LOW,- video_type: Optional[str] = None,) -> "BaseMessage":return cls(role_name,- RoleType.USER,+ RoleType.ASSISTANT,meta_dict,content,video_bytes,image_list,OpenAIVisionDetailType(image_detail).value,OpenAIVisionDetailType(video_detail).value,- video_type,)def create_new_instance(self, content: str) -> "BaseMessage":@@ -257,11 +249,7 @@ class BaseMessage:if role_at_backend == OpenAIBackendRole.SYSTEM:return self.to_openai_system_message()elif role_at_backend == OpenAIBackendRole.USER:- is_multimodal_message_type = isinstance(self.video_bytes, List)- if is_multimodal_message_type:- return self.to_multimodal_model_user_message()- else:- return self.to_openai_user_message()+ return self.to_openai_user_message()elif role_at_backend == OpenAIBackendRole.ASSISTANT:return self.to_openai_assistant_message()else:@@ -282,8 +270,8 @@ class BaseMessage:Returns:OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object."""- hybrid_content: List[Any] = []- hybrid_content.append(+ hybird_content: List[Any] = []+ hybird_content.append({"type": "text","text": self.content,@@ -312,7 +300,7 @@ class BaseMessage:"utf-8")image_prefix = f"data:image/{image_type};base64,"- hybrid_content.append(+ hybird_content.append({"type": "image_url","image_url": {@@ -323,12 +311,6 @@ class BaseMessage:)if self.video_bytes:- if not isinstance(self.video_bytes, bytes):- raise ValueError(- "Video bytes should be a bytes object for the OpenAI "- "model"- )-import imageio.v3 as iiobase64Frames: List[str] = []@@ -377,113 +359,12 @@ class BaseMessage:},}- hybrid_content.append(item)-- if len(hybrid_content) > 1:- return {- "role": "user",- "content": hybrid_content,- }- # This return just for str message- else:- return {- "role": "user",- "content": self.content,- }-- def to_multimodal_model_user_message(self) -> OpenAIUserMessage:- r"""Converts the message to an :obj:`OpenAIUserMessage` object.-- Returns:- OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.- """- hybrid_content: List[Any] = [- {- "type": "text",- "text": self.content,- }- ]-- if self.image_list and len(self.image_list) > 0:- for image in self.image_list:- if image.format is None:- raise ValueError(- f"Image's `format` is `None`, please "- f"transform the `PIL.Image.Image` to one of "- f"following supported formats, such as "- f"{list(MultiModalModelType)}"- )-- if not isinstance(image, Image.Image):- raise ValueError(- "Image should be a PIL Image object for the "- "multimodal model"- )-- image_type: str = image.format.lower()- if image_type not in MultiModalModelType:- raise ValueError(- f"Image type {image.format} "- f"is not supported by OpenAI vision model"- )- with io.BytesIO() as buffer:- image.save(fp=buffer, format=image.format)- encoded_image = base64.b64encode(buffer.getvalue()).decode(- "utf-8"- )- image_prefix = f"data:image/{image_type};base64,"- hybrid_content.append(- {- "type": "image_url",- "image_url": {- "url": f"{image_prefix}{encoded_image}",- "detail": self.image_detail,- },- }- )-- if self.video_bytes:- if not isinstance(self.video_bytes, List):- raise ValueError(- "Video bytes should be a list of bytes for the multimodal "- "model"- )-- if self.video_type is None:- raise ValueError(- f"Video's `format` is `None`, please "- f"transform the `PIL.Image.Image` to one of "- f"following supported formats, such as "- f"{list(MultiModalModelType)}"- )- if self.video_type.lower() not in MultiModalModelType:- raise ValueError(- f"Video type {self.video_type} "- f"is not supported by multimodal model"- )-- for video in self.video_bytes:- with io.BytesIO() as buffer:- buffer.write(video)- encoded_video = base64.b64encode(buffer.getvalue()).decode(- "utf-8"- )- video_prefix = f"data:video/{self.video_type.lower()};bytes,"-- hybrid_content.append(- {- "type": "video_url",- "video_url": {- "url": f"{video_prefix}{encoded_video}",- "detail": self.video_detail,- },- }- )+ hybird_content.append(item)- if len(hybrid_content) > 1:+ if len(hybird_content) > 1:return {"role": "user",- "content": hybrid_content,+ "content": hybird_content,}# This return just for str messageelse:diff --git a/camel/models/__init__.py b/camel/models/__init__.pyindex 63ae17381..59c03cd95 100644--- a/camel/models/__init__.py+++ b/camel/models/__init__.py@@ -16,7 +16,6 @@ from .azure_openai_model import AzureOpenAIModelfrom .base_model import BaseModelBackendfrom .gemini_model import GeminiModelfrom .groq_model import GroqModel-from .interlm_xcomposer_model import InternLMXComposerModelfrom .litellm_model import LiteLLMModelfrom .mistral_model import MistralModelfrom .model_factory import ModelFactory@@ -49,7 +48,6 @@ __all__ = ['OllamaModel','VLLMModel','GeminiModel',- 'InternLMXComposerModel','OpenAICompatibilityModel','SambaModel','TogetherAIModel',diff --git a/camel/models/interlm_xcomposer_model.py b/camel/models/interlm_xcomposer_model.pydeleted file mode 100644index 842dbf8b2..000000000--- a/camel/models/interlm_xcomposer_model.py+++ /dev/null@@ -1,325 +0,0 @@-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-# Licensed under the Apache License, Version 2.0 (the “License”);-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an “AS IS” BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-from enum import Enum-from typing import Any, Dict, List, Optional, Union--from openai import Stream--from camel.messages import OpenAIMessage-from camel.models import BaseModelBackend-from camel.types import (- ChatCompletion,- ChatCompletionChunk,- ChatCompletionMessage,- Choice,- CompletionUsage,- ModelType,-)-from camel.utils import (- BaseTokenCounter,- OpenAITokenCounter,-)---# The model provides different operation modes/functionalities.-# Refer to https://github.com/InternLM/InternLM-XComposer-class OperationMode(Enum):- CHAT = "chat" # the normal mode for the model- WRITE_WEBPAGE = "write_webpage"- RESUME_TO_WEBPAGE = "resume_2_webpage"- WRITE_ARTICLE = "write_article"---class InternLMXComposerModel(BaseModelBackend):- r"""Class for interace with OpenAI-API-compatible servers running- open-source InternLM-XComposer models.- """-- def __init__(- self,- model_type: ModelType,- model_config_dict: Dict[str, Any],- ) -> None:- r"""Constructor for model backends of Open-source models.-- Args:- model_type (ModelType): Model for which a backend is created.- model_config_dict (Dict[str, Any]): A dictionary that will- be fed into :obj:`openai.ChatCompletion.create()`.- """- super().__init__(- model_type,- model_config_dict,- api_key=None,- url=None,- token_counter=None,- )-- from pathlib import Path-- self.multimodal_src_directory: Path = Path(- model_config_dict.get("multimodal_src_path", "./examples/src")- )-- # `model_path` is the path to the open-source model- model_path: Optional[str] = self.model_config_dict.get(- "model_path", None- )- if not model_path:- raise ValueError("Path to open-source model is not provided.")- self.model_path: str = model_path-- self.model_name: str = self.model_path.split('/')[-1]-- try:- # `operation_mode` is the operation mode of the open-source model- # Refer to https://github.com/InternLM/InternLM-XComposer- self.operation_mode = model_config_dict.get(- "operation_mode", OperationMode.CHAT- )- except ValueError:- valid_modes = [mode.value for mode in OperationMode]- raise ValueError(- "Invalid operation mode for open-source model "- f"backend. Choose from {valid_modes}"- )-- import torch-- model_kwargs = self.model_config_dict.get("model_kwargs", {})- model_kwargs["torch_dtype"] = torch.bfloat16- model_kwargs["trust_remote_code"] = True-- from transformers import AutoModel, AutoTokenizer-- self._client = (- AutoModel.from_pretrained(- self.model_name,- **model_kwargs,- )- .to(- model_kwargs.get("device") or "cuda"- if torch.cuda.is_available()- else "cpu"- ) # move the model to the specified device (GPU or CPU)- .eval() # set the model to evaluation mode- .half() # convert the model parameters to half-precision floating point (FP16). # noqa: E501- )-- tokenizer_kwargs = self.model_config_dict.get("tokenizer_kwargs", {})- tokenizer_kwargs["trust_remote_code"] = True-- self._tokenizer = AutoTokenizer.from_pretrained(- self.model_name, **tokenizer_kwargs- )- self._client.tokenizer = self._tokenizer # mypy: ignore[attr-defined]-- # Replace `model_config_dict` with only the `model_kwargs`- self.model_config_dict = self.model_config_dict["model_kwargs"]-- @property- def token_counter(self) -> BaseTokenCounter:- r"""Initialize the token counter for the model backend.-- Returns:- BaseTokenCounter: The token counter following the model's- tokenization style.- """- if not self._token_counter:- self._token_counter = OpenAITokenCounter(ModelType.GPT_3_5_TURBO)- return self._token_counter-- def run(- self,- messages: List[OpenAIMessage],- ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:- r"""Runs inference of OpenAI-API-style chat completion.-- Args:- messages (List[OpenAIMessage]): Message list with the chat history- in OpenAI API format.-- Returns:- Union[ChatCompletion, Stream[ChatCompletionChunk]]:- `ChatCompletion` in the non-stream mode, or- `Stream[ChatCompletionChunk]` in the stream mode.- """-- # Extract the text content from List[OpenAIMessage].- last_message = messages[-1]-- if isinstance(last_message["content"], str):- # If the message does not contain image nor video:- message_content = last_message["content"]- elif isinstance(last_message["content"], list):- # If the message contains image or video:-- import base64- import shutil-- n_image = 0 # number of images- n_video = 0 # number of videos- image = [] # list of images local path-- # Check and clean the images folder- if self.multimodal_src_directory.exists():- shutil.rmtree(self.multimodal_src_directory)- self.multimodal_src_directory.mkdir(parents=True, exist_ok=True)-- # Get `message_content`- for item in last_message["content"]:- if item["type"] == "text":- message_content = item["text"]- elif item["type"] == "image_url":- # Get `image` list-- n_image += 1- base64_string = item['image_url']['url'].split(',')[1]-- # Convert the base64 string into images- # and save them as the local files.- image_data = base64.b64decode(base64_string)- image_file_path = (- self.multimodal_src_directory / f"image_{n_image}.jpg"- )-- with open(image_file_path, 'wb') as file:- file.write(image_data)- image.append(str(image_file_path))- elif item["type"] == "video_url":- # Get `video` list-- n_video += 1- base64_string = item['video_url']['url'].split(',')[1]-- # Convert the base64 string into video- # and save them as the local files.- video_data = base64.b64decode(base64_string)- video_file_path = (- self.multimodal_src_directory / f"video_{n_video}.mp4"- )-- with open(video_file_path, 'wb') as file:- file.write(video_data)- image.append(str(video_file_path))-- else:- message_content = ""- image = []-- import torch-- torch.set_grad_enabled(False)-- with torch.autocast(device_type='cuda', dtype=torch.float16):- if self.operation_mode == OperationMode.CHAT:- response_str, _ = ( # output param `his` is ignored- self._client.chat(- self._tokenizer,- message_content,- image,- do_sample=False,- num_beams=3,- use_meta=True,- )- )- elif self.operation_mode == OperationMode.WRITE_WEBPAGE:- response_str, _ = ( # output param `his` is ignored- self._client.write_webpage(- message_content,- image,- seed=202,- task="Instruction-aware webpage generation",- repetition_penalty=3.0,- )- )- elif self.operation_mode == OperationMode.RESUME_TO_WEBPAGE:- response_str, _ = ( # output param `his` is ignored- self._client.resume_to_webpage(- message_content,- image,- seed=202,- repetition_penalty=3.0,- )- )- elif self.operation_mode == OperationMode.WRITE_ARTICLE:- response_str, _ = ( # output param `his` is ignored- self._client.write_article(- message_content,- seed=8192,- )- )-- # Adapt to OpenAI's response format.- response = ChatCompletion(- id="interlm-xcomposer-output-id",- choices=[- Choice(- finish_reason="stop",- index=0,- logprobs=None,- message=ChatCompletionMessage(- content=response_str,- role="assistant",- function_call=None,- tool_calls=None,- ),- ),- ],- created=0,- model=self.model_name,- object="chat.completion",- service_tier=None,- system_fingerprint="fp_interlm_xcomposer_system_fingerprint",- usage=CompletionUsage(- completion_tokens=0,- prompt_tokens=0,- total_tokens=0,- ),- )-- return response-- def check_model_config(self):- r"""Check whether the model configuration is valid for open-source- model backends.-- Raises:- ValueError: If the model configuration dictionary contains any- unexpected arguments to OpenAI API, or it does not contain- :obj:`model_path`.- """- if "model_path" not in self.model_config_dict:- raise ValueError(- "Invalid configuration for open-source model backend with "- ":obj:`model_path` missing."- )-- if "operation_mode" in self.model_config_dict:- orperation_mode = self.model_config_dict["operation_mode"]- if orperation_mode not in OperationMode:- raise ValueError(- "Invalid configuration for open-source model backend with "- ":obj:`operation_mode` not in OperationMode."- )-- @property- def stream(self) -> bool:- r"""Returns whether the model is in stream mode,- which sends partial results each time.-- Returns:- bool: Whether the model is in stream mode.- """- return self.model_config_dict.get('stream', False)diff --git a/camel/models/model_factory.py b/camel/models/model_factory.pyindex 2c21397be..e51ff3175 100644--- a/camel/models/model_factory.py+++ b/camel/models/model_factory.py@@ -18,7 +18,6 @@ from camel.models.azure_openai_model import AzureOpenAIModelfrom camel.models.base_model import BaseModelBackendfrom camel.models.gemini_model import GeminiModelfrom camel.models.groq_model import GroqModel-from camel.models.interlm_xcomposer_model import InternLMXComposerModelfrom camel.models.litellm_model import LiteLLMModelfrom camel.models.mistral_model import MistralModelfrom camel.models.ollama_model import OllamaModel@@ -113,9 +112,6 @@ class ModelFactory:model_class = VLLMModelelif model_platform.is_litellm:model_class = LiteLLMModel- elif model_platform.is_internlm:- model_class = InternLMXComposerModel- return model_class(model_type, model_config_dict)elif model_platform.is_openai_compatibility_model:model_class = OpenAICompatibilityModelelif model_platform.is_together:diff --git a/camel/types/__init__.py b/camel/types/__init__.pyindex f55a9c101..5228436ec 100644--- a/camel/types/__init__.py+++ b/camel/types/__init__.py@@ -16,7 +16,6 @@ from .enums import (EmbeddingModelType,ModelPlatformType,ModelType,- MultiModalModelType,OpenAIBackendRole,OpenAIImageType,OpenAIVisionDetailType,@@ -64,7 +63,6 @@ __all__ = ['OpenAIImageType','OpenAIVisionDetailType','OpenAPIName',- 'MultiModalModelType','ModelPlatformType','AudioModelType','VoiceType',diff --git a/camel/types/enums.py b/camel/types/enums.pyindex 55d469373..fa6681675 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -429,20 +429,6 @@ class OpenAIVisionDetailType(Enum):HIGH = "high"-class MultiModalModelType(Enum, metaclass=OpenAIImageTypeMeta):- r"""Image types supported by the multimodal model."""-- # Align with OpenAI's specifications- # https://platform.openai.com/docs/guides/vision- PNG = "png"- JPEG = "jpeg"- JPG = "jpg"- WEBP = "webp"- GIF = "gif"-- MP4 = "mp4"--class StorageType(Enum):MILVUS = "milvus"QDRANT = "qdrant"@@ -475,7 +461,6 @@ class ModelPlatformType(Enum):TOGETHER = "together"OPENAI_COMPATIBILITY_MODEL = "openai-compatibility-model"SAMBA = "samba-nova"- INTERNLM = "internlm"@propertydef is_openai(self) -> bool:@@ -548,11 +533,6 @@ class ModelPlatformType(Enum):r"""Returns whether this platform is Samba Nova."""return self is ModelPlatformType.SAMBA- @property- def is_internlm(self) -> bool:- r"""Returns whether this platform is InternLM."""- return self in [ModelPlatformType.INTERNLM]-class AudioModelType(Enum):TTS_1 = "tts-1"diff --git a/examples/models/interlm_xcomposer_model_example.py b/examples/models/interlm_xcomposer_model_example.pydeleted file mode 100644index 061f02ccf..000000000--- a/examples/models/interlm_xcomposer_model_example.py+++ /dev/null@@ -1,109 +0,0 @@-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-# Licensed under the Apache License, Version 2.0 (the “License”);-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an “AS IS” BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-from io import BytesIO--import requests-from PIL import Image--from camel.agents import ChatAgent-from camel.messages import BaseMessage-from camel.models import ModelFactory-from camel.models.interlm_xcomposer_model import OperationMode-from camel.types import ModelPlatformType---def load_image_from_url(url: str) -> Image.Image:- r"""Load an image from a given URL.-- Args:- url (str): The url resource for image.-- Returns:- Image: The `Image` of the image.- """- response = requests.get(url)- if response.status_code != 200:- raise ValueError(f"Failed to load image from URL: {url}")- image = Image.open(BytesIO(response.content))- return image---def load_video_from_url(url: str) -> bytes:- r"""Load video data from a given URL.-- Args:- url (str): The url resource for video.-- Returns:- bytes: The byte format of the video.- """- response = requests.get(url)- if response.status_code != 200:- raise ValueError(f"Failed to load video from URL: {url}")- return response.content---# Get the resources from the urls.-video_url = 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4'-video_bytes = load_video_from_url(video_url)-video_bytes_list = [video_bytes]-image_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'-image = load_image_from_url(image_url)-images = [image, image]---ollama_model = ModelFactory.create(- model_platform=ModelPlatformType.INTERNLM,- model_type="internlm-xcomposer2d5-7b",- model_config_dict={- "model_path": "internlm/internlm-xcomposer2d5-7b",- # InterLM XComposer provides different operation modes.- "operation_mode": OperationMode.CHAT,- "model_kwargs": { # use `AutoModel` config in `transformers` library- "cache_dir": None,- },- },-)---assistant_sys_msg = BaseMessage.make_assistant_message(- role_name="Assistant",- content="You are a helpful assistant.",-)---agent = ChatAgent(assistant_sys_msg, model=ollama_model, token_limit=4096)---user_msg = BaseMessage.make_user_message(- role_name="User",- content="What did you see in the images?",- video_bytes=video_bytes_list,- video_type="mp4",- image_list=images,-)-assistant_response = agent.step(user_msg)-print(assistant_response.msg.content)--"""-===============================================================================-The image shows a serene natural landscape featuring a wooden boardwalk-stretching straight through a lush, vibrant green meadow. The scene is set-against a beautifully dynamic sky with patches of fluffy white clouds. The view-is tranquil and inviting, reminiscent of a peaceful, rural setting. There is a-rich abundance of greenery surrounding the path, which suggests a well--preserved natural environment, possibly a nature reserve or park. This setting-is great for a leisurely walk or a moment of reflection amidst nature.-===============================================================================-"""