Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Add support for open-sourced multi-modal model (#787)

Appointat4f289158e3db
diff --git a/camel/messages/base.py b/camel/messages/base.py
index f58895250..f0c428a2f 100644
--- a/camel/messages/base.py
+++ b/camel/messages/base.py
@@ -27,6 +27,7 @@ from camel.messages import (
)
from camel.prompts import CodePrompt, TextPrompt
from camel.types import (
+ MultiModalModelType,
OpenAIBackendRole,
OpenAIImageType,
OpenAIVisionDetailType,
@@ -46,14 +47,16 @@ class BaseMessage:
meta_dict (Optional[Dict[str, str]]): Additional metadata dictionary
for the message.
content (str): The content of the message.
- video_bytes (Optional[bytes]): Optional bytes of a video associated
- with the message. Default is None.
+ video_bytes (Optional[Union[bytes, List[bytes]]): Optional bytes
+ object or list of bytes objects associated with the message.
image_list (Optional[List[Image.Image]]): Optional list of PIL Image
objects associated with the message. Default is None.
image_detail (Literal["auto", "low", "high"]): Detail level of the
images associated with the message. Default is "auto".
video_detail (Literal["auto", "low", "high"]): Detail level of the
videos associated with the message. Default is "low".
+ video_type (Optional[str]): Optional type of the video associated
+ with the message. Default is None.
"""
role_name: str
@@ -61,10 +64,11 @@ class BaseMessage:
meta_dict: Optional[Dict[str, str]]
content: str
- video_bytes: Optional[bytes] = None
+ video_bytes: Optional[Union[bytes, List[bytes]]] = None
image_list: Optional[List[Image.Image]] = None
image_detail: Literal["auto", "low", "high"] = "auto"
video_detail: Literal["auto", "low", "high"] = "low"
+ video_type: Optional[str] = None
@classmethod
def make_user_message(
@@ -72,7 +76,7 @@ class BaseMessage:
role_name: str,
content: str,
meta_dict: Optional[Dict[str, str]] = None,
- video_bytes: Optional[bytes] = None,
+ video_bytes: Optional[Union[bytes, List[bytes]]] = None,
image_list: Optional[List[Image.Image]] = None,
image_detail: Union[
OpenAIVisionDetailType, str
@@ -80,6 +84,7 @@ class BaseMessage:
video_detail: Union[
OpenAIVisionDetailType, str
] = OpenAIVisionDetailType.LOW,
+ video_type: Optional[str] = None,
) -> "BaseMessage":
return cls(
role_name,
@@ -90,6 +95,7 @@ class BaseMessage:
image_list,
OpenAIVisionDetailType(image_detail).value,
OpenAIVisionDetailType(video_detail).value,
+ video_type,
)
@classmethod
@@ -98,7 +104,7 @@ class BaseMessage:
role_name: str,
content: str,
meta_dict: Optional[Dict[str, str]] = None,
- video_bytes: Optional[bytes] = None,
+ video_bytes: Optional[Union[bytes, List[bytes]]] = None,
image_list: Optional[List[Image.Image]] = None,
image_detail: Union[
OpenAIVisionDetailType, str
@@ -106,16 +112,18 @@ class BaseMessage:
video_detail: Union[
OpenAIVisionDetailType, str
] = OpenAIVisionDetailType.LOW,
+ video_type: Optional[str] = None,
) -> "BaseMessage":
return cls(
role_name,
- RoleType.ASSISTANT,
+ RoleType.USER,
meta_dict,
content,
video_bytes,
image_list,
OpenAIVisionDetailType(image_detail).value,
OpenAIVisionDetailType(video_detail).value,
+ video_type,
)
def create_new_instance(self, content: str) -> "BaseMessage":
@@ -249,7 +257,11 @@ class BaseMessage:
if role_at_backend == OpenAIBackendRole.SYSTEM:
return self.to_openai_system_message()
elif role_at_backend == OpenAIBackendRole.USER:
- return self.to_openai_user_message()
+ is_multimodal_message_type = isinstance(self.video_bytes, List)
+ if is_multimodal_message_type:
+ return self.to_multimodal_model_user_message()
+ else:
+ return self.to_openai_user_message()
elif role_at_backend == OpenAIBackendRole.ASSISTANT:
return self.to_openai_assistant_message()
else:
@@ -270,8 +282,8 @@ class BaseMessage:
Returns:
OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
"""
- hybird_content: List[Any] = []
- hybird_content.append(
+ hybrid_content: List[Any] = []
+ hybrid_content.append(
{
"type": "text",
"text": self.content,
@@ -300,7 +312,7 @@ class BaseMessage:
"utf-8"
)
image_prefix = f"data:image/{image_type};base64,"
- hybird_content.append(
+ hybrid_content.append(
{
"type": "image_url",
"image_url": {
@@ -311,6 +323,12 @@ class BaseMessage:
)
if self.video_bytes:
+ if not isinstance(self.video_bytes, bytes):
+ raise ValueError(
+ "Video bytes should be a bytes object for the OpenAI "
+ "model"
+ )
+
import imageio.v3 as iio
base64Frames: List[str] = []
@@ -359,12 +377,113 @@ class BaseMessage:
},
}
- hybird_content.append(item)
+ hybrid_content.append(item)
+
+ if len(hybrid_content) > 1:
+ return {
+ "role": "user",
+ "content": hybrid_content,
+ }
+ # This return just for str message
+ else:
+ return {
+ "role": "user",
+ "content": self.content,
+ }
+
+ def to_multimodal_model_user_message(self) -> OpenAIUserMessage:
+ r"""Converts the message to an :obj:`OpenAIUserMessage` object.
+
+ Returns:
+ OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
+ """
+ hybrid_content: List[Any] = [
+ {
+ "type": "text",
+ "text": self.content,
+ }
+ ]
+
+ if self.image_list and len(self.image_list) > 0:
+ for image in self.image_list:
+ if image.format is None:
+ raise ValueError(
+ f"Image's `format` is `None`, please "
+ f"transform the `PIL.Image.Image` to one of "
+ f"following supported formats, such as "
+ f"{list(MultiModalModelType)}"
+ )
+
+ if not isinstance(image, Image.Image):
+ raise ValueError(
+ "Image should be a PIL Image object for the "
+ "multimodal model"
+ )
+
+ image_type: str = image.format.lower()
+ if image_type not in MultiModalModelType:
+ raise ValueError(
+ f"Image type {image.format} "
+ f"is not supported by OpenAI vision model"
+ )
+ with io.BytesIO() as buffer:
+ image.save(fp=buffer, format=image.format)
+ encoded_image = base64.b64encode(buffer.getvalue()).decode(
+ "utf-8"
+ )
+ image_prefix = f"data:image/{image_type};base64,"
+ hybrid_content.append(
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": f"{image_prefix}{encoded_image}",
+ "detail": self.image_detail,
+ },
+ }
+ )
+
+ if self.video_bytes:
+ if not isinstance(self.video_bytes, List):
+ raise ValueError(
+ "Video bytes should be a list of bytes for the multimodal "
+ "model"
+ )
+
+ if self.video_type is None:
+ raise ValueError(
+ f"Video's `format` is `None`, please "
+ f"transform the `PIL.Image.Image` to one of "
+ f"following supported formats, such as "
+ f"{list(MultiModalModelType)}"
+ )
+ if self.video_type.lower() not in MultiModalModelType:
+ raise ValueError(
+ f"Video type {self.video_type} "
+ f"is not supported by multimodal model"
+ )
+
+ for video in self.video_bytes:
+ with io.BytesIO() as buffer:
+ buffer.write(video)
+ encoded_video = base64.b64encode(buffer.getvalue()).decode(
+ "utf-8"
+ )
+ video_prefix = f"data:video/{self.video_type.lower()};bytes,"
+
+ hybrid_content.append(
+ {
+ "type": "video_url",
+ "video_url": {
+ "url": f"{video_prefix}{encoded_video}",
+ "detail": self.video_detail,
+ },
+ }
+ )
- if len(hybird_content) > 1:
+ if len(hybrid_content) > 1:
return {
"role": "user",
- "content": hybird_content,
+ "content": hybrid_content,
}
# This return just for str message
else:
diff --git a/camel/models/__init__.py b/camel/models/__init__.py
index 5525e11e1..eed3cd2dc 100644
--- a/camel/models/__init__.py
+++ b/camel/models/__init__.py
@@ -16,6 +16,7 @@ from .azure_openai_model import AzureOpenAIModel
from .base_model import BaseModelBackend
from .gemini_model import GeminiModel
from .groq_model import GroqModel
+from .interlm_xcomposer_model import InternLMXComposerModel
from .litellm_model import LiteLLMModel
from .mistral_model import MistralModel
from .model_factory import ModelFactory
@@ -46,5 +47,6 @@ __all__ = [
'OllamaModel',
'VLLMModel',
'GeminiModel',
+ 'InternLMXComposerModel',
'OpenAICompatibilityModel',
]
diff --git a/camel/models/interlm_xcomposer_model.py b/camel/models/interlm_xcomposer_model.py
new file mode 100644
index 000000000..842dbf8b2
--- /dev/null
+++ b/camel/models/interlm_xcomposer_model.py
@@ -0,0 +1,325 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from openai import Stream
+
+from camel.messages import OpenAIMessage
+from camel.models import BaseModelBackend
+from camel.types import (
+ ChatCompletion,
+ ChatCompletionChunk,
+ ChatCompletionMessage,
+ Choice,
+ CompletionUsage,
+ ModelType,
+)
+from camel.utils import (
+ BaseTokenCounter,
+ OpenAITokenCounter,
+)
+
+
+# The model provides different operation modes/functionalities.
+# Refer to https://github.com/InternLM/InternLM-XComposer
+class OperationMode(Enum):
+ CHAT = "chat" # the normal mode for the model
+ WRITE_WEBPAGE = "write_webpage"
+ RESUME_TO_WEBPAGE = "resume_2_webpage"
+ WRITE_ARTICLE = "write_article"
+
+
+class InternLMXComposerModel(BaseModelBackend):
+ r"""Class for interace with OpenAI-API-compatible servers running
+ open-source InternLM-XComposer models.
+ """
+
+ def __init__(
+ self,
+ model_type: ModelType,
+ model_config_dict: Dict[str, Any],
+ ) -> None:
+ r"""Constructor for model backends of Open-source models.
+
+ Args:
+ model_type (ModelType): Model for which a backend is created.
+ model_config_dict (Dict[str, Any]): A dictionary that will
+ be fed into :obj:`openai.ChatCompletion.create()`.
+ """
+ super().__init__(
+ model_type,
+ model_config_dict,
+ api_key=None,
+ url=None,
+ token_counter=None,
+ )
+
+ from pathlib import Path
+
+ self.multimodal_src_directory: Path = Path(
+ model_config_dict.get("multimodal_src_path", "./examples/src")
+ )
+
+ # `model_path` is the path to the open-source model
+ model_path: Optional[str] = self.model_config_dict.get(
+ "model_path", None
+ )
+ if not model_path:
+ raise ValueError("Path to open-source model is not provided.")
+ self.model_path: str = model_path
+
+ self.model_name: str = self.model_path.split('/')[-1]
+
+ try:
+ # `operation_mode` is the operation mode of the open-source model
+ # Refer to https://github.com/InternLM/InternLM-XComposer
+ self.operation_mode = model_config_dict.get(
+ "operation_mode", OperationMode.CHAT
+ )
+ except ValueError:
+ valid_modes = [mode.value for mode in OperationMode]
+ raise ValueError(
+ "Invalid operation mode for open-source model "
+ f"backend. Choose from {valid_modes}"
+ )
+
+ import torch
+
+ model_kwargs = self.model_config_dict.get("model_kwargs", {})
+ model_kwargs["torch_dtype"] = torch.bfloat16
+ model_kwargs["trust_remote_code"] = True
+
+ from transformers import AutoModel, AutoTokenizer
+
+ self._client = (
+ AutoModel.from_pretrained(
+ self.model_name,
+ **model_kwargs,
+ )
+ .to(
+ model_kwargs.get("device") or "cuda"
+ if torch.cuda.is_available()
+ else "cpu"
+ ) # move the model to the specified device (GPU or CPU)
+ .eval() # set the model to evaluation mode
+ .half() # convert the model parameters to half-precision floating point (FP16). # noqa: E501
+ )
+
+ tokenizer_kwargs = self.model_config_dict.get("tokenizer_kwargs", {})
+ tokenizer_kwargs["trust_remote_code"] = True
+
+ self._tokenizer = AutoTokenizer.from_pretrained(
+ self.model_name, **tokenizer_kwargs
+ )
+ self._client.tokenizer = self._tokenizer # mypy: ignore[attr-defined]
+
+ # Replace `model_config_dict` with only the `model_kwargs`
+ self.model_config_dict = self.model_config_dict["model_kwargs"]
+
+ @property
+ def token_counter(self) -> BaseTokenCounter:
+ r"""Initialize the token counter for the model backend.
+
+ Returns:
+ BaseTokenCounter: The token counter following the model's
+ tokenization style.
+ """
+ if not self._token_counter:
+ self._token_counter = OpenAITokenCounter(ModelType.GPT_3_5_TURBO)
+ return self._token_counter
+
+ def run(
+ self,
+ messages: List[OpenAIMessage],
+ ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+ r"""Runs inference of OpenAI-API-style chat completion.
+
+ Args:
+ messages (List[OpenAIMessage]): Message list with the chat history
+ in OpenAI API format.
+
+ Returns:
+ Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+ `ChatCompletion` in the non-stream mode, or
+ `Stream[ChatCompletionChunk]` in the stream mode.
+ """
+
+ # Extract the text content from List[OpenAIMessage].
+ last_message = messages[-1]
+
+ if isinstance(last_message["content"], str):
+ # If the message does not contain image nor video:
+ message_content = last_message["content"]
+ elif isinstance(last_message["content"], list):
+ # If the message contains image or video:
+
+ import base64
+ import shutil
+
+ n_image = 0 # number of images
+ n_video = 0 # number of videos
+ image = [] # list of images local path
+
+ # Check and clean the images folder
+ if self.multimodal_src_directory.exists():
+ shutil.rmtree(self.multimodal_src_directory)
+ self.multimodal_src_directory.mkdir(parents=True, exist_ok=True)
+
+ # Get `message_content`
+ for item in last_message["content"]:
+ if item["type"] == "text":
+ message_content = item["text"]
+ elif item["type"] == "image_url":
+ # Get `image` list
+
+ n_image += 1
+ base64_string = item['image_url']['url'].split(',')[1]
+
+ # Convert the base64 string into images
+ # and save them as the local files.
+ image_data = base64.b64decode(base64_string)
+ image_file_path = (
+ self.multimodal_src_directory / f"image_{n_image}.jpg"
+ )
+
+ with open(image_file_path, 'wb') as file:
+ file.write(image_data)
+ image.append(str(image_file_path))
+ elif item["type"] == "video_url":
+ # Get `video` list
+
+ n_video += 1
+ base64_string = item['video_url']['url'].split(',')[1]
+
+ # Convert the base64 string into video
+ # and save them as the local files.
+ video_data = base64.b64decode(base64_string)
+ video_file_path = (
+ self.multimodal_src_directory / f"video_{n_video}.mp4"
+ )
+
+ with open(video_file_path, 'wb') as file:
+ file.write(video_data)
+ image.append(str(video_file_path))
+
+ else:
+ message_content = ""
+ image = []
+
+ import torch
+
+ torch.set_grad_enabled(False)
+
+ with torch.autocast(device_type='cuda', dtype=torch.float16):
+ if self.operation_mode == OperationMode.CHAT:
+ response_str, _ = ( # output param `his` is ignored
+ self._client.chat(
+ self._tokenizer,
+ message_content,
+ image,
+ do_sample=False,
+ num_beams=3,
+ use_meta=True,
+ )
+ )
+ elif self.operation_mode == OperationMode.WRITE_WEBPAGE:
+ response_str, _ = ( # output param `his` is ignored
+ self._client.write_webpage(
+ message_content,
+ image,
+ seed=202,
+ task="Instruction-aware webpage generation",
+ repetition_penalty=3.0,
+ )
+ )
+ elif self.operation_mode == OperationMode.RESUME_TO_WEBPAGE:
+ response_str, _ = ( # output param `his` is ignored
+ self._client.resume_to_webpage(
+ message_content,
+ image,
+ seed=202,
+ repetition_penalty=3.0,
+ )
+ )
+ elif self.operation_mode == OperationMode.WRITE_ARTICLE:
+ response_str, _ = ( # output param `his` is ignored
+ self._client.write_article(
+ message_content,
+ seed=8192,
+ )
+ )
+
+ # Adapt to OpenAI's response format.
+ response = ChatCompletion(
+ id="interlm-xcomposer-output-id",
+ choices=[
+ Choice(
+ finish_reason="stop",
+ index=0,
+ logprobs=None,
+ message=ChatCompletionMessage(
+ content=response_str,
+ role="assistant",
+ function_call=None,
+ tool_calls=None,
+ ),
+ ),
+ ],
+ created=0,
+ model=self.model_name,
+ object="chat.completion",
+ service_tier=None,
+ system_fingerprint="fp_interlm_xcomposer_system_fingerprint",
+ usage=CompletionUsage(
+ completion_tokens=0,
+ prompt_tokens=0,
+ total_tokens=0,
+ ),
+ )
+
+ return response
+
+ def check_model_config(self):
+ r"""Check whether the model configuration is valid for open-source
+ model backends.
+
+ Raises:
+ ValueError: If the model configuration dictionary contains any
+ unexpected arguments to OpenAI API, or it does not contain
+ :obj:`model_path`.
+ """
+ if "model_path" not in self.model_config_dict:
+ raise ValueError(
+ "Invalid configuration for open-source model backend with "
+ ":obj:`model_path` missing."
+ )
+
+ if "operation_mode" in self.model_config_dict:
+ orperation_mode = self.model_config_dict["operation_mode"]
+ if orperation_mode not in OperationMode:
+ raise ValueError(
+ "Invalid configuration for open-source model backend with "
+ ":obj:`operation_mode` not in OperationMode."
+ )
+
+ @property
+ def stream(self) -> bool:
+ r"""Returns whether the model is in stream mode,
+ which sends partial results each time.
+
+ Returns:
+ bool: Whether the model is in stream mode.
+ """
+ return self.model_config_dict.get('stream', False)
diff --git a/camel/models/model_factory.py b/camel/models/model_factory.py
index 772bb37d1..9756a0125 100644
--- a/camel/models/model_factory.py
+++ b/camel/models/model_factory.py
@@ -18,6 +18,7 @@ from camel.models.azure_openai_model import AzureOpenAIModel
from camel.models.base_model import BaseModelBackend
from camel.models.gemini_model import GeminiModel
from camel.models.groq_model import GroqModel
+from camel.models.interlm_xcomposer_model import InternLMXComposerModel
from camel.models.litellm_model import LiteLLMModel
from camel.models.mistral_model import MistralModel
from camel.models.ollama_model import OllamaModel
@@ -108,6 +109,9 @@ class ModelFactory:
model_class = VLLMModel
elif model_platform.is_litellm:
model_class = LiteLLMModel
+ elif model_platform.is_internlm:
+ model_class = InternLMXComposerModel
+ return model_class(model_type, model_config_dict)
elif model_platform.is_openai_compatibility_model:
model_class = OpenAICompatibilityModel
else:
diff --git a/camel/types/__init__.py b/camel/types/__init__.py
index 5228436ec..f55a9c101 100644
--- a/camel/types/__init__.py
+++ b/camel/types/__init__.py
@@ -16,6 +16,7 @@ from .enums import (
EmbeddingModelType,
ModelPlatformType,
ModelType,
+ MultiModalModelType,
OpenAIBackendRole,
OpenAIImageType,
OpenAIVisionDetailType,
@@ -63,6 +64,7 @@ __all__ = [
'OpenAIImageType',
'OpenAIVisionDetailType',
'OpenAPIName',
+ 'MultiModalModelType',
'ModelPlatformType',
'AudioModelType',
'VoiceType',
diff --git a/camel/types/enums.py b/camel/types/enums.py
index 90f68ef33..23b329a0b 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -415,6 +415,20 @@ class OpenAIVisionDetailType(Enum):
HIGH = "high"
+class MultiModalModelType(Enum, metaclass=OpenAIImageTypeMeta):
+ r"""Image types supported by the multimodal model."""
+
+ # Align with OpenAI's specifications
+ # https://platform.openai.com/docs/guides/vision
+ PNG = "png"
+ JPEG = "jpeg"
+ JPG = "jpg"
+ WEBP = "webp"
+ GIF = "gif"
+
+ MP4 = "mp4"
+
+
class StorageType(Enum):
MILVUS = "milvus"
QDRANT = "qdrant"
@@ -445,6 +459,7 @@ class ModelPlatformType(Enum):
VLLM = "vllm"
MISTRAL = "mistral"
OPENAI_COMPATIBILITY_MODEL = "openai-compatibility-model"
+ INTERNLM = "internlm"
@property
def is_openai(self) -> bool:
@@ -507,6 +522,11 @@ class ModelPlatformType(Enum):
r"""Returns whether this platform is Gemini."""
return self is ModelPlatformType.GEMINI
+ @property
+ def is_internlm(self) -> bool:
+ r"""Returns whether this platform is InternLM."""
+ return self in [ModelPlatformType.INTERNLM]
+
class AudioModelType(Enum):
TTS_1 = "tts-1"
diff --git a/examples/models/interlm_xcomposer_model_example.py b/examples/models/interlm_xcomposer_model_example.py
new file mode 100644
index 000000000..061f02ccf
--- /dev/null
+++ b/examples/models/interlm_xcomposer_model_example.py
@@ -0,0 +1,109 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from io import BytesIO
+
+import requests
+from PIL import Image
+
+from camel.agents import ChatAgent
+from camel.messages import BaseMessage
+from camel.models import ModelFactory
+from camel.models.interlm_xcomposer_model import OperationMode
+from camel.types import ModelPlatformType
+
+
+def load_image_from_url(url: str) -> Image.Image:
+ r"""Load an image from a given URL.
+
+ Args:
+ url (str): The url resource for image.
+
+ Returns:
+ Image: The `Image` of the image.
+ """
+ response = requests.get(url)
+ if response.status_code != 200:
+ raise ValueError(f"Failed to load image from URL: {url}")
+ image = Image.open(BytesIO(response.content))
+ return image
+
+
+def load_video_from_url(url: str) -> bytes:
+ r"""Load video data from a given URL.
+
+ Args:
+ url (str): The url resource for video.
+
+ Returns:
+ bytes: The byte format of the video.
+ """
+ response = requests.get(url)
+ if response.status_code != 200:
+ raise ValueError(f"Failed to load video from URL: {url}")
+ return response.content
+
+
+# Get the resources from the urls.
+video_url = 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4'
+video_bytes = load_video_from_url(video_url)
+video_bytes_list = [video_bytes]
+image_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
+image = load_image_from_url(image_url)
+images = [image, image]
+
+
+ollama_model = ModelFactory.create(
+ model_platform=ModelPlatformType.INTERNLM,
+ model_type="internlm-xcomposer2d5-7b",
+ model_config_dict={
+ "model_path": "internlm/internlm-xcomposer2d5-7b",
+ # InterLM XComposer provides different operation modes.
+ "operation_mode": OperationMode.CHAT,
+ "model_kwargs": { # use `AutoModel` config in `transformers` library
+ "cache_dir": None,
+ },
+ },
+)
+
+
+assistant_sys_msg = BaseMessage.make_assistant_message(
+ role_name="Assistant",
+ content="You are a helpful assistant.",
+)
+
+
+agent = ChatAgent(assistant_sys_msg, model=ollama_model, token_limit=4096)
+
+
+user_msg = BaseMessage.make_user_message(
+ role_name="User",
+ content="What did you see in the images?",
+ video_bytes=video_bytes_list,
+ video_type="mp4",
+ image_list=images,
+)
+assistant_response = agent.step(user_msg)
+print(assistant_response.msg.content)
+
+"""
+===============================================================================
+The image shows a serene natural landscape featuring a wooden boardwalk
+stretching straight through a lush, vibrant green meadow. The scene is set
+against a beautifully dynamic sky with patches of fluffy white clouds. The view
+is tranquil and inviting, reminiscent of a peaceful, rural setting. There is a
+rich abundance of greenery surrounding the path, which suggests a well-
+preserved natural environment, possibly a nature reserve or park. This setting
+is great for a leisurely walk or a moment of reflection amidst nature.
+===============================================================================
+"""