Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Revert "feat: Add support for open-sourced multi-modal model (#787)" (#876)

Wendong-Fanb21279fc29bf
diff --git a/camel/messages/base.py b/camel/messages/base.py
index f0c428a2f..f58895250 100644
--- a/camel/messages/base.py
+++ b/camel/messages/base.py
@@ -27,7 +27,6 @@ from camel.messages import (
)
from camel.prompts import CodePrompt, TextPrompt
from camel.types import (
- MultiModalModelType,
OpenAIBackendRole,
OpenAIImageType,
OpenAIVisionDetailType,
@@ -47,16 +46,14 @@ class BaseMessage:
meta_dict (Optional[Dict[str, str]]): Additional metadata dictionary
for the message.
content (str): The content of the message.
- video_bytes (Optional[Union[bytes, List[bytes]]): Optional bytes
- object or list of bytes objects associated with the message.
+ video_bytes (Optional[bytes]): Optional bytes of a video associated
+ with the message. Default is None.
image_list (Optional[List[Image.Image]]): Optional list of PIL Image
objects associated with the message. Default is None.
image_detail (Literal["auto", "low", "high"]): Detail level of the
images associated with the message. Default is "auto".
video_detail (Literal["auto", "low", "high"]): Detail level of the
videos associated with the message. Default is "low".
- video_type (Optional[str]): Optional type of the video associated
- with the message. Default is None.
"""
role_name: str
@@ -64,11 +61,10 @@ class BaseMessage:
meta_dict: Optional[Dict[str, str]]
content: str
- video_bytes: Optional[Union[bytes, List[bytes]]] = None
+ video_bytes: Optional[bytes] = None
image_list: Optional[List[Image.Image]] = None
image_detail: Literal["auto", "low", "high"] = "auto"
video_detail: Literal["auto", "low", "high"] = "low"
- video_type: Optional[str] = None
@classmethod
def make_user_message(
@@ -76,7 +72,7 @@ class BaseMessage:
role_name: str,
content: str,
meta_dict: Optional[Dict[str, str]] = None,
- video_bytes: Optional[Union[bytes, List[bytes]]] = None,
+ video_bytes: Optional[bytes] = None,
image_list: Optional[List[Image.Image]] = None,
image_detail: Union[
OpenAIVisionDetailType, str
@@ -84,7 +80,6 @@ class BaseMessage:
video_detail: Union[
OpenAIVisionDetailType, str
] = OpenAIVisionDetailType.LOW,
- video_type: Optional[str] = None,
) -> "BaseMessage":
return cls(
role_name,
@@ -95,7 +90,6 @@ class BaseMessage:
image_list,
OpenAIVisionDetailType(image_detail).value,
OpenAIVisionDetailType(video_detail).value,
- video_type,
)
@classmethod
@@ -104,7 +98,7 @@ class BaseMessage:
role_name: str,
content: str,
meta_dict: Optional[Dict[str, str]] = None,
- video_bytes: Optional[Union[bytes, List[bytes]]] = None,
+ video_bytes: Optional[bytes] = None,
image_list: Optional[List[Image.Image]] = None,
image_detail: Union[
OpenAIVisionDetailType, str
@@ -112,18 +106,16 @@ class BaseMessage:
video_detail: Union[
OpenAIVisionDetailType, str
] = OpenAIVisionDetailType.LOW,
- video_type: Optional[str] = None,
) -> "BaseMessage":
return cls(
role_name,
- RoleType.USER,
+ RoleType.ASSISTANT,
meta_dict,
content,
video_bytes,
image_list,
OpenAIVisionDetailType(image_detail).value,
OpenAIVisionDetailType(video_detail).value,
- video_type,
)
def create_new_instance(self, content: str) -> "BaseMessage":
@@ -257,11 +249,7 @@ class BaseMessage:
if role_at_backend == OpenAIBackendRole.SYSTEM:
return self.to_openai_system_message()
elif role_at_backend == OpenAIBackendRole.USER:
- is_multimodal_message_type = isinstance(self.video_bytes, List)
- if is_multimodal_message_type:
- return self.to_multimodal_model_user_message()
- else:
- return self.to_openai_user_message()
+ return self.to_openai_user_message()
elif role_at_backend == OpenAIBackendRole.ASSISTANT:
return self.to_openai_assistant_message()
else:
@@ -282,8 +270,8 @@ class BaseMessage:
Returns:
OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
"""
- hybrid_content: List[Any] = []
- hybrid_content.append(
+ hybird_content: List[Any] = []
+ hybird_content.append(
{
"type": "text",
"text": self.content,
@@ -312,7 +300,7 @@ class BaseMessage:
"utf-8"
)
image_prefix = f"data:image/{image_type};base64,"
- hybrid_content.append(
+ hybird_content.append(
{
"type": "image_url",
"image_url": {
@@ -323,12 +311,6 @@ class BaseMessage:
)
if self.video_bytes:
- if not isinstance(self.video_bytes, bytes):
- raise ValueError(
- "Video bytes should be a bytes object for the OpenAI "
- "model"
- )
-
import imageio.v3 as iio
base64Frames: List[str] = []
@@ -377,113 +359,12 @@ class BaseMessage:
},
}
- hybrid_content.append(item)
-
- if len(hybrid_content) > 1:
- return {
- "role": "user",
- "content": hybrid_content,
- }
- # This return just for str message
- else:
- return {
- "role": "user",
- "content": self.content,
- }
-
- def to_multimodal_model_user_message(self) -> OpenAIUserMessage:
- r"""Converts the message to an :obj:`OpenAIUserMessage` object.
-
- Returns:
- OpenAIUserMessage: The converted :obj:`OpenAIUserMessage` object.
- """
- hybrid_content: List[Any] = [
- {
- "type": "text",
- "text": self.content,
- }
- ]
-
- if self.image_list and len(self.image_list) > 0:
- for image in self.image_list:
- if image.format is None:
- raise ValueError(
- f"Image's `format` is `None`, please "
- f"transform the `PIL.Image.Image` to one of "
- f"following supported formats, such as "
- f"{list(MultiModalModelType)}"
- )
-
- if not isinstance(image, Image.Image):
- raise ValueError(
- "Image should be a PIL Image object for the "
- "multimodal model"
- )
-
- image_type: str = image.format.lower()
- if image_type not in MultiModalModelType:
- raise ValueError(
- f"Image type {image.format} "
- f"is not supported by OpenAI vision model"
- )
- with io.BytesIO() as buffer:
- image.save(fp=buffer, format=image.format)
- encoded_image = base64.b64encode(buffer.getvalue()).decode(
- "utf-8"
- )
- image_prefix = f"data:image/{image_type};base64,"
- hybrid_content.append(
- {
- "type": "image_url",
- "image_url": {
- "url": f"{image_prefix}{encoded_image}",
- "detail": self.image_detail,
- },
- }
- )
-
- if self.video_bytes:
- if not isinstance(self.video_bytes, List):
- raise ValueError(
- "Video bytes should be a list of bytes for the multimodal "
- "model"
- )
-
- if self.video_type is None:
- raise ValueError(
- f"Video's `format` is `None`, please "
- f"transform the `PIL.Image.Image` to one of "
- f"following supported formats, such as "
- f"{list(MultiModalModelType)}"
- )
- if self.video_type.lower() not in MultiModalModelType:
- raise ValueError(
- f"Video type {self.video_type} "
- f"is not supported by multimodal model"
- )
-
- for video in self.video_bytes:
- with io.BytesIO() as buffer:
- buffer.write(video)
- encoded_video = base64.b64encode(buffer.getvalue()).decode(
- "utf-8"
- )
- video_prefix = f"data:video/{self.video_type.lower()};bytes,"
-
- hybrid_content.append(
- {
- "type": "video_url",
- "video_url": {
- "url": f"{video_prefix}{encoded_video}",
- "detail": self.video_detail,
- },
- }
- )
+ hybird_content.append(item)
- if len(hybrid_content) > 1:
+ if len(hybird_content) > 1:
return {
"role": "user",
- "content": hybrid_content,
+ "content": hybird_content,
}
# This return just for str message
else:
diff --git a/camel/models/__init__.py b/camel/models/__init__.py
index 63ae17381..59c03cd95 100644
--- a/camel/models/__init__.py
+++ b/camel/models/__init__.py
@@ -16,7 +16,6 @@ from .azure_openai_model import AzureOpenAIModel
from .base_model import BaseModelBackend
from .gemini_model import GeminiModel
from .groq_model import GroqModel
-from .interlm_xcomposer_model import InternLMXComposerModel
from .litellm_model import LiteLLMModel
from .mistral_model import MistralModel
from .model_factory import ModelFactory
@@ -49,7 +48,6 @@ __all__ = [
'OllamaModel',
'VLLMModel',
'GeminiModel',
- 'InternLMXComposerModel',
'OpenAICompatibilityModel',
'SambaModel',
'TogetherAIModel',
diff --git a/camel/models/interlm_xcomposer_model.py b/camel/models/interlm_xcomposer_model.py
deleted file mode 100644
index 842dbf8b2..000000000
--- a/camel/models/interlm_xcomposer_model.py
+++ /dev/null
@@ -1,325 +0,0 @@
-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from enum import Enum
-from typing import Any, Dict, List, Optional, Union
-
-from openai import Stream
-
-from camel.messages import OpenAIMessage
-from camel.models import BaseModelBackend
-from camel.types import (
- ChatCompletion,
- ChatCompletionChunk,
- ChatCompletionMessage,
- Choice,
- CompletionUsage,
- ModelType,
-)
-from camel.utils import (
- BaseTokenCounter,
- OpenAITokenCounter,
-)
-
-
-# The model provides different operation modes/functionalities.
-# Refer to https://github.com/InternLM/InternLM-XComposer
-class OperationMode(Enum):
- CHAT = "chat" # the normal mode for the model
- WRITE_WEBPAGE = "write_webpage"
- RESUME_TO_WEBPAGE = "resume_2_webpage"
- WRITE_ARTICLE = "write_article"
-
-
-class InternLMXComposerModel(BaseModelBackend):
- r"""Class for interace with OpenAI-API-compatible servers running
- open-source InternLM-XComposer models.
- """
-
- def __init__(
- self,
- model_type: ModelType,
- model_config_dict: Dict[str, Any],
- ) -> None:
- r"""Constructor for model backends of Open-source models.
-
- Args:
- model_type (ModelType): Model for which a backend is created.
- model_config_dict (Dict[str, Any]): A dictionary that will
- be fed into :obj:`openai.ChatCompletion.create()`.
- """
- super().__init__(
- model_type,
- model_config_dict,
- api_key=None,
- url=None,
- token_counter=None,
- )
-
- from pathlib import Path
-
- self.multimodal_src_directory: Path = Path(
- model_config_dict.get("multimodal_src_path", "./examples/src")
- )
-
- # `model_path` is the path to the open-source model
- model_path: Optional[str] = self.model_config_dict.get(
- "model_path", None
- )
- if not model_path:
- raise ValueError("Path to open-source model is not provided.")
- self.model_path: str = model_path
-
- self.model_name: str = self.model_path.split('/')[-1]
-
- try:
- # `operation_mode` is the operation mode of the open-source model
- # Refer to https://github.com/InternLM/InternLM-XComposer
- self.operation_mode = model_config_dict.get(
- "operation_mode", OperationMode.CHAT
- )
- except ValueError:
- valid_modes = [mode.value for mode in OperationMode]
- raise ValueError(
- "Invalid operation mode for open-source model "
- f"backend. Choose from {valid_modes}"
- )
-
- import torch
-
- model_kwargs = self.model_config_dict.get("model_kwargs", {})
- model_kwargs["torch_dtype"] = torch.bfloat16
- model_kwargs["trust_remote_code"] = True
-
- from transformers import AutoModel, AutoTokenizer
-
- self._client = (
- AutoModel.from_pretrained(
- self.model_name,
- **model_kwargs,
- )
- .to(
- model_kwargs.get("device") or "cuda"
- if torch.cuda.is_available()
- else "cpu"
- ) # move the model to the specified device (GPU or CPU)
- .eval() # set the model to evaluation mode
- .half() # convert the model parameters to half-precision floating point (FP16). # noqa: E501
- )
-
- tokenizer_kwargs = self.model_config_dict.get("tokenizer_kwargs", {})
- tokenizer_kwargs["trust_remote_code"] = True
-
- self._tokenizer = AutoTokenizer.from_pretrained(
- self.model_name, **tokenizer_kwargs
- )
- self._client.tokenizer = self._tokenizer # mypy: ignore[attr-defined]
-
- # Replace `model_config_dict` with only the `model_kwargs`
- self.model_config_dict = self.model_config_dict["model_kwargs"]
-
- @property
- def token_counter(self) -> BaseTokenCounter:
- r"""Initialize the token counter for the model backend.
-
- Returns:
- BaseTokenCounter: The token counter following the model's
- tokenization style.
- """
- if not self._token_counter:
- self._token_counter = OpenAITokenCounter(ModelType.GPT_3_5_TURBO)
- return self._token_counter
-
- def run(
- self,
- messages: List[OpenAIMessage],
- ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
- r"""Runs inference of OpenAI-API-style chat completion.
-
- Args:
- messages (List[OpenAIMessage]): Message list with the chat history
- in OpenAI API format.
-
- Returns:
- Union[ChatCompletion, Stream[ChatCompletionChunk]]:
- `ChatCompletion` in the non-stream mode, or
- `Stream[ChatCompletionChunk]` in the stream mode.
- """
-
- # Extract the text content from List[OpenAIMessage].
- last_message = messages[-1]
-
- if isinstance(last_message["content"], str):
- # If the message does not contain image nor video:
- message_content = last_message["content"]
- elif isinstance(last_message["content"], list):
- # If the message contains image or video:
-
- import base64
- import shutil
-
- n_image = 0 # number of images
- n_video = 0 # number of videos
- image = [] # list of images local path
-
- # Check and clean the images folder
- if self.multimodal_src_directory.exists():
- shutil.rmtree(self.multimodal_src_directory)
- self.multimodal_src_directory.mkdir(parents=True, exist_ok=True)
-
- # Get `message_content`
- for item in last_message["content"]:
- if item["type"] == "text":
- message_content = item["text"]
- elif item["type"] == "image_url":
- # Get `image` list
-
- n_image += 1
- base64_string = item['image_url']['url'].split(',')[1]
-
- # Convert the base64 string into images
- # and save them as the local files.
- image_data = base64.b64decode(base64_string)
- image_file_path = (
- self.multimodal_src_directory / f"image_{n_image}.jpg"
- )
-
- with open(image_file_path, 'wb') as file:
- file.write(image_data)
- image.append(str(image_file_path))
- elif item["type"] == "video_url":
- # Get `video` list
-
- n_video += 1
- base64_string = item['video_url']['url'].split(',')[1]
-
- # Convert the base64 string into video
- # and save them as the local files.
- video_data = base64.b64decode(base64_string)
- video_file_path = (
- self.multimodal_src_directory / f"video_{n_video}.mp4"
- )
-
- with open(video_file_path, 'wb') as file:
- file.write(video_data)
- image.append(str(video_file_path))
-
- else:
- message_content = ""
- image = []
-
- import torch
-
- torch.set_grad_enabled(False)
-
- with torch.autocast(device_type='cuda', dtype=torch.float16):
- if self.operation_mode == OperationMode.CHAT:
- response_str, _ = ( # output param `his` is ignored
- self._client.chat(
- self._tokenizer,
- message_content,
- image,
- do_sample=False,
- num_beams=3,
- use_meta=True,
- )
- )
- elif self.operation_mode == OperationMode.WRITE_WEBPAGE:
- response_str, _ = ( # output param `his` is ignored
- self._client.write_webpage(
- message_content,
- image,
- seed=202,
- task="Instruction-aware webpage generation",
- repetition_penalty=3.0,
- )
- )
- elif self.operation_mode == OperationMode.RESUME_TO_WEBPAGE:
- response_str, _ = ( # output param `his` is ignored
- self._client.resume_to_webpage(
- message_content,
- image,
- seed=202,
- repetition_penalty=3.0,
- )
- )
- elif self.operation_mode == OperationMode.WRITE_ARTICLE:
- response_str, _ = ( # output param `his` is ignored
- self._client.write_article(
- message_content,
- seed=8192,
- )
- )
-
- # Adapt to OpenAI's response format.
- response = ChatCompletion(
- id="interlm-xcomposer-output-id",
- choices=[
- Choice(
- finish_reason="stop",
- index=0,
- logprobs=None,
- message=ChatCompletionMessage(
- content=response_str,
- role="assistant",
- function_call=None,
- tool_calls=None,
- ),
- ),
- ],
- created=0,
- model=self.model_name,
- object="chat.completion",
- service_tier=None,
- system_fingerprint="fp_interlm_xcomposer_system_fingerprint",
- usage=CompletionUsage(
- completion_tokens=0,
- prompt_tokens=0,
- total_tokens=0,
- ),
- )
-
- return response
-
- def check_model_config(self):
- r"""Check whether the model configuration is valid for open-source
- model backends.
-
- Raises:
- ValueError: If the model configuration dictionary contains any
- unexpected arguments to OpenAI API, or it does not contain
- :obj:`model_path`.
- """
- if "model_path" not in self.model_config_dict:
- raise ValueError(
- "Invalid configuration for open-source model backend with "
- ":obj:`model_path` missing."
- )
-
- if "operation_mode" in self.model_config_dict:
- orperation_mode = self.model_config_dict["operation_mode"]
- if orperation_mode not in OperationMode:
- raise ValueError(
- "Invalid configuration for open-source model backend with "
- ":obj:`operation_mode` not in OperationMode."
- )
-
- @property
- def stream(self) -> bool:
- r"""Returns whether the model is in stream mode,
- which sends partial results each time.
-
- Returns:
- bool: Whether the model is in stream mode.
- """
- return self.model_config_dict.get('stream', False)
diff --git a/camel/models/model_factory.py b/camel/models/model_factory.py
index 2c21397be..e51ff3175 100644
--- a/camel/models/model_factory.py
+++ b/camel/models/model_factory.py
@@ -18,7 +18,6 @@ from camel.models.azure_openai_model import AzureOpenAIModel
from camel.models.base_model import BaseModelBackend
from camel.models.gemini_model import GeminiModel
from camel.models.groq_model import GroqModel
-from camel.models.interlm_xcomposer_model import InternLMXComposerModel
from camel.models.litellm_model import LiteLLMModel
from camel.models.mistral_model import MistralModel
from camel.models.ollama_model import OllamaModel
@@ -113,9 +112,6 @@ class ModelFactory:
model_class = VLLMModel
elif model_platform.is_litellm:
model_class = LiteLLMModel
- elif model_platform.is_internlm:
- model_class = InternLMXComposerModel
- return model_class(model_type, model_config_dict)
elif model_platform.is_openai_compatibility_model:
model_class = OpenAICompatibilityModel
elif model_platform.is_together:
diff --git a/camel/types/__init__.py b/camel/types/__init__.py
index f55a9c101..5228436ec 100644
--- a/camel/types/__init__.py
+++ b/camel/types/__init__.py
@@ -16,7 +16,6 @@ from .enums import (
EmbeddingModelType,
ModelPlatformType,
ModelType,
- MultiModalModelType,
OpenAIBackendRole,
OpenAIImageType,
OpenAIVisionDetailType,
@@ -64,7 +63,6 @@ __all__ = [
'OpenAIImageType',
'OpenAIVisionDetailType',
'OpenAPIName',
- 'MultiModalModelType',
'ModelPlatformType',
'AudioModelType',
'VoiceType',
diff --git a/camel/types/enums.py b/camel/types/enums.py
index 55d469373..fa6681675 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -429,20 +429,6 @@ class OpenAIVisionDetailType(Enum):
HIGH = "high"
-class MultiModalModelType(Enum, metaclass=OpenAIImageTypeMeta):
- r"""Image types supported by the multimodal model."""
-
- # Align with OpenAI's specifications
- # https://platform.openai.com/docs/guides/vision
- PNG = "png"
- JPEG = "jpeg"
- JPG = "jpg"
- WEBP = "webp"
- GIF = "gif"
-
- MP4 = "mp4"
-
-
class StorageType(Enum):
MILVUS = "milvus"
QDRANT = "qdrant"
@@ -475,7 +461,6 @@ class ModelPlatformType(Enum):
TOGETHER = "together"
OPENAI_COMPATIBILITY_MODEL = "openai-compatibility-model"
SAMBA = "samba-nova"
- INTERNLM = "internlm"
@property
def is_openai(self) -> bool:
@@ -548,11 +533,6 @@ class ModelPlatformType(Enum):
r"""Returns whether this platform is Samba Nova."""
return self is ModelPlatformType.SAMBA
- @property
- def is_internlm(self) -> bool:
- r"""Returns whether this platform is InternLM."""
- return self in [ModelPlatformType.INTERNLM]
-
class AudioModelType(Enum):
TTS_1 = "tts-1"
diff --git a/examples/models/interlm_xcomposer_model_example.py b/examples/models/interlm_xcomposer_model_example.py
deleted file mode 100644
index 061f02ccf..000000000
--- a/examples/models/interlm_xcomposer_model_example.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from io import BytesIO
-
-import requests
-from PIL import Image
-
-from camel.agents import ChatAgent
-from camel.messages import BaseMessage
-from camel.models import ModelFactory
-from camel.models.interlm_xcomposer_model import OperationMode
-from camel.types import ModelPlatformType
-
-
-def load_image_from_url(url: str) -> Image.Image:
- r"""Load an image from a given URL.
-
- Args:
- url (str): The url resource for image.
-
- Returns:
- Image: The `Image` of the image.
- """
- response = requests.get(url)
- if response.status_code != 200:
- raise ValueError(f"Failed to load image from URL: {url}")
- image = Image.open(BytesIO(response.content))
- return image
-
-
-def load_video_from_url(url: str) -> bytes:
- r"""Load video data from a given URL.
-
- Args:
- url (str): The url resource for video.
-
- Returns:
- bytes: The byte format of the video.
- """
- response = requests.get(url)
- if response.status_code != 200:
- raise ValueError(f"Failed to load video from URL: {url}")
- return response.content
-
-
-# Get the resources from the urls.
-video_url = 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4'
-video_bytes = load_video_from_url(video_url)
-video_bytes_list = [video_bytes]
-image_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'
-image = load_image_from_url(image_url)
-images = [image, image]
-
-
-ollama_model = ModelFactory.create(
- model_platform=ModelPlatformType.INTERNLM,
- model_type="internlm-xcomposer2d5-7b",
- model_config_dict={
- "model_path": "internlm/internlm-xcomposer2d5-7b",
- # InterLM XComposer provides different operation modes.
- "operation_mode": OperationMode.CHAT,
- "model_kwargs": { # use `AutoModel` config in `transformers` library
- "cache_dir": None,
- },
- },
-)
-
-
-assistant_sys_msg = BaseMessage.make_assistant_message(
- role_name="Assistant",
- content="You are a helpful assistant.",
-)
-
-
-agent = ChatAgent(assistant_sys_msg, model=ollama_model, token_limit=4096)
-
-
-user_msg = BaseMessage.make_user_message(
- role_name="User",
- content="What did you see in the images?",
- video_bytes=video_bytes_list,
- video_type="mp4",
- image_list=images,
-)
-assistant_response = agent.step(user_msg)
-print(assistant_response.msg.content)
-
-"""
-===============================================================================
-The image shows a serene natural landscape featuring a wooden boardwalk
-stretching straight through a lush, vibrant green meadow. The scene is set
-against a beautifully dynamic sky with patches of fluffy white clouds. The view
-is tranquil and inviting, reminiscent of a peaceful, rural setting. There is a
-rich abundance of greenery surrounding the path, which suggests a well-
-preserved natural environment, possibly a nature reserve or park. This setting
-is great for a leisurely walk or a moment of reflection amidst nature.
-===============================================================================
-"""