Skip to content

Building AI agents, atomically

Add typed VideoURL multimodal support

Eigenwisec80c7520a703
diff --git a/atomic-agents/atomic_agents/__init__.py b/atomic-agents/atomic_agents/__init__.py
index 7e54296..c4ed2d2 100644
--- a/atomic-agents/atomic_agents/__init__.py
+++ b/atomic-agents/atomic_agents/__init__.py
@@ -4,7 +4,7 @@ Atomic Agents - A modular framework for building AI agents.
# Core exports - base classes only
from .agents.atomic_agent import AtomicAgent, AgentConfig, BasicChatInputSchema, BasicChatOutputSchema
-from .base import BaseIOSchema, BaseTool, BaseToolConfig
+from .base import BaseIOSchema, BaseTool, BaseToolConfig, VideoURL
# Version info - read from pyproject.toml via package metadata
from importlib.metadata import version as _version
@@ -19,4 +19,5 @@ __all__ = [
"BaseIOSchema",
"BaseTool",
"BaseToolConfig",
+ "VideoURL",
]
diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.py
index d2ccfe6..801026b 100644
--- a/atomic-agents/atomic_agents/agents/atomic_agent.py
+++ b/atomic-agents/atomic_agents/agents/atomic_agent.py
@@ -431,7 +431,8 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
This method converts instructor multimodal objects (Image, Audio, PDF) to the
OpenAI format that LiteLLM's token counter expects. Text content is also
- converted to the proper multimodal text format when mixed with media.
+ converted to the proper multimodal text format when mixed with media. Video
+ content parts become text placeholders because LiteLLM cannot count them.
Returns:
List[Dict[str, Any]]: History messages in LiteLLM-compatible format.
@@ -462,6 +463,10 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:
f"Using placeholder for estimation."
)
serialized_content.append({"type": "text", "text": f"[{media_type.lower()} content]"})
+ elif isinstance(item, dict) and item.get("type") == "video_url":
+ # LiteLLM's token counter raises on video_url parts, so use a
+ # placeholder for estimation, same as failed media serialization.
+ serialized_content.append({"type": "text", "text": "[video content]"})
else:
# Unknown type - convert to string
serialized_content.append({"type": "text", "text": str(item)})
diff --git a/atomic-agents/atomic_agents/base/__init__.py b/atomic-agents/atomic_agents/base/__init__.py
index 9bbe931..18d1d5a 100644
--- a/atomic-agents/atomic_agents/base/__init__.py
+++ b/atomic-agents/atomic_agents/base/__init__.py
@@ -4,6 +4,7 @@ from .base_io_schema import BaseIOSchema
from .base_tool import BaseTool, BaseToolConfig
from .base_resource import BaseResource, BaseResourceConfig
from .base_prompt import BasePrompt, BasePromptConfig
+from .multimodal import VideoURL
__all__ = [
"BaseIOSchema",
@@ -13,4 +14,5 @@ __all__ = [
"BaseResourceConfig",
"BasePrompt",
"BasePromptConfig",
+ "VideoURL",
]
diff --git a/atomic-agents/atomic_agents/base/multimodal.py b/atomic-agents/atomic_agents/base/multimodal.py
new file mode 100644
index 0000000..69f6d4b
--- /dev/null
+++ b/atomic-agents/atomic_agents/base/multimodal.py
@@ -0,0 +1,40 @@
+"""Multimodal content types that Instructor does not provide."""
+
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class VideoURL(BaseModel):
+ """
+ Video reference sent to the LLM as an OpenAI-compatible ``video_url`` content part.
+
+ Instructor ships Image, Audio, and PDF types but no video type
+ (see https://github.com/567-labs/instructor/discussions/2520), so this class fills
+ the gap for providers that accept ``video_url`` content parts, such as MiniMax
+ and Qwen-VL.
+
+ Attributes:
+ url (str): HTTP(S) or data: URL of the video.
+ fps (Optional[float]): Frame sampling rate, for providers that accept it.
+ detail (Optional[str]): Detail level, for providers that accept it.
+ """
+
+ url: str = Field(..., description="HTTP(S) or data: URL of the video.")
+ fps: Optional[float] = Field(default=None, description="Frame sampling rate, for providers that accept it.")
+ detail: Optional[str] = Field(default=None, description="Detail level, for providers that accept it.")
+
+ def to_openai(self) -> Dict[str, Any]:
+ """
+ Build the OpenAI-compatible content part for this video.
+
+ Returns:
+ Dict[str, Any]: A ``{"type": "video_url", "video_url": {...}}`` content part,
+ omitting optional parameters that were not set.
+ """
+ video_url: Dict[str, Any] = {"url": self.url}
+ if self.fps is not None:
+ video_url["fps"] = self.fps
+ if self.detail is not None:
+ video_url["detail"] = self.detail
+ return {"type": "video_url", "video_url": video_url}
diff --git a/atomic-agents/atomic_agents/context/chat_history.py b/atomic-agents/atomic_agents/context/chat_history.py
index e4fbf91..d4ec83a 100644
--- a/atomic-agents/atomic_agents/context/chat_history.py
+++ b/atomic-agents/atomic_agents/context/chat_history.py
@@ -2,16 +2,21 @@ import json
import uuid
from enum import Enum
from pathlib import Path
-from typing import Dict, List, Optional, Type
+from typing import Dict, List, Optional, Tuple, Type, Union
from instructor.processing.multimodal import PDF, Image, Audio
from pydantic import BaseModel, Field
from atomic_agents.base.base_io_schema import BaseIOSchema
+from atomic_agents.base.multimodal import VideoURL
from atomic_agents.context.base_chat_history import BaseChatHistory
INSTRUCTOR_MULTIMODAL_TYPES = (Image, Audio, PDF)
+MULTIMODAL_TYPES = INSTRUCTOR_MULTIMODAL_TYPES + (VideoURL,)
+
+MultimodalObject = Union[Image, Audio, PDF, VideoURL]
+ExcludeSpec = Union[Dict, bool, None]
class Message(BaseModel):
@@ -109,10 +114,16 @@ class ChatHistory(BaseChatHistory):
if multimodal_objects:
processed_content = []
- content_json = input_content.model_dump_json(exclude=exclude_spec)
+ excluded_fields = exclude_spec if isinstance(exclude_spec, dict) else None
+ content_json = input_content.model_dump_json(exclude=excluded_fields)
if content_json and content_json != "{}":
processed_content.append(content_json)
- processed_content.extend(multimodal_objects)
+ # Instructor forwards dict content parts to the provider unchanged but
+ # raises on object types it does not know, so VideoURL is converted here.
+ processed_content.extend(
+ multimodal_object.to_openai() if isinstance(multimodal_object, VideoURL) else multimodal_object
+ for multimodal_object in multimodal_objects
+ )
history.append({"role": message.role, "content": processed_content})
else:
content_json = input_content.model_dump_json()
@@ -121,13 +132,14 @@ class ChatHistory(BaseChatHistory):
return history
@staticmethod
- def _extract_multimodal_info(obj):
+ def _extract_multimodal_info(obj) -> Tuple[List[MultimodalObject], ExcludeSpec]:
"""
Recursively extract multimodal objects and build a Pydantic-compatible exclude spec.
- Walks the object tree to find all Instructor multimodal types (Image, Audio, PDF)
- at any nesting depth, collecting them into a flat list and building an exclude
- specification that can be passed to model_dump_json(exclude=...).
+ Walks the object tree to find all multimodal types (Instructor's Image, Audio,
+ and PDF, plus VideoURL) at any nesting depth, collecting them into a flat list
+ and building an exclude specification that can be passed to
+ model_dump_json(exclude=...).
Args:
obj: The object to inspect (BaseIOSchema, list, dict, or primitive).
@@ -137,7 +149,7 @@ class ChatHistory(BaseChatHistory):
- multimodal_objects: flat list of all multimodal objects found
- exclude_spec: Pydantic exclude dict, True (exclude entirely), or None
"""
- if isinstance(obj, INSTRUCTOR_MULTIMODAL_TYPES):
+ if isinstance(obj, MULTIMODAL_TYPES):
return [obj], True
if hasattr(obj, "__class__") and hasattr(obj.__class__, "model_fields"):
diff --git a/atomic-agents/tests/agents/test_atomic_agent.py b/atomic-agents/tests/agents/test_atomic_agent.py
index c626791..90622af 100644
--- a/atomic-agents/tests/agents/test_atomic_agent.py
+++ b/atomic-agents/tests/agents/test_atomic_agent.py
@@ -4,6 +4,7 @@ from enum import Enum
from pydantic import BaseModel, Field
from pydantic import ValidationError
import instructor
+import litellm
from atomic_agents import (
BaseIOSchema,
AtomicAgent,
@@ -1120,6 +1121,19 @@ def test_get_context_token_count_multimodal_content(mock_get_token_counter, mock
assert image_entry["image_url"]["url"] == "https://example.com/test.png"
+def test_serialize_history_for_token_count_video_placeholder(agent, mock_history):
+ video_part = {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}}
+ mock_history.get_history.return_value = [{"role": "user", "content": ['{"prompt":"Summarize the video"}', video_part]}]
+
+ serialized = agent._serialize_history_for_token_count()
+
+ assert serialized[0]["content"] == [
+ {"type": "text", "text": '{"prompt":"Summarize the video"}'},
+ {"type": "text", "text": "[video content]"},
+ ]
+ assert litellm.token_counter(model="gpt-4o", messages=serialized) > 0
+
+
# --- Tests for tool_result_role and Gemini system message remapping (issue #221) ---
diff --git a/atomic-agents/tests/base/test_multimodal.py b/atomic-agents/tests/base/test_multimodal.py
new file mode 100644
index 0000000..2892584
--- /dev/null
+++ b/atomic-agents/tests/base/test_multimodal.py
@@ -0,0 +1,16 @@
+from atomic_agents import VideoURL
+
+
+def test_to_openai_with_url_only():
+ content_part = VideoURL(url="https://example.com/clip.mp4").to_openai()
+
+ assert content_part == {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}}
+
+
+def test_to_openai_with_sampling_parameters():
+ content_part = VideoURL(url="https://example.com/clip.mp4", fps=1.0, detail="low").to_openai()
+
+ assert content_part == {
+ "type": "video_url",
+ "video_url": {"url": "https://example.com/clip.mp4", "fps": 1.0, "detail": "low"},
+ }
diff --git a/atomic-agents/tests/context/test_chat_history.py b/atomic-agents/tests/context/test_chat_history.py
index 4089aff..df1c0bc 100644
--- a/atomic-agents/tests/context/test_chat_history.py
+++ b/atomic-agents/tests/context/test_chat_history.py
@@ -6,7 +6,7 @@ from typing import List, Dict, Union
from pathlib import Path
from pydantic import Field
from atomic_agents.context import ChatHistory, Message
-from atomic_agents import BaseIOSchema
+from atomic_agents import BaseIOSchema, VideoURL
import instructor
@@ -865,3 +865,40 @@ def test_get_history_dict_of_images(history):
assert "image_map" not in json_part
assert img_a in result[0]["content"]
assert img_b in result[0]["content"]
+
+
+class VideoMessageSchema(BaseIOSchema):
+ """Test schema with a video field"""
+
+ prompt: str = Field(..., description="Instruction for the video")
+ video: VideoURL = Field(..., description="The video to analyze")
+
+
+def test_get_history_video_url_becomes_content_part(history):
+ """VideoURL fields are separated from the JSON text as video_url content parts"""
+ content = VideoMessageSchema(
+ prompt="Summarize the video",
+ video=VideoURL(url="https://example.com/clip.mp4", fps=1.0),
+ )
+
+ history.add_message("user", content)
+ result = history.get_history()
+
+ assert len(result) == 1
+ assert json.loads(result[0]["content"][0]) == {"prompt": "Summarize the video"}
+ assert result[0]["content"][1] == {
+ "type": "video_url",
+ "video_url": {"url": "https://example.com/clip.mp4", "fps": 1.0},
+ }
+
+
+def test_dump_and_load_video_url(history):
+ """VideoURL round-trips through dump() and load() as a typed object"""
+ history.add_message("user", VideoMessageSchema(prompt="Summarize", video=VideoURL(url="https://example.com/clip.mp4")))
+
+ loaded_history = ChatHistory()
+ loaded_history.load(history.dump())
+
+ loaded_content = loaded_history.history[0].content
+ assert isinstance(loaded_content.video, VideoURL)
+ assert loaded_content.video.url == "https://example.com/clip.mp4"
diff --git a/docs/guides/memory.md b/docs/guides/memory.md
index d0bf563..1fb32d4 100644
--- a/docs/guides/memory.md
+++ b/docs/guides/memory.md
@@ -477,7 +477,8 @@ There's a full runnable example of this pattern (a dependency-free SQLite-backed
## Multimodal Content in History
-ChatHistory supports images, PDFs, and audio through Instructor's multimodal types.
+ChatHistory supports images, PDFs, and audio through Instructor's multimodal types, plus
+video through the framework's own `VideoURL` type.
### Adding Multimodal Messages
@@ -522,6 +523,29 @@ for message in history_data:
json_content = message["content"]
```
+### Video
+
+Instructor has no video type yet, so Atomic Agents ships its own `VideoURL` for providers
+that accept OpenAI-compatible `video_url` content parts (MiniMax, Qwen-VL, ...):
+
+```python
+from atomic_agents import BaseIOSchema, VideoURL
+from pydantic import Field
+
+class VideoAnalysisInput(BaseIOSchema):
+ """Input with a video for analysis"""
+ question: str = Field(..., description="Question about the video")
+ video: VideoURL = Field(..., description="Video to analyze")
+
+input_with_video = VideoAnalysisInput(
+ question="What happens in this clip?",
+ video=VideoURL(url="https://example.com/clip.mp4", fps=1.0),
+)
+```
+
+`get_history()` emits the video as a `{"type": "video_url", ...}` content part, which
+Instructor passes through to the provider unchanged.
+
### Serialization with Multimodal
```{note}