Building AI agents, atomically
Add typed VideoURL multimodal support
Eigenwisec80c7520a703
diff --git a/atomic-agents/atomic_agents/__init__.py b/atomic-agents/atomic_agents/__init__.pyindex 7e54296..c4ed2d2 100644--- a/atomic-agents/atomic_agents/__init__.py+++ b/atomic-agents/atomic_agents/__init__.py@@ -4,7 +4,7 @@ Atomic Agents - A modular framework for building AI agents.# Core exports - base classes onlyfrom .agents.atomic_agent import AtomicAgent, AgentConfig, BasicChatInputSchema, BasicChatOutputSchema-from .base import BaseIOSchema, BaseTool, BaseToolConfig+from .base import BaseIOSchema, BaseTool, BaseToolConfig, VideoURL# Version info - read from pyproject.toml via package metadatafrom importlib.metadata import version as _version@@ -19,4 +19,5 @@ __all__ = ["BaseIOSchema","BaseTool","BaseToolConfig",+ "VideoURL",]diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.pyindex d2ccfe6..801026b 100644--- a/atomic-agents/atomic_agents/agents/atomic_agent.py+++ b/atomic-agents/atomic_agents/agents/atomic_agent.py@@ -431,7 +431,8 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:This method converts instructor multimodal objects (Image, Audio, PDF) to theOpenAI format that LiteLLM's token counter expects. Text content is also- converted to the proper multimodal text format when mixed with media.+ converted to the proper multimodal text format when mixed with media. Video+ content parts become text placeholders because LiteLLM cannot count them.Returns:List[Dict[str, Any]]: History messages in LiteLLM-compatible format.@@ -462,6 +463,10 @@ class AtomicAgent[InputSchema: BaseIOSchema, OutputSchema: BaseIOSchema]:f"Using placeholder for estimation.")serialized_content.append({"type": "text", "text": f"[{media_type.lower()} content]"})+ elif isinstance(item, dict) and item.get("type") == "video_url":+ # LiteLLM's token counter raises on video_url parts, so use a+ # placeholder for estimation, same as failed media serialization.+ serialized_content.append({"type": "text", "text": "[video content]"})else:# Unknown type - convert to stringserialized_content.append({"type": "text", "text": str(item)})diff --git a/atomic-agents/atomic_agents/base/__init__.py b/atomic-agents/atomic_agents/base/__init__.pyindex 9bbe931..18d1d5a 100644--- a/atomic-agents/atomic_agents/base/__init__.py+++ b/atomic-agents/atomic_agents/base/__init__.py@@ -4,6 +4,7 @@ from .base_io_schema import BaseIOSchemafrom .base_tool import BaseTool, BaseToolConfigfrom .base_resource import BaseResource, BaseResourceConfigfrom .base_prompt import BasePrompt, BasePromptConfig+from .multimodal import VideoURL__all__ = ["BaseIOSchema",@@ -13,4 +14,5 @@ __all__ = ["BaseResourceConfig","BasePrompt","BasePromptConfig",+ "VideoURL",]diff --git a/atomic-agents/atomic_agents/base/multimodal.py b/atomic-agents/atomic_agents/base/multimodal.pynew file mode 100644index 0000000..69f6d4b--- /dev/null+++ b/atomic-agents/atomic_agents/base/multimodal.py@@ -0,0 +1,40 @@+"""Multimodal content types that Instructor does not provide."""++from typing import Any, Dict, Optional++from pydantic import BaseModel, Field+++class VideoURL(BaseModel):+ """+ Video reference sent to the LLM as an OpenAI-compatible ``video_url`` content part.++ Instructor ships Image, Audio, and PDF types but no video type+ (see https://github.com/567-labs/instructor/discussions/2520), so this class fills+ the gap for providers that accept ``video_url`` content parts, such as MiniMax+ and Qwen-VL.++ Attributes:+ url (str): HTTP(S) or data: URL of the video.+ fps (Optional[float]): Frame sampling rate, for providers that accept it.+ detail (Optional[str]): Detail level, for providers that accept it.+ """++ url: str = Field(..., description="HTTP(S) or data: URL of the video.")+ fps: Optional[float] = Field(default=None, description="Frame sampling rate, for providers that accept it.")+ detail: Optional[str] = Field(default=None, description="Detail level, for providers that accept it.")++ def to_openai(self) -> Dict[str, Any]:+ """+ Build the OpenAI-compatible content part for this video.++ Returns:+ Dict[str, Any]: A ``{"type": "video_url", "video_url": {...}}`` content part,+ omitting optional parameters that were not set.+ """+ video_url: Dict[str, Any] = {"url": self.url}+ if self.fps is not None:+ video_url["fps"] = self.fps+ if self.detail is not None:+ video_url["detail"] = self.detail+ return {"type": "video_url", "video_url": video_url}diff --git a/atomic-agents/atomic_agents/context/chat_history.py b/atomic-agents/atomic_agents/context/chat_history.pyindex e4fbf91..d4ec83a 100644--- a/atomic-agents/atomic_agents/context/chat_history.py+++ b/atomic-agents/atomic_agents/context/chat_history.py@@ -2,16 +2,21 @@ import jsonimport uuidfrom enum import Enumfrom pathlib import Path-from typing import Dict, List, Optional, Type+from typing import Dict, List, Optional, Tuple, Type, Unionfrom instructor.processing.multimodal import PDF, Image, Audiofrom pydantic import BaseModel, Fieldfrom atomic_agents.base.base_io_schema import BaseIOSchema+from atomic_agents.base.multimodal import VideoURLfrom atomic_agents.context.base_chat_history import BaseChatHistoryINSTRUCTOR_MULTIMODAL_TYPES = (Image, Audio, PDF)+MULTIMODAL_TYPES = INSTRUCTOR_MULTIMODAL_TYPES + (VideoURL,)++MultimodalObject = Union[Image, Audio, PDF, VideoURL]+ExcludeSpec = Union[Dict, bool, None]class Message(BaseModel):@@ -109,10 +114,16 @@ class ChatHistory(BaseChatHistory):if multimodal_objects:processed_content = []- content_json = input_content.model_dump_json(exclude=exclude_spec)+ excluded_fields = exclude_spec if isinstance(exclude_spec, dict) else None+ content_json = input_content.model_dump_json(exclude=excluded_fields)if content_json and content_json != "{}":processed_content.append(content_json)- processed_content.extend(multimodal_objects)+ # Instructor forwards dict content parts to the provider unchanged but+ # raises on object types it does not know, so VideoURL is converted here.+ processed_content.extend(+ multimodal_object.to_openai() if isinstance(multimodal_object, VideoURL) else multimodal_object+ for multimodal_object in multimodal_objects+ )history.append({"role": message.role, "content": processed_content})else:content_json = input_content.model_dump_json()@@ -121,13 +132,14 @@ class ChatHistory(BaseChatHistory):return history@staticmethod- def _extract_multimodal_info(obj):+ def _extract_multimodal_info(obj) -> Tuple[List[MultimodalObject], ExcludeSpec]:"""Recursively extract multimodal objects and build a Pydantic-compatible exclude spec.- Walks the object tree to find all Instructor multimodal types (Image, Audio, PDF)- at any nesting depth, collecting them into a flat list and building an exclude- specification that can be passed to model_dump_json(exclude=...).+ Walks the object tree to find all multimodal types (Instructor's Image, Audio,+ and PDF, plus VideoURL) at any nesting depth, collecting them into a flat list+ and building an exclude specification that can be passed to+ model_dump_json(exclude=...).Args:obj: The object to inspect (BaseIOSchema, list, dict, or primitive).@@ -137,7 +149,7 @@ class ChatHistory(BaseChatHistory):- multimodal_objects: flat list of all multimodal objects found- exclude_spec: Pydantic exclude dict, True (exclude entirely), or None"""- if isinstance(obj, INSTRUCTOR_MULTIMODAL_TYPES):+ if isinstance(obj, MULTIMODAL_TYPES):return [obj], Trueif hasattr(obj, "__class__") and hasattr(obj.__class__, "model_fields"):diff --git a/atomic-agents/tests/agents/test_atomic_agent.py b/atomic-agents/tests/agents/test_atomic_agent.pyindex c626791..90622af 100644--- a/atomic-agents/tests/agents/test_atomic_agent.py+++ b/atomic-agents/tests/agents/test_atomic_agent.py@@ -4,6 +4,7 @@ from enum import Enumfrom pydantic import BaseModel, Fieldfrom pydantic import ValidationErrorimport instructor+import litellmfrom atomic_agents import (BaseIOSchema,AtomicAgent,@@ -1120,6 +1121,19 @@ def test_get_context_token_count_multimodal_content(mock_get_token_counter, mockassert image_entry["image_url"]["url"] == "https://example.com/test.png"+def test_serialize_history_for_token_count_video_placeholder(agent, mock_history):+ video_part = {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}}+ mock_history.get_history.return_value = [{"role": "user", "content": ['{"prompt":"Summarize the video"}', video_part]}]++ serialized = agent._serialize_history_for_token_count()++ assert serialized[0]["content"] == [+ {"type": "text", "text": '{"prompt":"Summarize the video"}'},+ {"type": "text", "text": "[video content]"},+ ]+ assert litellm.token_counter(model="gpt-4o", messages=serialized) > 0++# --- Tests for tool_result_role and Gemini system message remapping (issue #221) ---diff --git a/atomic-agents/tests/base/test_multimodal.py b/atomic-agents/tests/base/test_multimodal.pynew file mode 100644index 0000000..2892584--- /dev/null+++ b/atomic-agents/tests/base/test_multimodal.py@@ -0,0 +1,16 @@+from atomic_agents import VideoURL+++def test_to_openai_with_url_only():+ content_part = VideoURL(url="https://example.com/clip.mp4").to_openai()++ assert content_part == {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}}+++def test_to_openai_with_sampling_parameters():+ content_part = VideoURL(url="https://example.com/clip.mp4", fps=1.0, detail="low").to_openai()++ assert content_part == {+ "type": "video_url",+ "video_url": {"url": "https://example.com/clip.mp4", "fps": 1.0, "detail": "low"},+ }diff --git a/atomic-agents/tests/context/test_chat_history.py b/atomic-agents/tests/context/test_chat_history.pyindex 4089aff..df1c0bc 100644--- a/atomic-agents/tests/context/test_chat_history.py+++ b/atomic-agents/tests/context/test_chat_history.py@@ -6,7 +6,7 @@ from typing import List, Dict, Unionfrom pathlib import Pathfrom pydantic import Fieldfrom atomic_agents.context import ChatHistory, Message-from atomic_agents import BaseIOSchema+from atomic_agents import BaseIOSchema, VideoURLimport instructor@@ -865,3 +865,40 @@ def test_get_history_dict_of_images(history):assert "image_map" not in json_partassert img_a in result[0]["content"]assert img_b in result[0]["content"]+++class VideoMessageSchema(BaseIOSchema):+ """Test schema with a video field"""++ prompt: str = Field(..., description="Instruction for the video")+ video: VideoURL = Field(..., description="The video to analyze")+++def test_get_history_video_url_becomes_content_part(history):+ """VideoURL fields are separated from the JSON text as video_url content parts"""+ content = VideoMessageSchema(+ prompt="Summarize the video",+ video=VideoURL(url="https://example.com/clip.mp4", fps=1.0),+ )++ history.add_message("user", content)+ result = history.get_history()++ assert len(result) == 1+ assert json.loads(result[0]["content"][0]) == {"prompt": "Summarize the video"}+ assert result[0]["content"][1] == {+ "type": "video_url",+ "video_url": {"url": "https://example.com/clip.mp4", "fps": 1.0},+ }+++def test_dump_and_load_video_url(history):+ """VideoURL round-trips through dump() and load() as a typed object"""+ history.add_message("user", VideoMessageSchema(prompt="Summarize", video=VideoURL(url="https://example.com/clip.mp4")))++ loaded_history = ChatHistory()+ loaded_history.load(history.dump())++ loaded_content = loaded_history.history[0].content+ assert isinstance(loaded_content.video, VideoURL)+ assert loaded_content.video.url == "https://example.com/clip.mp4"diff --git a/docs/guides/memory.md b/docs/guides/memory.mdindex d0bf563..1fb32d4 100644--- a/docs/guides/memory.md+++ b/docs/guides/memory.md@@ -477,7 +477,8 @@ There's a full runnable example of this pattern (a dependency-free SQLite-backed## Multimodal Content in History-ChatHistory supports images, PDFs, and audio through Instructor's multimodal types.+ChatHistory supports images, PDFs, and audio through Instructor's multimodal types, plus+video through the framework's own `VideoURL` type.### Adding Multimodal Messages@@ -522,6 +523,29 @@ for message in history_data:json_content = message["content"]```+### Video++Instructor has no video type yet, so Atomic Agents ships its own `VideoURL` for providers+that accept OpenAI-compatible `video_url` content parts (MiniMax, Qwen-VL, ...):++```python+from atomic_agents import BaseIOSchema, VideoURL+from pydantic import Field++class VideoAnalysisInput(BaseIOSchema):+ """Input with a video for analysis"""+ question: str = Field(..., description="Question about the video")+ video: VideoURL = Field(..., description="Video to analyze")++input_with_video = VideoAnalysisInput(+ question="What happens in this clip?",+ video=VideoURL(url="https://example.com/clip.mp4", fps=1.0),+)+```++`get_history()` emits the video as a `{"type": "video_url", ...}` content part, which+Instructor passes through to the provider unchanged.+### Serialization with Multimodal```{note}