Skip to content

Building AI agents, atomically

feature: improve handling of multi-modal content in AgentMemory.get_history

Renaud4abb3c9875a2
diff --git a/atomic-agents/atomic_agents/lib/components/agent_memory.py b/atomic-agents/atomic_agents/lib/components/agent_memory.py
index 0890153..a37e379 100644
--- a/atomic-agents/atomic_agents/lib/components/agent_memory.py
+++ b/atomic-agents/atomic_agents/lib/components/agent_memory.py
@@ -1,10 +1,16 @@
-import uuid
import json
+import uuid
from typing import Dict, List, Optional, Type
+
+from instructor.multimodal import PDF, Image, Audio
from pydantic import BaseModel, Field
+
from atomic_agents.lib.base.base_io_schema import BaseIOSchema
+INSTRUCTOR_MULTIMODAL_TYPES = (Image, Audio, PDF)
+
+
class Message(BaseModel):
"""
Represents a message in the chat history.
@@ -85,43 +91,40 @@ class AgentMemory:
Returns:
List[Dict]: The list of messages in the chat history as dictionaries.
+ Each dictionary has 'role' and 'content' keys, where 'content' is a list
+ that may contain strings (JSON) or multimodal objects.
+
+ Note:
+ This method does not support nested multimodal content. If your schema
+ contains nested objects that themselves contain multimodal content,
+ only the top-level multimodal content will be properly processed.
+
"""
history = []
for message in self.history:
- content = message.content
- message_content = content.model_dump()
-
- images = []
- image_keys = []
-
- for key, value in message_content.items():
- if isinstance(value, list):
- for list_item in value:
- if isinstance(list_item, dict) and list_item.get("media_type", "").startswith("image"):
- images.extend(value)
- image_keys.append(key)
- break
-
- if isinstance(value, dict) and value.get("media_type", "").startswith("image"):
- images.append(value)
- image_keys.append(key)
-
- if len(images) > 0:
- # For multimodal content, format as a list with text and images
- # delete image keys from model
- images = []
- for key in image_keys:
- message_content.pop(key)
- image_content = getattr(content, key)
- if isinstance(image_content, list):
- images.extend(image_content)
+
+ # TODO: this does not support nested multi-modal content
+
+ input_content = message.content
+ processed_content = []
+ for field_name, field in input_content.__class__.model_fields.items():
+
+ field_value = getattr(input_content, field_name)
+
+ if isinstance(field_value, list):
+ for item in field_value:
+ if isinstance(item, INSTRUCTOR_MULTIMODAL_TYPES):
+ processed_content.append(item)
+ else:
+ processed_content.append(json.dumps({field_name: field_value}))
+ else:
+ if isinstance(field_value, INSTRUCTOR_MULTIMODAL_TYPES):
+ processed_content.append(field_value)
else:
- images.append(image_content)
+ processed_content.append(json.dumps({field_name: field_value}))
+
+ history.append({"role": message.role, "content": processed_content})
- history.append({"role": message.role, "content": [json.dumps(message_content), *images]})
- else:
- # For regular content, serialize to JSON string
- history.append({"role": message.role, "content": json.dumps(content.model_dump(mode="json"))})
return history
def copy(self) -> "AgentMemory":
diff --git a/atomic-agents/tests/lib/components/test_agent_memory.py b/atomic-agents/tests/lib/components/test_agent_memory.py
index fb8484c..b20d1c3 100644
--- a/atomic-agents/tests/lib/components/test_agent_memory.py
+++ b/atomic-agents/tests/lib/components/test_agent_memory.py
@@ -48,6 +48,7 @@ class TestMultimodalSchema(BaseIOSchema):
instruction_text: str = Field(..., description="The instruction text")
images: List[instructor.Image] = Field(..., description="The images to analyze")
+ pdfs: List[instructor.multimodal.PDF] = Field(..., description="The PDFs to analyze")
@pytest.fixture
@@ -87,7 +88,8 @@ def test_get_history(memory):
history = memory.get_history()
assert len(history) == 2
assert history[0]["role"] == "user"
- assert json.loads(history[0]["content"]) == {"test_field": "Hello"}
+ assert json.loads(history[0]["content"][0]) == {"test_field": "Hello"}
+ assert json.loads(history[1]["content"][0]) == {"test_field": "Hi there"}
def test_copy(memory):
@@ -288,9 +290,14 @@ def test_get_history_with_multimodal_content(memory):
"""Test that get_history correctly handles multimodal content"""
# Create a mock image
mock_image = instructor.Image(source="test_url", media_type="image/jpeg", detail="low")
+ mock_pdf = instructor.multimodal.PDF(source="test_pdf_url")
# Add a multimodal message
- memory.add_message("user", TestMultimodalSchema(instruction_text="Analyze this image", images=[mock_image]))
+ memory.add_message("user", TestMultimodalSchema(
+ instruction_text="Analyze this image",
+ images=[mock_image],
+ pdfs=[mock_pdf]
+ ))
# Get history and verify format
history = memory.get_history()
@@ -299,6 +306,7 @@ def test_get_history_with_multimodal_content(memory):
assert isinstance(history[0]["content"], list)
assert history[0]["content"][0] == '{"instruction_text": "Analyze this image"}'
assert history[0]["content"][1] == mock_image
+ assert history[0]["content"][2] == mock_pdf
def test_get_history_with_multiple_images_multimodal_content(memory):