Building AI agents, atomically
feature: improve handling of multi-modal content in AgentMemory.get_history
Renaud4abb3c9875a2
diff --git a/atomic-agents/atomic_agents/lib/components/agent_memory.py b/atomic-agents/atomic_agents/lib/components/agent_memory.pyindex 0890153..a37e379 100644--- a/atomic-agents/atomic_agents/lib/components/agent_memory.py+++ b/atomic-agents/atomic_agents/lib/components/agent_memory.py@@ -1,10 +1,16 @@-import uuidimport json+import uuidfrom typing import Dict, List, Optional, Type++from instructor.multimodal import PDF, Image, Audiofrom pydantic import BaseModel, Field+from atomic_agents.lib.base.base_io_schema import BaseIOSchema+INSTRUCTOR_MULTIMODAL_TYPES = (Image, Audio, PDF)++class Message(BaseModel):"""Represents a message in the chat history.@@ -85,43 +91,40 @@ class AgentMemory:Returns:List[Dict]: The list of messages in the chat history as dictionaries.+ Each dictionary has 'role' and 'content' keys, where 'content' is a list+ that may contain strings (JSON) or multimodal objects.++ Note:+ This method does not support nested multimodal content. If your schema+ contains nested objects that themselves contain multimodal content,+ only the top-level multimodal content will be properly processed.+"""history = []for message in self.history:- content = message.content- message_content = content.model_dump()-- images = []- image_keys = []-- for key, value in message_content.items():- if isinstance(value, list):- for list_item in value:- if isinstance(list_item, dict) and list_item.get("media_type", "").startswith("image"):- images.extend(value)- image_keys.append(key)- break-- if isinstance(value, dict) and value.get("media_type", "").startswith("image"):- images.append(value)- image_keys.append(key)-- if len(images) > 0:- # For multimodal content, format as a list with text and images- # delete image keys from model- images = []- for key in image_keys:- message_content.pop(key)- image_content = getattr(content, key)- if isinstance(image_content, list):- images.extend(image_content)++ # TODO: this does not support nested multi-modal content++ input_content = message.content+ processed_content = []+ for field_name, field in input_content.__class__.model_fields.items():++ field_value = getattr(input_content, field_name)++ if isinstance(field_value, list):+ for item in field_value:+ if isinstance(item, INSTRUCTOR_MULTIMODAL_TYPES):+ processed_content.append(item)+ else:+ processed_content.append(json.dumps({field_name: field_value}))+ else:+ if isinstance(field_value, INSTRUCTOR_MULTIMODAL_TYPES):+ processed_content.append(field_value)else:- images.append(image_content)+ processed_content.append(json.dumps({field_name: field_value}))++ history.append({"role": message.role, "content": processed_content})- history.append({"role": message.role, "content": [json.dumps(message_content), *images]})- else:- # For regular content, serialize to JSON string- history.append({"role": message.role, "content": json.dumps(content.model_dump(mode="json"))})return historydef copy(self) -> "AgentMemory":diff --git a/atomic-agents/tests/lib/components/test_agent_memory.py b/atomic-agents/tests/lib/components/test_agent_memory.pyindex fb8484c..b20d1c3 100644--- a/atomic-agents/tests/lib/components/test_agent_memory.py+++ b/atomic-agents/tests/lib/components/test_agent_memory.py@@ -48,6 +48,7 @@ class TestMultimodalSchema(BaseIOSchema):instruction_text: str = Field(..., description="The instruction text")images: List[instructor.Image] = Field(..., description="The images to analyze")+ pdfs: List[instructor.multimodal.PDF] = Field(..., description="The PDFs to analyze")@pytest.fixture@@ -87,7 +88,8 @@ def test_get_history(memory):history = memory.get_history()assert len(history) == 2assert history[0]["role"] == "user"- assert json.loads(history[0]["content"]) == {"test_field": "Hello"}+ assert json.loads(history[0]["content"][0]) == {"test_field": "Hello"}+ assert json.loads(history[1]["content"][0]) == {"test_field": "Hi there"}def test_copy(memory):@@ -288,9 +290,14 @@ def test_get_history_with_multimodal_content(memory):"""Test that get_history correctly handles multimodal content"""# Create a mock imagemock_image = instructor.Image(source="test_url", media_type="image/jpeg", detail="low")+ mock_pdf = instructor.multimodal.PDF(source="test_pdf_url")# Add a multimodal message- memory.add_message("user", TestMultimodalSchema(instruction_text="Analyze this image", images=[mock_image]))+ memory.add_message("user", TestMultimodalSchema(+ instruction_text="Analyze this image",+ images=[mock_image],+ pdfs=[mock_pdf]+ ))# Get history and verify formathistory = memory.get_history()@@ -299,6 +306,7 @@ def test_get_history_with_multimodal_content(memory):assert isinstance(history[0]["content"], list)assert history[0]["content"][0] == '{"instruction_text": "Analyze this image"}'assert history[0]["content"][1] == mock_image+ assert history[0]["content"][2] == mock_pdfdef test_get_history_with_multiple_images_multimodal_content(memory):