Building AI agents, atomically
Add support for arbitary image keys for multi modal queries
Stefan Hessec26ea9ecf63a
diff --git a/atomic-agents/atomic_agents/lib/components/agent_memory.py b/atomic-agents/atomic_agents/lib/components/agent_memory.pyindex d5a365b..e0f1cc4 100644--- a/atomic-agents/atomic_agents/lib/components/agent_memory.py+++ b/atomic-agents/atomic_agents/lib/components/agent_memory.py@@ -2,7 +2,6 @@ import uuidimport jsonfrom typing import Dict, List, Optional, Typefrom pydantic import BaseModel, Field-from atomic_agents.lib.base.base_io_schema import BaseIOSchema@@ -50,9 +49,9 @@ class AgentMemory:self.current_turn_id = str(uuid.uuid4())def add_message(- self,- role: str,- content: BaseIOSchema,+ self,+ role: str,+ content: BaseIOSchema,) -> None:"""Adds a message to the chat history and manages overflow.@@ -90,11 +89,36 @@ class AgentMemory:history = []for message in self.history:content = message.content- if hasattr(content, "images") and content.images:+ message_content = content.model_dump()++ images = []+ image_keys = []++ for key, value in message_content.items():+ if isinstance(value, list):+ for list_item in value:+ if isinstance(list_item, dict) and list_item.get("media_type", "").startswith("image"):+ images.extend(value)+ image_keys.append(key)+ break++ if isinstance(value, dict) and value.get("media_type", "").startswith("image"):+ images.append(value)+ image_keys.append(key)++ if len(images) > 0:# For multimodal content, format as a list with text and images- text_field = next((field for field in content.model_fields if field.endswith("text")), None)- instruction_text = getattr(content, text_field) if text_field else str(content)- history.append({"role": message.role, "content": [instruction_text, *content.images]})+ # delete image keys from model+ images = []+ for key in image_keys:+ message_content.pop(key)+ image_content = getattr(content, key)+ if isinstance(image_content, list):+ images.extend(image_content)+ else:+ images.append(image_content)++ history.append({"role": message.role, "content": [json.dumps(message_content), *images]})else:# For regular content, serialize to JSON stringhistory.append({"role": message.role, "content": json.dumps(content.model_dump())})@@ -233,6 +257,7 @@ if __name__ == "__main__":from typing import List as TypeList, Dict as TypeDictimport os+# Define complex test schemasclass NestedSchema(BaseIOSchema):"""A nested schema for testing"""@@ -240,6 +265,7 @@ if __name__ == "__main__":nested_field: str = Field(..., description="A nested field")nested_int: int = Field(..., description="A nested integer")+class ComplexInputSchema(BaseIOSchema):"""Complex Input Schema"""@@ -248,6 +274,7 @@ if __name__ == "__main__":list_field: TypeList[str] = Field(..., description="A list of strings")nested_field: NestedSchema = Field(..., description="A nested schema")+class ComplexOutputSchema(BaseIOSchema):"""Complex Output Schema"""@@ -255,6 +282,7 @@ if __name__ == "__main__":calculated_value: int = Field(..., description="A calculated value")data_dict: TypeDict[str, NestedSchema] = Field(..., description="A dictionary of nested schemas")+# Add a new multimodal schema for testingclass MultimodalSchema(BaseIOSchema):"""Schema for testing multimodal content"""@@ -262,6 +290,7 @@ if __name__ == "__main__":instruction_text: str = Field(..., description="The instruction text")images: List[instructor.Image] = Field(..., description="The images to analyze")+# Create and populate the original memory with complex dataoriginal_memory = AgentMemory(max_messages=10)diff --git a/atomic-agents/tests/lib/components/test_agent_memory.py b/atomic-agents/tests/lib/components/test_agent_memory.pyindex a5f2eb7..cfd0b7b 100644--- a/atomic-agents/tests/lib/components/test_agent_memory.py+++ b/atomic-agents/tests/lib/components/test_agent_memory.py@@ -297,5 +297,36 @@ def test_get_history_with_multimodal_content(memory):assert len(history) == 1assert history[0]["role"] == "user"assert isinstance(history[0]["content"], list)- assert history[0]["content"][0] == "Analyze this image"+ assert history[0]["content"][0] == '{"instruction_text": "Analyze this image"}'assert history[0]["content"][1] == mock_image+++def test_get_history_with_multiple_images_multimodal_content(memory):+ """Test that get_history correctly handles multimodal content"""++ class TestMultimodalSchemaArbitraryKeys(BaseIOSchema):+ """Test schema for multimodal content"""++ instruction_text: str = Field(..., description="The instruction text")+ some_key_for_images: List[instructor.Image] = Field(..., description="The images to analyze")+ some_other_key_with_image: instructor.Image = Field(..., description="The images to analyze")++ # Create a mock image+ mock_image = instructor.Image(source="test_url", media_type="image/jpeg", detail="low")+ mock_image_2 = instructor.Image(source="test_url_2", media_type="image/jpeg", detail="low")+ mock_image_3 = instructor.Image(source="test_url_3", media_type="image/jpeg", detail="low")++ # Add a multimodal message+ memory.add_message("user", TestMultimodalSchemaArbitraryKeys(instruction_text="Analyze this image",+ some_other_key_with_image=mock_image,+ some_key_for_images=[mock_image_2, mock_image_3]))++ # Get history and verify format+ history = memory.get_history()+ assert len(history) == 1+ assert history[0]["role"] == "user"+ assert isinstance(history[0]["content"], list)+ assert history[0]["content"][0] == '{"instruction_text": "Analyze this image"}'+ assert mock_image in history[0]["content"]+ assert mock_image_2 in history[0]["content"]+ assert mock_image_3 in history[0]["content"]