Skip to content

Building AI agents, atomically

Finish Multimodality

KennyVaneetveldeaf09845a4185
diff --git a/atomic-agents/atomic_agents/lib/components/agent_memory.py b/atomic-agents/atomic_agents/lib/components/agent_memory.py
index fd7bdab..d5a365b 100644
--- a/atomic-agents/atomic_agents/lib/components/agent_memory.py
+++ b/atomic-agents/atomic_agents/lib/components/agent_memory.py
@@ -64,7 +64,11 @@ class AgentMemory:
if self.current_turn_id is None:
self.initialize_turn()
- message = Message(role=role, content=content, turn_id=self.current_turn_id)
+ message = Message(
+ role=role,
+ content=content,
+ turn_id=self.current_turn_id,
+ )
self.history.append(message)
self._manage_overflow()
@@ -78,18 +82,23 @@ class AgentMemory:
def get_history(self) -> List[Dict]:
"""
- Retrieves the chat history, filtering out unnecessary fields and serializing content.
+ Retrieves the chat history, handling both regular and multimodal content.
Returns:
List[Dict]: The list of messages in the chat history as dictionaries.
"""
- return [
- {
- "role": message.role,
- "content": json.dumps(message.content.model_dump()),
- }
- for message in self.history
- ]
+ history = []
+ for message in self.history:
+ content = message.content
+ if hasattr(content, "images") and content.images:
+ # For multimodal content, format as a list with text and images
+ text_field = next((field for field in content.model_fields if field.endswith("text")), None)
+ instruction_text = getattr(content, text_field) if text_field else str(content)
+ history.append({"role": message.role, "content": [instruction_text, *content.images]})
+ else:
+ # For regular content, serialize to JSON string
+ history.append({"role": message.role, "content": json.dumps(content.model_dump())})
+ return history
def copy(self) -> "AgentMemory":
"""
@@ -220,7 +229,9 @@ class AgentMemory:
if __name__ == "__main__":
+ import instructor
from typing import List as TypeList, Dict as TypeDict
+ import os
# Define complex test schemas
class NestedSchema(BaseIOSchema):
@@ -244,6 +255,13 @@ if __name__ == "__main__":
calculated_value: int = Field(..., description="A calculated value")
data_dict: TypeDict[str, NestedSchema] = Field(..., description="A dictionary of nested schemas")
+ # Add a new multimodal schema for testing
+ class MultimodalSchema(BaseIOSchema):
+ """Schema for testing multimodal content"""
+
+ instruction_text: str = Field(..., description="The instruction text")
+ images: List[instructor.Image] = Field(..., description="The images to analyze")
+
# Create and populate the original memory with complex data
original_memory = AgentMemory(max_messages=10)
@@ -271,18 +289,18 @@ if __name__ == "__main__":
),
)
- # Add another input message
- original_memory.add_message(
- "user",
- ComplexInputSchema(
- text_field="Another complex input",
- number_field=2.71828,
- list_field=["apple", "banana", "cherry"],
- nested_field=NestedSchema(nested_field="More nested input", nested_int=99),
- ),
- )
-
- # Dump the memory
+ # Test multimodal functionality if test image exists
+ test_image_path = os.path.join("test_images", "test.jpg")
+ if os.path.exists(test_image_path):
+ # Add a multimodal message
+ original_memory.add_message(
+ "user",
+ MultimodalSchema(
+ instruction_text="Please analyze this image", images=[instructor.Image.from_path(test_image_path)]
+ ),
+ )
+
+ # Continue with existing tests...
dumped_data = original_memory.dump()
print("Dumped data:")
print(dumped_data)
@@ -291,9 +309,6 @@ if __name__ == "__main__":
loaded_memory = AgentMemory()
loaded_memory.load(dumped_data)
- # Verify that the loaded memory matches the original
- print("\nOriginal memory dump == Loaded memory dump:", original_memory.dump() == loaded_memory.dump())
-
# Print detailed information about the loaded memory
print("\nLoaded memory details:")
for i, message in enumerate(loaded_memory.history):
@@ -305,19 +320,6 @@ if __name__ == "__main__":
for field, value in message.content.model_dump().items():
print(f" {field}: {value}")
- # Verify that the loaded memory can be used to add new messages
- loaded_memory.add_message(
- "assistant",
- ComplexOutputSchema(
- response_text="This is a new message added to the loaded memory",
- calculated_value=200,
- data_dict={"key3": NestedSchema(nested_field="New nested output", nested_int=30)},
- ),
- )
-
- print("\nAdded a new message to the loaded memory.")
- print(f"Total messages in loaded memory: {loaded_memory.get_message_count()}")
-
# Final verification
print("\nFinal verification:")
print(f"Max messages: {loaded_memory.max_messages}")
diff --git a/atomic-agents/tests/lib/components/test_agent_memory.py b/atomic-agents/tests/lib/components/test_agent_memory.py
index 2d1ee2a..a5f2eb7 100644
--- a/atomic-agents/tests/lib/components/test_agent_memory.py
+++ b/atomic-agents/tests/lib/components/test_agent_memory.py
@@ -4,6 +4,7 @@ from typing import List, Dict
from pydantic import Field
from atomic_agents.lib.components.agent_memory import AgentMemory, Message
from atomic_agents.lib.base.base_io_schema import BaseIOSchema
+import instructor
class TestInputSchema(BaseIOSchema):
@@ -42,6 +43,13 @@ class TestComplexOutputSchema(BaseIOSchema):
data_dict: Dict[str, TestNestedSchema] = Field(..., description="A dictionary of nested schemas")
+class TestMultimodalSchema(BaseIOSchema):
+ """Test schema for multimodal content"""
+
+ instruction_text: str = Field(..., description="The instruction text")
+ images: List[instructor.Image] = Field(..., description="The images to analyze")
+
+
@pytest.fixture
def memory():
return AgentMemory(max_messages=5)
@@ -274,3 +282,20 @@ def test_agent_memory_delete_turn_id(memory):
# Assert: Trying to delete a non-existing turn ID should raise a ValueError
with pytest.raises(ValueError, match="Turn ID non-existent-id not found in memory."):
memory.delete_turn_id("non-existent-id")
+
+
+def test_get_history_with_multimodal_content(memory):
+ """Test that get_history correctly handles multimodal content"""
+ # Create a mock image
+ mock_image = instructor.Image(source="test_url", media_type="image/jpeg", detail="low")
+
+ # Add a multimodal message
+ memory.add_message("user", TestMultimodalSchema(instruction_text="Analyze this image", images=[mock_image]))
+
+ # Get history and verify format
+ history = memory.get_history()
+ assert len(history) == 1
+ assert history[0]["role"] == "user"
+ assert isinstance(history[0]["content"], list)
+ assert history[0]["content"][0] == "Analyze this image"
+ assert history[0]["content"][1] == mock_image
diff --git a/atomic-examples/basic-multimodal/basic_multimodal/main.py b/atomic-examples/basic-multimodal/basic_multimodal/main.py
index 36c1930..2b40b70 100644
--- a/atomic-examples/basic-multimodal/basic_multimodal/main.py
+++ b/atomic-examples/basic-multimodal/basic_multimodal/main.py
@@ -4,17 +4,14 @@ from atomic_agents.lib.components.system_prompt_generator import SystemPromptGen
from atomic_agents.lib.components.agent_memory import AgentMemory
import instructor
import openai
-from pydantic import Field, BaseModel
-from typing import List, Union, Any, Optional
+from pydantic import Field
+from typing import List
import os
-import json
-class NutritionFacts(BaseIOSchema):
- """Represents nutrition facts from a food label"""
+class NutritionLabel(BaseIOSchema):
+ """Represents the complete nutritional information from a food label"""
- serving_size: str = Field(..., description="Serving size information")
- servings_per_container: float = Field(..., description="Number of servings per container")
calories: int = Field(..., description="Calories per serving")
total_fat: float = Field(..., description="Total fat in grams")
saturated_fat: float = Field(..., description="Saturated fat in grams")
@@ -30,144 +27,107 @@ class NutritionFacts(BaseIOSchema):
calcium: int = Field(..., description="Calcium in milligrams")
iron: float = Field(..., description="Iron in milligrams")
potassium: int = Field(..., description="Potassium in milligrams")
+ serving_size: str = Field(..., description="The size of a single serving of this product")
+ servings_per_container: float = Field(..., description="Number of servings contained in the package")
+ product_name: str = Field(
+ ...,
+ description="The full name or description of the type of the food/drink. e.g: 'Coca Cola Light', 'Pepsi Max', 'Smoked Bacon', 'Chianti Wine'",
+ )
-class ImageMessage(BaseModel):
- """A message that can contain multiple images"""
-
- text: str
- image_paths: List[str]
-
- def to_content(self):
- """Convert to the format expected by the API"""
- return [self.text, *[instructor.Image.from_path(path) for path in self.image_paths]]
-
- def model_dump(self, *args, **kwargs):
- """Custom serialization that only includes the text and paths"""
- return {"text": self.text, "image_paths": [str(path) for path in self.image_paths]}
-
-
-class NutritionInputSchema(BaseIOSchema):
- """Schema for nutrition facts input"""
-
- message: ImageMessage
-
- def get_api_messages(self):
- """Get the messages in the format expected by the API"""
- return self.message.to_content()
-
-
-class NutritionOutputSchema(BaseIOSchema):
- """Schema for nutrition facts extraction from multiple images"""
-
- nutrition_facts: List[NutritionFacts] = Field(..., description="Nutrition facts extracted from each image")
-
-
-# Monkey patch the get_history method to handle our custom message format
-original_get_history = AgentMemory.get_history
+class NutritionAnalysisInput(BaseIOSchema):
+ """Input schema for nutrition label analysis"""
+ instruction_text: str = Field(..., description="The instruction for analyzing the nutrition label")
+ images: List[instructor.Image] = Field(..., description="The nutrition label images to analyze")
-def patched_get_history(self):
- history = []
- for message in self.history:
- if isinstance(message.content, NutritionInputSchema) and message.role == "user":
- # For our custom input schema, use the API format
- history.append({"role": message.role, "content": message.content.get_api_messages()})
- else:
- # For other messages, use the normal serialization
- history.append({"role": message.role, "content": json.dumps(message.content.model_dump())})
- return history
+class NutritionAnalysisOutput(BaseIOSchema):
+ """Output schema containing extracted nutrition information"""
-AgentMemory.get_history = patched_get_history
+ analyzed_labels: List[NutritionLabel] = Field(
+ ..., description="List of nutrition labels extracted from the provided images"
+ )
-# Set up the system prompt
-system_prompt_generator = SystemPromptGenerator(
- background=[
- "You are an expert at extracting nutrition information from food labels.",
- "You can accurately read and parse nutrition facts panels from images.",
- "You understand serving sizes, measurements, and nutritional content.",
- "You can process multiple nutrition labels from multiple images.",
- ],
- steps=[
- "For each image:",
- "1. Identify the nutrition facts panel",
- "2. Extract all nutritional values and serving information",
- "3. Convert all values to their appropriate units",
- "4. Add the nutrition facts to the output list",
- ],
- output_instructions=[
- "For each nutrition label image:",
- "1. Extract serving size and servings per container",
- "2. Record all nutritional values in their specified units",
- "3. Ensure accuracy of measurements and conversions",
- "Return all nutrition facts in the list",
- ],
-)
-# Initialize the agent
-agent = BaseAgent(
+# Configure the nutrition analysis system
+nutrition_analyzer = BaseAgent(
config=BaseAgentConfig(
client=instructor.from_openai(openai.OpenAI()),
model="gpt-4o-mini",
- system_prompt_generator=system_prompt_generator,
+ system_prompt_generator=SystemPromptGenerator(
+ background=[
+ "You are a specialized nutrition label analyzer.",
+ "You excel at extracting precise nutritional information from food label images.",
+ "You understand various serving size formats and measurement units.",
+ "You can process multiple nutrition labels simultaneously.",
+ ],
+ steps=[
+ "For each nutrition label image:",
+ "1. Locate and identify the nutrition facts panel",
+ "2. Extract all serving information and nutritional values",
+ "3. Validate measurements and units for accuracy",
+ "4. Compile the nutrition facts into structured data",
+ ],
+ output_instructions=[
+ "For each analyzed nutrition label:",
+ "1. Record complete serving size information",
+ "2. Extract all nutrient values with correct units",
+ "3. Ensure all measurements are properly converted",
+ "4. Include all extracted labels in the final result",
+ ],
+ ),
memory=AgentMemory(),
- output_schema=NutritionOutputSchema,
+ input_schema=NutritionAnalysisInput,
+ output_schema=NutritionAnalysisOutput,
)
)
def main():
- print("Starting nutrition facts extraction...")
- # Get the directory where the script is located
- script_dir = os.path.dirname(os.path.abspath(__file__))
- # Go up one level to the basic-multimodal directory and then to test_images
- image_dir = os.path.join(os.path.dirname(script_dir), "test_images")
- image_paths = [os.path.join(image_dir, "nutrition_label_2.jpg")]
- print(f"Image paths: {image_paths}")
-
- # Create the message with the image
- message = ImageMessage(
- text="Extract all nutrition information from these food labels. Include serving size, calories, and all nutrient values.",
- image_paths=[str(path) for path in image_paths],
+ print("Starting nutrition label analysis...")
+
+ # Construct the path to the test images
+ script_directory = os.path.dirname(os.path.abspath(__file__))
+ test_images_directory = os.path.join(os.path.dirname(script_directory), "test_images")
+ image_path_1 = os.path.join(test_images_directory, "nutrition_label_1.png")
+ image_path_2 = os.path.join(test_images_directory, "nutrition_label_2.jpg")
+ # Create and submit the analysis request
+ analysis_request = NutritionAnalysisInput(
+ instruction_text="Please analyze these nutrition labels and extract all nutritional information.",
+ images=[instructor.Image.from_path(image_path_1), instructor.Image.from_path(image_path_2)],
)
- # Create input schema with the message
- input_schema = NutritionInputSchema(message=message)
- print("Created input schema with images")
-
- # Add the message to memory
- agent.memory.add_message("user", input_schema)
- print("Added message to memory")
-
try:
- # Get response
- print("Getting response...")
- response = agent.get_response()
- print("Got response successfully")
-
- for i, facts in enumerate(response.nutrition_facts, 1):
- print(f"\nNutrition Facts {i}:")
- print(f"Serving Size: {facts.serving_size}")
- print(f"Servings Per Container: {facts.servings_per_container}")
- print(f"Calories: {facts.calories}")
- print(f"Total Fat: {facts.total_fat}g")
- print(f"Saturated Fat: {facts.saturated_fat}g")
- print(f"Trans Fat: {facts.trans_fat}g")
- print(f"Cholesterol: {facts.cholesterol}mg")
- print(f"Sodium: {facts.sodium}mg")
- print(f"Total Carbohydrates: {facts.total_carbohydrates}g")
- print(f"Dietary Fiber: {facts.dietary_fiber}g")
- print(f"Total Sugars: {facts.total_sugars}g")
- print(f"Added Sugars: {facts.added_sugars}g")
- print(f"Protein: {facts.protein}g")
- print(f"Vitamin D: {facts.vitamin_d}mcg")
- print(f"Calcium: {facts.calcium}mg")
- print(f"Iron: {facts.iron}mg")
- print(f"Potassium: {facts.potassium}mg")
+ # Process the nutrition labels
+ print("Analyzing nutrition labels...")
+ analysis_result = nutrition_analyzer.run(analysis_request)
+ print("Analysis completed successfully")
+
+ # Display the results
+ for i, label in enumerate(analysis_result.analyzed_labels, 1):
+ print(f"\nNutrition Label {i}:")
+ print(f"Product Name: {label.product_name}")
+ print(f"Serving Size: {label.serving_size}")
+ print(f"Servings Per Container: {label.servings_per_container}")
+ print(f"Calories: {label.calories}")
+ print(f"Total Fat: {label.total_fat}g")
+ print(f"Saturated Fat: {label.saturated_fat}g")
+ print(f"Trans Fat: {label.trans_fat}g")
+ print(f"Cholesterol: {label.cholesterol}mg")
+ print(f"Sodium: {label.sodium}mg")
+ print(f"Total Carbohydrates: {label.total_carbohydrates}g")
+ print(f"Dietary Fiber: {label.dietary_fiber}g")
+ print(f"Total Sugars: {label.total_sugars}g")
+ print(f"Added Sugars: {label.added_sugars}g")
+ print(f"Protein: {label.protein}g")
+ print(f"Vitamin D: {label.vitamin_d}mcg")
+ print(f"Calcium: {label.calcium}mg")
+ print(f"Iron: {label.iron}mg")
+ print(f"Potassium: {label.potassium}mg")
except Exception as e:
- print(f"Error occurred: {str(e)}")
+ print(f"Analysis failed: {str(e)}")
raise