Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Image retention in OpenAI, Anthropic (#186). Removed unused `BaseMessageManager`
Dillon DuPonta7574174272d
diff --git a/libs/agent/agent/core/__init__.py b/libs/agent/agent/core/__init__.pyindex 7fa4aae156..7c013c802b 100644--- a/libs/agent/agent/core/__init__.py+++ b/libs/agent/agent/core/__init__.py@@ -2,7 +2,7 @@from .factory import BaseLoopfrom .messages import (- BaseMessageManager,+ StandardMessageManager,ImageRetentionConfig,)from .callbacks import (@@ -18,7 +18,7 @@ __all__ = ["BaseLoop","CallbackManager","CallbackHandler",- "BaseMessageManager",+ "StandardMessageManager","ImageRetentionConfig","BaseCallbackManager","ContentCallback",diff --git a/libs/agent/agent/core/messages.py b/libs/agent/agent/core/messages.pyindex 821ca82c5f..a53622a47d 100644--- a/libs/agent/agent/core/messages.py+++ b/libs/agent/agent/core/messages.py@@ -22,106 +22,6 @@ class ImageRetentionConfig:"""Check if image retention is enabled."""return self.num_images_to_keep is not None and self.num_images_to_keep > 0--class BaseMessageManager:- """Base class for message preparation and management."""-- def __init__(self, image_retention_config: Optional[ImageRetentionConfig] = None):- """Initialize the message manager.-- Args:- image_retention_config: Configuration for image retention- """- self.image_retention_config = image_retention_config or ImageRetentionConfig()- if self.image_retention_config.min_removal_threshold < 1:- raise ValueError("min_removal_threshold must be at least 1")-- # Track provider for message formatting- self.provider = "openai" # Default provider-- def set_provider(self, provider: str) -> None:- """Set the current provider to format messages for.-- Args:- provider: Provider name (e.g., 'openai', 'anthropic')- """- self.provider = provider.lower()-- def prepare_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:- """Prepare messages by applying image retention and caching as configured.-- Args:- messages: List of messages to prepare-- Returns:- Prepared messages- """- if self.image_retention_config.should_retain_images():- self._filter_images(messages)- if self.image_retention_config.enable_caching:- self._inject_caching(messages)- return messages-- def _filter_images(self, messages: List[Dict[str, Any]]) -> None:- """Filter messages to retain only the specified number of most recent images.-- Args:- messages: Messages to filter- """- # Find all tool result blocks that contain images- tool_results = [- item- for message in messages- for item in (message["content"] if isinstance(message["content"], list) else [])- if isinstance(item, dict) and item.get("type") == "tool_result"- ]-- # Count total images- total_images = sum(- 1- for result in tool_results- for content in result.get("content", [])- if isinstance(content, dict) and content.get("type") == "image"- )-- # Calculate how many images to remove- images_to_remove = total_images - (self.image_retention_config.num_images_to_keep or 0)- images_to_remove -= images_to_remove % self.image_retention_config.min_removal_threshold-- # Remove oldest images first- for result in tool_results:- if isinstance(result.get("content"), list):- new_content = []- for content in result["content"]:- if isinstance(content, dict) and content.get("type") == "image":- if images_to_remove > 0:- images_to_remove -= 1- continue- new_content.append(content)- result["content"] = new_content-- def _inject_caching(self, messages: List[Dict[str, Any]]) -> None:- """Inject caching control for recent message turns.-- Args:- messages: Messages to inject caching into- """- # Only apply cache_control for Anthropic API, not OpenAI- if self.provider != "anthropic":- return-- # Default to caching last 3 turns- turns_to_cache = 3- for message in reversed(messages):- if message["role"] == "user" and isinstance(content := message["content"], list):- if turns_to_cache:- turns_to_cache -= 1- content[-1]["cache_control"] = {"type": "ephemeral"}- else:- content[-1].pop("cache_control", None)- break--class StandardMessageManager:"""Manages messages in a standardized OpenAI format across different providers."""@@ -160,6 +60,7 @@ class StandardMessageManager:def get_messages(self) -> List[Dict[str, Any]]:"""Get all messages in standard format.+ This method applies image retention policy if configured.Returns:List of messagesdiff --git a/libs/agent/agent/providers/anthropic/loop.py b/libs/agent/agent/providers/anthropic/loop.pyindex 8b7ef1d683..dfdfbc6ce4 100644--- a/libs/agent/agent/providers/anthropic/loop.py+++ b/libs/agent/agent/providers/anthropic/loop.py@@ -283,8 +283,12 @@ class AnthropicLoop(BaseLoop):# Create new turn directory for this API callself._create_turn_dir()++ # Apply image retention policy+ self.message_manager.messages = messages.copy()+ prepared_messages = self.message_manager.get_messages()# Convert standard messages to Anthropic format using utility function- anthropic_messages, system_content = to_anthropic_format(messages.copy())+ anthropic_messages, system_content = to_anthropic_format(prepared_messages)# Use API handler to make API call with Anthropic formatresponse = await self.api_handler.make_api_call(diff --git a/libs/agent/agent/providers/openai/loop.py b/libs/agent/agent/providers/openai/loop.pyindex e791b8c98b..4095cdc0b1 100644--- a/libs/agent/agent/providers/openai/loop.py+++ b/libs/agent/agent/providers/openai/loop.py@@ -276,7 +276,7 @@ class OpenAILoop(BaseLoop):# Call APIscreen_size = await self.computer.interface.get_screen_size()response = await self.api_handler.send_initial_request(- messages=messages,+ messages=self.message_manager.get_messages(), # Apply image retention policydisplay_width=str(screen_size["width"]),display_height=str(screen_size["height"]),previous_response_id=self.last_response_id,@@ -397,7 +397,7 @@ class OpenAILoop(BaseLoop):# The API handler will extract this from the message historyif isinstance(self.last_response_id, str):response = await self.api_handler.send_computer_call_request(- messages=self.message_manager.messages,+ messages=self.message_manager.get_messages(), # Apply image retention policydisplay_width=str(screen_size["width"]),display_height=str(screen_size["height"]),previous_response_id=self.last_response_id, # Use instance variablediff --git a/libs/agent/agent/providers/uitars/clients/oaicompat.py b/libs/agent/agent/providers/uitars/clients/oaicompat.pyindex 423b1d3a54..a3778be6ad 100644--- a/libs/agent/agent/providers/uitars/clients/oaicompat.py+++ b/libs/agent/agent/providers/uitars/clients/oaicompat.py@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Anyimport aiohttpimport refrom .base import BaseUITarsClient+import asynciologger = logging.getLogger(__name__)@@ -144,7 +145,7 @@ class OAICompatClient(BaseUITarsClient):else:message = {"role": "user", "content": [{"type": "text", "text": item}]}final_messages.append(message)-+payload = {"model": self.model,"messages": final_messages,@@ -192,7 +193,8 @@ class OAICompatClient(BaseUITarsClient):# if 503, then the endpoint is still warming upif response.status == 503:- logger.error(f"Endpoint is still warming up, please try again later")+ logger.error(f"Endpoint is still warming up, trying again in 30 seconds...")+ await asyncio.sleep(30)raise Exception(f"Endpoint is still warming up: {response_text}")# Try to parse as JSON if the content type is appropriate