Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Image retention in OpenAI, Anthropic (#186). Removed unused `BaseMessageManager`

Dillon DuPonta7574174272d
diff --git a/libs/agent/agent/core/__init__.py b/libs/agent/agent/core/__init__.py
index 7fa4aae156..7c013c802b 100644
--- a/libs/agent/agent/core/__init__.py
+++ b/libs/agent/agent/core/__init__.py
@@ -2,7 +2,7 @@
from .factory import BaseLoop
from .messages import (
- BaseMessageManager,
+ StandardMessageManager,
ImageRetentionConfig,
)
from .callbacks import (
@@ -18,7 +18,7 @@ __all__ = [
"BaseLoop",
"CallbackManager",
"CallbackHandler",
- "BaseMessageManager",
+ "StandardMessageManager",
"ImageRetentionConfig",
"BaseCallbackManager",
"ContentCallback",
diff --git a/libs/agent/agent/core/messages.py b/libs/agent/agent/core/messages.py
index 821ca82c5f..a53622a47d 100644
--- a/libs/agent/agent/core/messages.py
+++ b/libs/agent/agent/core/messages.py
@@ -22,106 +22,6 @@ class ImageRetentionConfig:
"""Check if image retention is enabled."""
return self.num_images_to_keep is not None and self.num_images_to_keep > 0
-
-class BaseMessageManager:
- """Base class for message preparation and management."""
-
- def __init__(self, image_retention_config: Optional[ImageRetentionConfig] = None):
- """Initialize the message manager.
-
- Args:
- image_retention_config: Configuration for image retention
- """
- self.image_retention_config = image_retention_config or ImageRetentionConfig()
- if self.image_retention_config.min_removal_threshold < 1:
- raise ValueError("min_removal_threshold must be at least 1")
-
- # Track provider for message formatting
- self.provider = "openai" # Default provider
-
- def set_provider(self, provider: str) -> None:
- """Set the current provider to format messages for.
-
- Args:
- provider: Provider name (e.g., 'openai', 'anthropic')
- """
- self.provider = provider.lower()
-
- def prepare_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
- """Prepare messages by applying image retention and caching as configured.
-
- Args:
- messages: List of messages to prepare
-
- Returns:
- Prepared messages
- """
- if self.image_retention_config.should_retain_images():
- self._filter_images(messages)
- if self.image_retention_config.enable_caching:
- self._inject_caching(messages)
- return messages
-
- def _filter_images(self, messages: List[Dict[str, Any]]) -> None:
- """Filter messages to retain only the specified number of most recent images.
-
- Args:
- messages: Messages to filter
- """
- # Find all tool result blocks that contain images
- tool_results = [
- item
- for message in messages
- for item in (message["content"] if isinstance(message["content"], list) else [])
- if isinstance(item, dict) and item.get("type") == "tool_result"
- ]
-
- # Count total images
- total_images = sum(
- 1
- for result in tool_results
- for content in result.get("content", [])
- if isinstance(content, dict) and content.get("type") == "image"
- )
-
- # Calculate how many images to remove
- images_to_remove = total_images - (self.image_retention_config.num_images_to_keep or 0)
- images_to_remove -= images_to_remove % self.image_retention_config.min_removal_threshold
-
- # Remove oldest images first
- for result in tool_results:
- if isinstance(result.get("content"), list):
- new_content = []
- for content in result["content"]:
- if isinstance(content, dict) and content.get("type") == "image":
- if images_to_remove > 0:
- images_to_remove -= 1
- continue
- new_content.append(content)
- result["content"] = new_content
-
- def _inject_caching(self, messages: List[Dict[str, Any]]) -> None:
- """Inject caching control for recent message turns.
-
- Args:
- messages: Messages to inject caching into
- """
- # Only apply cache_control for Anthropic API, not OpenAI
- if self.provider != "anthropic":
- return
-
- # Default to caching last 3 turns
- turns_to_cache = 3
- for message in reversed(messages):
- if message["role"] == "user" and isinstance(content := message["content"], list):
- if turns_to_cache:
- turns_to_cache -= 1
- content[-1]["cache_control"] = {"type": "ephemeral"}
- else:
- content[-1].pop("cache_control", None)
- break
-
-
class StandardMessageManager:
"""Manages messages in a standardized OpenAI format across different providers."""
@@ -160,6 +60,7 @@ class StandardMessageManager:
def get_messages(self) -> List[Dict[str, Any]]:
"""Get all messages in standard format.
+ This method applies image retention policy if configured.
Returns:
List of messages
diff --git a/libs/agent/agent/providers/anthropic/loop.py b/libs/agent/agent/providers/anthropic/loop.py
index 8b7ef1d683..dfdfbc6ce4 100644
--- a/libs/agent/agent/providers/anthropic/loop.py
+++ b/libs/agent/agent/providers/anthropic/loop.py
@@ -283,8 +283,12 @@ class AnthropicLoop(BaseLoop):
# Create new turn directory for this API call
self._create_turn_dir()
+
+ # Apply image retention policy
+ self.message_manager.messages = messages.copy()
+ prepared_messages = self.message_manager.get_messages()
# Convert standard messages to Anthropic format using utility function
- anthropic_messages, system_content = to_anthropic_format(messages.copy())
+ anthropic_messages, system_content = to_anthropic_format(prepared_messages)
# Use API handler to make API call with Anthropic format
response = await self.api_handler.make_api_call(
diff --git a/libs/agent/agent/providers/openai/loop.py b/libs/agent/agent/providers/openai/loop.py
index e791b8c98b..4095cdc0b1 100644
--- a/libs/agent/agent/providers/openai/loop.py
+++ b/libs/agent/agent/providers/openai/loop.py
@@ -276,7 +276,7 @@ class OpenAILoop(BaseLoop):
# Call API
screen_size = await self.computer.interface.get_screen_size()
response = await self.api_handler.send_initial_request(
- messages=messages,
+ messages=self.message_manager.get_messages(), # Apply image retention policy
display_width=str(screen_size["width"]),
display_height=str(screen_size["height"]),
previous_response_id=self.last_response_id,
@@ -397,7 +397,7 @@ class OpenAILoop(BaseLoop):
# The API handler will extract this from the message history
if isinstance(self.last_response_id, str):
response = await self.api_handler.send_computer_call_request(
- messages=self.message_manager.messages,
+ messages=self.message_manager.get_messages(), # Apply image retention policy
display_width=str(screen_size["width"]),
display_height=str(screen_size["height"]),
previous_response_id=self.last_response_id, # Use instance variable
diff --git a/libs/agent/agent/providers/uitars/clients/oaicompat.py b/libs/agent/agent/providers/uitars/clients/oaicompat.py
index 423b1d3a54..a3778be6ad 100644
--- a/libs/agent/agent/providers/uitars/clients/oaicompat.py
+++ b/libs/agent/agent/providers/uitars/clients/oaicompat.py
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Any
import aiohttp
import re
from .base import BaseUITarsClient
+import asyncio
logger = logging.getLogger(__name__)
@@ -144,7 +145,7 @@ class OAICompatClient(BaseUITarsClient):
else:
message = {"role": "user", "content": [{"type": "text", "text": item}]}
final_messages.append(message)
-
+
payload = {
"model": self.model,
"messages": final_messages,
@@ -192,7 +193,8 @@ class OAICompatClient(BaseUITarsClient):
# if 503, then the endpoint is still warming up
if response.status == 503:
- logger.error(f"Endpoint is still warming up, please try again later")
+ logger.error(f"Endpoint is still warming up, trying again in 30 seconds...")
+ await asyncio.sleep(30)
raise Exception(f"Endpoint is still warming up: {response_text}")
# Try to parse as JSON if the content type is appropriate