Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Added disable_response_storage
Dillon DuPont9068ec32d834
diff --git a/libs/python/agent/agent/core/agent.py b/libs/python/agent/agent/core/agent.pyindex e9d3b86668..d6a400fa01 100644--- a/libs/python/agent/agent/core/agent.py+++ b/libs/python/agent/agent/core/agent.py@@ -29,6 +29,7 @@ class ComputerAgent:trajectory_dir: str = "trajectories",only_n_most_recent_images: Optional[int] = None,verbosity: int = logging.INFO,+ disable_response_storage: bool = False,):"""Initialize the ComputerAgent.@@ -45,6 +46,7 @@ class ComputerAgent:trajectory_dir: Directory to save the trajectory.only_n_most_recent_images: Maximum number of recent screenshots to include in API requests.verbosity: Logging level.+ disable_response_storage: Whether to disable response storage on the provider side. Turn this on if you are participating in a Zero Data Retention policy."""# Basic agent configurationself.max_retries = max_retries@@ -55,6 +57,7 @@ class ComputerAgent:self._retry_count = 0self._initialized = Falseself._in_context = False+ self.disable_response_storage = disable_response_storage# Set logging levellogger.setLevel(verbosity)@@ -105,6 +108,7 @@ class ComputerAgent:trajectory_dir=trajectory_dir,only_n_most_recent_images=only_n_most_recent_images,provider_base_url=self.provider_base_url,+ disable_response_storage=disable_response_storage,)except ValueError as e:logger.error(f"Failed to create loop: {str(e)}")diff --git a/libs/python/agent/agent/core/base.py b/libs/python/agent/agent/core/base.pyindex fe0f07ad9a..8e40313f39 100644--- a/libs/python/agent/agent/core/base.py+++ b/libs/python/agent/agent/core/base.py@@ -29,6 +29,7 @@ class BaseLoop(ABC):save_trajectory: bool = True,only_n_most_recent_images: Optional[int] = 2,callback_handlers: Optional[List[CallbackHandler]] = None,+ disable_response_storage: bool = False,**kwargs,):"""Initialize base agent loop.@@ -43,6 +44,7 @@ class BaseLoop(ABC):base_dir: Base directory for saving experiment datasave_trajectory: Whether to save trajectory dataonly_n_most_recent_images: Maximum number of recent screenshots to include in API requests+ disable_response_storage: Whether to disable response storage on the provider side. Turn this on if you are participating in a Zero Data Retention policy.**kwargs: Additional provider-specific arguments"""self.computer = computer@@ -54,6 +56,7 @@ class BaseLoop(ABC):self.base_dir = base_dirself.save_trajectory = save_trajectoryself.only_n_most_recent_images = only_n_most_recent_images+ self.disable_response_storage = disable_response_storageself._kwargs = kwargs# Initialize message managerdiff --git a/libs/python/agent/agent/core/factory.py b/libs/python/agent/agent/core/factory.pyindex f0c6046e5d..9d1ef47643 100644--- a/libs/python/agent/agent/core/factory.py+++ b/libs/python/agent/agent/core/factory.py@@ -30,6 +30,7 @@ class LoopFactory:only_n_most_recent_images: Optional[int] = None,acknowledge_safety_check_callback: Optional[Callable[[str], Awaitable[bool]]] = None,provider_base_url: Optional[str] = None,+ disable_response_storage: bool = False,) -> BaseLoop:"""Create and return an appropriate loop instance based on type."""if loop_type == AgentLoop.ANTHROPIC:@@ -49,6 +50,7 @@ class LoopFactory:save_trajectory=save_trajectory,base_dir=trajectory_dir,only_n_most_recent_images=only_n_most_recent_images,+ disable_response_storage=disable_response_storage,)elif loop_type == AgentLoop.OPENAI:# Lazy import OpenAILoop only when needed@@ -68,6 +70,7 @@ class LoopFactory:base_dir=trajectory_dir,only_n_most_recent_images=only_n_most_recent_images,acknowledge_safety_check_callback=acknowledge_safety_check_callback,+ disable_response_storage=disable_response_storage,)elif loop_type == AgentLoop.OMNI:# Lazy import OmniLoop and related classes only when needed@@ -97,6 +100,7 @@ class LoopFactory:only_n_most_recent_images=only_n_most_recent_images,parser=OmniParser(),provider_base_url=provider_base_url,+ disable_response_storage=disable_response_storage,)elif loop_type == AgentLoop.UITARS:# Lazy import UITARSLoop only when needed@@ -117,6 +121,7 @@ class LoopFactory:only_n_most_recent_images=only_n_most_recent_images,provider_base_url=provider_base_url,provider=provider,+ disable_response_storage=disable_response_storage,)else:raise ValueError(f"Unsupported loop type: {loop_type}")diff --git a/libs/python/agent/agent/core/messages.py b/libs/python/agent/agent/core/messages.pyindex d2c7055806..9218790fa5 100644--- a/libs/python/agent/agent/core/messages.py+++ b/libs/python/agent/agent/core/messages.py@@ -69,6 +69,44 @@ class StandardMessageManager:return self._apply_image_retention(self.messages)return self.messages+ def add_openai_response(self, response: Dict[str, Any]) -> None:+ """Add OpenAI response output to message history.++ This method extracts the output items from an OpenAI response and adds them+ as assistant messages to maintain conversation state manually.++ Args:+ response: OpenAI API response containing output items+ """+ if not isinstance(response, dict) or "output" not in response:+ logger.warning("Invalid OpenAI response format for adding to message history")+ return++ output_items = response.get("output", [])+ if not isinstance(output_items, list):+ logger.warning("OpenAI response output is not a list")+ return++ # Convert output items to assistant message content+ assistant_content = []+ for item in output_items:+ if not isinstance(item, dict):+ continue++ item_type = item.get("type")+ if item_type == "output_text":+ assistant_content.append({+ "type": "text",+ "text": item.get("text", "")+ })+ elif item_type == "computer_call":+ # Keep computer calls as-is for tool execution tracking+ assistant_content.append(item)++ # Add as assistant message if we have content+ if assistant_content:+ self.add_assistant_message(assistant_content)+def _apply_image_retention(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:"""Apply image retention policy to messages.diff --git a/libs/python/agent/agent/providers/openai/api_handler.py b/libs/python/agent/agent/providers/openai/api_handler.pyindex e4b5de7552..790bfc96b6 100644--- a/libs/python/agent/agent/providers/openai/api_handler.py+++ b/libs/python/agent/agent/providers/openai/api_handler.py@@ -15,11 +15,12 @@ logger = logging.getLogger(__name__)class OpenAIAPIHandler:"""Handler for OpenAI API interactions."""- def __init__(self, loop: "OpenAILoop"):+ def __init__(self, loop: "OpenAILoop", disable_response_storage: bool = False):"""Initialize the API handler.Args:loop: OpenAI loop instance+ disable_response_storage: Whether to disable response storage"""self.loop = loopself.api_key = os.getenv("OPENAI_API_KEY")@@ -45,7 +46,7 @@ class OpenAIAPIHandler:display_width: str,display_height: str,previous_response_id: Optional[str] = None,- os_type: str,+ os_type: str = "mac",) -> Dict[str, Any]:"""Send an initial request to the OpenAI API with a screenshot.@@ -61,10 +62,7 @@ class OpenAIAPIHandler:# Convert from our internal OS types to the ones OpenAI expectsif os_type == "macos":os_type = "mac"- elif os_type == "linux":- os_type = "ubuntu"-- if os_type not in ["mac", "windows", "ubuntu", "browser"]:+ if os_type not in ["mac", "windows", "linux", "browser"]:raise ValueError(f"Invalid OS type: {os_type}")# Convert display dimensions to integers@@ -143,7 +141,7 @@ class OpenAIAPIHandler:],"input": input_array,"reasoning": {- "generate_summary": "concise",+ "summary": "concise",},"truncation": "auto",}@@ -207,10 +205,8 @@ class OpenAIAPIHandler:# Convert from our internal OS types to the ones OpenAI expectsif os_type == "macos":os_type = "mac"- elif os_type == "linux":- os_type = "ubuntu"- if os_type not in ["mac", "windows", "ubuntu", "browser"]:+ if os_type not in ["mac", "windows", "linux", "browser"]:raise ValueError(f"Invalid OS type: {os_type}")# Convert display dimensions to integers@@ -289,6 +285,9 @@ class OpenAIAPIHandler:},}],+ "reasoning": {+ "summary": "concise",+ },"truncation": "auto",}diff --git a/libs/python/agent/agent/providers/openai/loop.py b/libs/python/agent/agent/providers/openai/loop.pyindex cc9a07dbfe..cb9fdbe73c 100644--- a/libs/python/agent/agent/providers/openai/loop.py+++ b/libs/python/agent/agent/providers/openai/loop.py@@ -40,6 +40,7 @@ class OpenAILoop(BaseLoop):retry_delay: float = 1.0,save_trajectory: bool = True,acknowledge_safety_check_callback: Optional[Callable[[str], Awaitable[bool]]] = None,+ disable_response_storage: bool = False,**kwargs,):"""Initialize the OpenAI loop.@@ -54,6 +55,7 @@ class OpenAILoop(BaseLoop):retry_delay: Delay between retries in secondssave_trajectory: Whether to save trajectory dataacknowledge_safety_check_callback: Optional callback for safety check acknowledgment+ disable_response_storage: Whether to disable response storage on the provider side. Turn this on if you are participating in a Zero Data Retention policy.**kwargs: Additional provider-specific arguments"""# Always use computer-use-preview model@@ -72,6 +74,7 @@ class OpenAILoop(BaseLoop):base_dir=base_dir,save_trajectory=save_trajectory,only_n_most_recent_images=only_n_most_recent_images,+ disable_response_storage=disable_response_storage,**kwargs,)@@ -90,7 +93,7 @@ class OpenAILoop(BaseLoop):self.loop_task = None # Store the loop task for cancellation# Initialize handlers- self.api_handler = OpenAIAPIHandler(self)+ self.api_handler = OpenAIAPIHandler(self, self.disable_response_storage)self.response_handler = OpenAIResponseHandler(self)# Initialize tool manager with callback@@ -275,24 +278,47 @@ class OpenAILoop(BaseLoop):# Call APIscreen_size = await self.computer.interface.get_screen_size()- response = await self.api_handler.send_initial_request(- messages=self.message_manager.get_messages(), # Apply image retention policy- display_width=str(screen_size["width"]),- display_height=str(screen_size["height"]),- previous_response_id=self.last_response_id,- os_type=self.computer.os_type,- )-- # Store response ID for next request- # OpenAI API response structure: the ID is in the response dictionary- if isinstance(response, dict) and "id" in response:- self.last_response_id = response["id"] # Update instance variable- logger.info(f"Received response with ID: {self.last_response_id}")++ # Choose API call method based on disable_response_storage setting+ if self.disable_response_storage:+ # Manual conversation state management - always send full message history+ response = await self.api_handler.send_initial_request(+ messages=self.message_manager.get_messages(), # Apply image retention policy+ display_width=str(screen_size["width"]),+ display_height=str(screen_size["height"]),+ previous_response_id=None, # Don't use response chaining+ os_type=self.computer.os_type,+ )else:- logger.warning(- f"Could not find response ID in OpenAI response: {type(response)}"+ # Use OpenAI's response storage with previous_response_id+ response = await self.api_handler.send_initial_request(+ messages=self.message_manager.get_messages(), # Apply image retention policy+ display_width=str(screen_size["width"]),+ display_height=str(screen_size["height"]),+ previous_response_id=self.last_response_id,+ os_type=self.computer.os_type,)- # Don't reset last_response_id to None - keep the previous value if available++ from pprint import pprint++ print("========== send_initial_request ===========")+ pprint(response)+ print("===========================================")++ if self.disable_response_storage:+ # Manual conversation state management - add response to message history+ self.message_manager.add_openai_response(response)+ else:+ # Store response ID for next request+ # OpenAI API response structure: the ID is in the response dictionary+ if isinstance(response, dict) and "id" in response:+ self.last_response_id = response["id"] # Update instance variable+ logger.info(f"Received response with ID: {self.last_response_id}")+ else:+ logger.warning(+ f"Could not find response ID in OpenAI response: {type(response)}"+ )+ # Don't reset last_response_id to None - keep the previous value if available# Log standardized response for ease of parsing@@ -393,27 +419,54 @@ class OpenAILoop(BaseLoop):)self.message_manager.add_user_message([computer_call_output])- # For follow-up requests with previous_response_id, we only need to send- # the computer_call_output, not the full message history- # The API handler will extract this from the message history- if isinstance(self.last_response_id, str):- response = await self.api_handler.send_computer_call_request(+ # Choose API call method based on disable_response_storage setting+ if self.disable_response_storage:+ # Manual conversation state management - send full message history+ response = await self.api_handler.send_initial_request(messages=self.message_manager.get_messages(), # Apply image retention policydisplay_width=str(screen_size["width"]),display_height=str(screen_size["height"]),- previous_response_id=self.last_response_id, # Use instance variable+ previous_response_id=None, # Don't use response chainingos_type=self.computer.os_type,)-- # Store response ID for next request- if isinstance(response, dict) and "id" in response:- self.last_response_id = response["id"] # Update instance variable- logger.info(f"Received response with ID: {self.last_response_id}")++ from pprint import pprint++ print("========== send_initial_request (manual mode) ===========")+ pprint(response)+ print("========================================================")++ # Add response to message history for manual state management+ self.message_manager.add_openai_response(response)else:- logger.warning(- f"Could not find response ID in OpenAI response: {type(response)}"- )- # Keep using the previous response ID if we can't find a new one+ # Use OpenAI's response storage with previous_response_id+ # For follow-up requests with previous_response_id, we only need to send+ # the computer_call_output, not the full message history+ # The API handler will extract this from the message history+ if isinstance(self.last_response_id, str):+ response = await self.api_handler.send_computer_call_request(+ messages=self.message_manager.get_messages(), # Apply image retention policy+ display_width=str(screen_size["width"]),+ display_height=str(screen_size["height"]),+ previous_response_id=self.last_response_id, # Use instance variable+ os_type=self.computer.os_type,+ )++ from pprint import pprint++ print("========== send_computer_call_request ===========")+ pprint(response)+ print("============================================")++ # Store response ID for next request+ if isinstance(response, dict) and "id" in response:+ self.last_response_id = response["id"] # Update instance variable+ logger.info(f"Received response with ID: {self.last_response_id}")+ else:+ logger.warning(+ f"Could not find response ID in OpenAI response: {type(response)}"+ )+ # Keep using the previous response ID if we can't find a new one# Process the response# await self.response_handler.process_response(response, queue)@@ -455,20 +508,3 @@ class OpenAILoop(BaseLoop):})await queue.put(None) # Signal that we're done-- def get_last_response_id(self) -> Optional[str]:- """Get the last response ID.-- Returns:- The last response ID or None if no response has been received- """- return self.last_response_id-- def set_last_response_id(self, response_id: str) -> None:- """Set the last response ID.-- Args:- response_id: OpenAI response ID to set- """- self.last_response_id = response_id- logger.info(f"Manually set response ID to: {self.last_response_id}")diff --git a/libs/python/agent/pyproject.toml b/libs/python/agent/pyproject.tomlindex 7f6af835fc..c967f5789c 100644--- a/libs/python/agent/pyproject.toml+++ b/libs/python/agent/pyproject.toml@@ -91,6 +91,9 @@ all = ["mlx-vlm>=0.1.27; sys_platform == 'darwin'"]+[tool.uv]+constraint-dependencies = ["fastrtc>0.43.0", "mlx-audio>0.2.3"]+[tool.pdm]distribution = truediff --git a/scripts/build-uv.sh b/scripts/build-uv.shnew file mode 100755index 0000000000..3b1db41a6a--- /dev/null+++ b/scripts/build-uv.sh@@ -0,0 +1,183 @@+#!/bin/bash++# Exit on error+set -e++# Colors for output+RED='\033[0;31m'+GREEN='\033[0;32m'+BLUE='\033[0;34m'+YELLOW='\033[1;33m'+NC='\033[0m' # No Color++# Function to print step information+print_step() {+ echo -e "${BLUE}==> $1${NC}"+}++# Function to print success message+print_success() {+ echo -e "${GREEN}==> Success: $1${NC}"+}++# Function to print error message+print_error() {+ echo -e "${RED}==> Error: $1${NC}" >&2+}++# Function to print warning message+print_warning() {+ echo -e "${YELLOW}==> Warning: $1${NC}"+}++# Function to check if UV is installed+check_uv() {+ if command -v uv &> /dev/null; then+ print_success "UV is already installed"+ uv --version+ return 0+ else+ return 1+ fi+}++# Function to install UV+install_uv() {+ print_step "UV not found. Installing UV..."++ # Detect OS+ if [[ "$OSTYPE" == "linux-gnu"* ]] || [[ "$OSTYPE" == "darwin"* ]]; then+ print_step "Installing UV for Unix-like system..."+ curl -LsSf https://astral.sh/uv/install.sh | sh++ # Add UV to PATH for current session+ export PATH="$HOME/.cargo/bin:$PATH"++ # Check if installation was successful+ if command -v uv &> /dev/null; then+ print_success "UV installed successfully"+ uv --version+ else+ print_error "UV installation failed"+ print_step "Please restart your terminal and try again, or install manually:"+ echo " curl -LsSf https://astral.sh/uv/install.sh | sh"+ exit 1+ fi+ elif [[ "$OSTYPE" == "msys" ]] || [[ "$OSTYPE" == "cygwin" ]]; then+ print_error "For Windows, please use PowerShell and run:"+ echo " powershell -ExecutionPolicy ByPass -c \"irm https://astral.sh/uv/install.ps1 | iex\""+ exit 1+ else+ print_error "Unsupported operating system: $OSTYPE"+ print_step "Please install UV manually from: https://docs.astral.sh/uv/getting-started/installation/"+ exit 1+ fi+}++# Get the script's directory+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"+PROJECT_ROOT="$( cd "${SCRIPT_DIR}/.." && pwd )"++# Change to project root+cd "$PROJECT_ROOT"++# Check if UV is installed, install if not+if ! check_uv; then+ install_uv+fi++# Load environment variables from .env.local+if [ -f .env.local ]; then+ print_step "Loading environment variables from .env.local..."+ set -a+ source .env.local+ set +a+ print_success "Environment variables loaded"+else+ print_error ".env.local file not found"+ exit 1+fi++# Clean up existing environments and cache+print_step "Cleaning up existing environments..."+find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true+find . -type d -name ".pytest_cache" -exec rm -rf {} + 2>/dev/null || true+find . -type d -name "dist" -exec rm -rf {} + 2>/dev/null || true+find . -type d -name ".venv" -exec rm -rf {} + 2>/dev/null || true+find . -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true+print_success "Environment cleanup complete"++# Install Python 3.12 using UV+print_step "Installing Python 3.12 using UV..."+uv python install 3.12+print_success "Python 3.12 installed"++# Create virtual environment using UV+print_step "Creating virtual environment with UV..."+uv venv .venv --python 3.12+print_success "Virtual environment created"++# Activate virtual environment+print_step "Activating virtual environment..."+source .venv/bin/activate+print_success "Virtual environment activated"++# Function to install a package and its dependencies using UV+install_package() {+ local package_dir=$1+ local package_name=$2+ local extras=$3+ print_step "Installing ${package_name} with UV..."+ cd "$package_dir"++ if [ -f "pyproject.toml" ]; then+ if [ -n "$extras" ]; then+ uv pip install -e ".[${extras}]"+ else+ uv pip install -e .+ fi+ else+ print_error "No pyproject.toml found in ${package_dir}"+ return 1+ fi++ cd "$PROJECT_ROOT"+}++# Install packages in order of dependency+print_step "Installing packages in development mode with UV..."++# Install core first (base package with telemetry support)+install_package "libs/python/core" "core"++# Install pylume (base dependency)+install_package "libs/python/pylume" "pylume"++# Install computer with all its dependencies and extras+install_package "libs/python/computer" "computer" "all"++# Install omniparser+install_package "libs/python/som" "som"++# Install agent with all its dependencies and extras+install_package "libs/python/agent" "agent" "all"++# Install computer-server+install_package "libs/python/computer-server" "computer-server"++# Install mcp-server+install_package "libs/python/mcp-server" "mcp-server"++# Install development tools from root project+print_step "Installing development dependencies with UV..."+uv pip install -e ".[dev,test,docs]"++# Create a .env file for VS Code to use the virtual environment+print_step "Creating .env file for VS Code..."+echo "PYTHONPATH=${PROJECT_ROOT}/libs/python/core:${PROJECT_ROOT}/libs/python/computer:${PROJECT_ROOT}/libs/python/agent:${PROJECT_ROOT}/libs/python/som:${PROJECT_ROOT}/libs/python/pylume:${PROJECT_ROOT}/libs/python/computer-server:${PROJECT_ROOT}/libs/python/mcp-server" > .env++print_success "All packages installed successfully with UV!"+print_step "Your virtual environment is ready. To activate it:"+echo " source .venv/bin/activate"+print_step "UV provides fast dependency resolution and installation."+print_step "You can also use 'uv run' to run commands in the virtual environment without activation."