Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-bench): robust agent runs with retry/backoff and fixed cb run stop (#1203)
ddupont1fdc813d7ff1
diff --git a/libs/cua-bench/cua_bench/agents/base.py b/libs/cua-bench/cua_bench/agents/base.pyindex ec3b35f39f..26cb772297 100644--- a/libs/cua-bench/cua_bench/agents/base.py+++ b/libs/cua-bench/cua_bench/agents/base.py@@ -17,6 +17,7 @@ class FailureMode(Enum):NONE = "none"UNKNOWN = "unknown"MAX_STEPS_EXCEEDED = "max_steps_exceeded"+ API_ERROR = "api_error"@dataclassdiff --git a/libs/cua-bench/cua_bench/agents/cua_agent.py b/libs/cua-bench/cua_bench/agents/cua_agent.pyindex 12c04fe21c..e4a783a4a3 100644--- a/libs/cua-bench/cua_bench/agents/cua_agent.py+++ b/libs/cua-bench/cua_bench/agents/cua_agent.py@@ -1,5 +1,6 @@"""CUA Agent implementation using the Computer Agent SDK."""+import asyncioimport base64import sysfrom pathlib import Path@@ -8,6 +9,31 @@ from typing import TYPE_CHECKING, Any, Dict, Listfrom . import register_agentfrom .base import AgentResult, BaseAgent, FailureMode++def _is_retryable_api_error(exc: BaseException) -> bool:+ """Return True if the exception is a transient error that can be retried."""+ if isinstance(exc, (asyncio.TimeoutError, TimeoutError)):+ return True+ try:+ import litellm.exceptions as _le++ if isinstance(+ exc,+ (+ _le.RateLimitError,+ _le.ServiceUnavailableError,+ _le.APIConnectionError,+ _le.Timeout,+ _le.InternalServerError,+ ),+ ):+ return True+ except Exception:+ pass+ msg = str(exc).lower()+ return any(k in msg for k in ("timeout", "rate limit", "503", "502", "429", "connection"))++if TYPE_CHECKING:from ..computers import DesktopSession@@ -20,6 +46,11 @@ class CuaAgent(BaseAgent):super().__init__(**kwargs)self.model = kwargs.get("model", "anthropic/claude-sonnet-4-20250514")self.max_steps = kwargs.get("max_steps", 100)+ # Number of times to retry the entire task when a transient API error occurs.+ # Task-level retry restarts agent.run() from scratch but does NOT reset the+ # desktop environment, so it is unsafe by default for evaluations.+ # Only enable via --agent-kwarg task_retries=N when the task is idempotent.+ self.task_retries = int(kwargs.get("task_retries", 0))@staticmethoddef name() -> str:@@ -200,82 +231,108 @@ class CuaAgent(BaseAgent):)print("CUA Agent initialized with model:", self.model)- # Run the agent and track usage- try:- total_usage = {- "prompt_tokens": 0,- "completion_tokens": 0,- "total_tokens": 0,- "response_cost": 0.0,- }-- step = 0- task_completed = False-- async for result in agent.run(instruction):- sys.stdout.flush() # Flush output-- step += 1- for k in total_usage:- total_usage[k] += result["usage"].get(k, 0)-- # Record agent step to tracer- if tracer:- try:- # Take screenshot- screenshot = await session.screenshot()- # Record the step with metadata- tracer.record(- "agent_step",- {- "step": step,- "agent": self.name(),- "model": self.model,- "usage": result["usage"],- "output": result["output"],- },- [screenshot],- )- except Exception as e:- print(f"Warning: Failed to record agent step to tracer: {e}")-- # Check if we've reached max_steps- if step >= self.max_steps:- print(f"\n[Max steps reached] Stopped at step {step}/{self.max_steps}")- break-- # Check if task is completed (agent returned done or similar)-- for item in result["output"]:- if item["type"] == "message":- if "DONE" in item["content"][0]["text"]:- print(f"\n[Task completed] Agent indicated completion at step {step}")- task_completed = True- break-- print(f"\nTotal usage: {total_usage}")- print(f"Steps completed: {step}/{self.max_steps}")-- # Determine failure mode- if task_completed:- failure_mode = FailureMode.NONE- elif step >= self.max_steps:- failure_mode = FailureMode.MAX_STEPS_EXCEEDED- else:- failure_mode = FailureMode.NONE # Completed within max_steps-- return AgentResult(- total_input_tokens=total_usage.get("prompt_tokens", 0),- total_output_tokens=total_usage.get("completion_tokens", 0),- failure_mode=failure_mode,- )- except Exception as e:- print(f"Agent execution failed: {e}")- import traceback-- traceback.print_exc()- return AgentResult(- total_input_tokens=0,- total_output_tokens=0,- failure_mode=FailureMode.UNKNOWN,- )+ # Run the agent and track usage (with task-level retry on transient API errors)+ total_usage = {+ "prompt_tokens": 0,+ "completion_tokens": 0,+ "total_tokens": 0,+ "response_cost": 0.0,+ }+ last_exc: BaseException | None = None+ for attempt in range(self.task_retries + 1):+ if attempt > 0:+ delay = 5.0 * (2 ** (attempt - 1))+ print(+ f"[cua-agent] Retrying task (attempt {attempt + 1}/{self.task_retries + 1}) "+ f"after transient error: {last_exc!r}. Waiting {delay:.0f}s …"+ )+ await asyncio.sleep(delay)++ try:+ step = 0+ task_completed = False++ async for result in agent.run(instruction):+ sys.stdout.flush() # Flush output++ step += 1+ for k in total_usage:+ total_usage[k] += result["usage"].get(k, 0)++ # Record agent step to tracer+ if tracer:+ try:+ # Take screenshot+ screenshot = await session.screenshot()+ # Record the step with metadata+ usage = result["usage"]+ if hasattr(usage, "model_dump"):+ usage = usage.model_dump()+ elif not isinstance(usage, dict):+ usage = dict(usage)+ tracer.record(+ "agent_step",+ {+ "step": step,+ "agent": self.name(),+ "model": self.model,+ "usage": usage,+ "output": result["output"],+ },+ [screenshot],+ )+ except Exception as e:+ print(f"Warning: Failed to record agent step to tracer: {e}")++ # Check if we've reached max_steps+ if step >= self.max_steps:+ print(f"\n[Max steps reached] Stopped at step {step}/{self.max_steps}")+ break++ # Check if task is completed (agent returned done or similar)+ for item in result["output"]:+ if item["type"] == "message":+ if "DONE" in item["content"][0]["text"]:+ print(+ f"\n[Task completed] Agent indicated completion at step {step}"+ )+ task_completed = True+ break++ print(f"\nTotal usage: {total_usage}")+ print(f"Steps completed: {step}/{self.max_steps}")++ # Determine failure mode+ if task_completed:+ failure_mode = FailureMode.NONE+ elif step >= self.max_steps:+ failure_mode = FailureMode.MAX_STEPS_EXCEEDED+ else:+ failure_mode = FailureMode.NONE # Completed within max_steps++ return AgentResult(+ total_input_tokens=total_usage.get("prompt_tokens", 0),+ total_output_tokens=total_usage.get("completion_tokens", 0),+ failure_mode=failure_mode,+ )++ except Exception as exc:+ last_exc = exc+ if attempt < self.task_retries and _is_retryable_api_error(exc):+ # Will retry in the next loop iteration+ continue++ # Non-retryable or out of retries+ import traceback++ print(f"Agent execution failed: {exc}")+ traceback.print_exc()++ failure_mode = (+ FailureMode.API_ERROR if _is_retryable_api_error(exc) else FailureMode.UNKNOWN+ )+ return AgentResult(+ total_input_tokens=0,+ total_output_tokens=0,+ failure_mode=failure_mode,+ )diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.pyindex 9b04510087..573eee3970 100644--- a/libs/cua-bench/cua_bench/cli/commands/run.py+++ b/libs/cua-bench/cua_bench/cli/commands/run.py@@ -586,6 +586,8 @@ def cmd_stop(args) -> int:async def _cmd_stop_async(args) -> int:"""Stop a run asynchronously."""+ import signal+from cua_bench.sessions import list_sessions, managerfrom cua_bench.sessions.providers.docker import DockerProvider@@ -594,12 +596,33 @@ async def _cmd_stop_async(args) -> int:print(f"{RED}Error: run_id required{RESET}")return 1+ # Kill the background orchestrator process first so it can't restart containers+ pid_file = _get_run_output_dir(run_id) / "run.pid"+ if pid_file.exists():+ try:+ pid = int(pid_file.read_text().strip())+ os.kill(pid, signal.SIGTERM)+ print(f"{CYAN}Sent SIGTERM to orchestrator process (pid {pid}){RESET}")+ # Give it a moment to clean up, then SIGKILL if still alive+ await asyncio.sleep(2)+ try:+ sigkill = getattr(signal, "SIGKILL", None)+ if sigkill is not None:+ os.kill(pid, sigkill)+ except ProcessLookupError:+ pass # Already exited cleanly+ except (ValueError, ProcessLookupError):+ pass # PID file stale or process already gone+ finally:+ pid_file.unlink(missing_ok=True)+sessions = list_sessions()run_sessions = [s for s in sessions if s.get("run_id") == run_id]if not run_sessions:- print(f"{RED}Error: No sessions found for run: {run_id}{RESET}")- return 1+ print(f"{YELLOW}No active sessions found for run: {run_id}{RESET}")+ print(f"\n{GREEN}✓ Run stopped{RESET}")+ return 0print(f"{CYAN}Stopping run: {run_id} ({len(run_sessions)} sessions)...{RESET}")@@ -1762,12 +1785,18 @@ async def _cmd_run_dataset_async(args) -> int:log_handle = open(global_log_file, "w", encoding="utf-8", buffering=1)# Start detached subprocess- subprocess.Popen(+ proc = subprocess.Popen(cmd,stdout=log_handle,stderr=subprocess.STDOUT,env=env_vars,)+ # Close the parent's copy of the fd; the subprocess has its own inherited copy.+ log_handle.close()++ # Save PID so `cb run stop` can kill the process+ pid_file = output_dir / "run.pid"+ pid_file.write_text(str(proc.pid))return 0diff --git a/libs/cua-bench/pyproject.toml b/libs/cua-bench/pyproject.tomlindex 19d0e29c23..399e71bb9b 100644--- a/libs/cua-bench/pyproject.toml+++ b/libs/cua-bench/pyproject.toml@@ -1,6 +1,6 @@[project]name = "cua-bench"-version = "0.2.6"+version = "0.2.5"description = "Toolkit for computer-use RL environments and benchmarks"readme = "README.md"license = "MIT"@@ -59,7 +59,7 @@ dependencies = [[project.optional-dependencies]# Full installation with all featuresall = [- "cua-bench[browser,agents,cloud]",+ "cua-bench[browser,agents,cloud,otel]",]# Browser automation (for simulated environments)@@ -73,6 +73,13 @@ agents = ["anthropic>=0.26.0",]+# OpenTelemetry tracing and metrics export+otel = [+ "opentelemetry-api>=1.20.0",+ "opentelemetry-sdk>=1.20.0",+ "opentelemetry-exporter-otlp-proto-http>=1.20.0",+]+# Cloud batch execution (Google Cloud)cloud = ["google-cloud-batch>=0.17.0",@@ -80,6 +87,7 @@ cloud = ["google-cloud-storage>=2.10.0","gcloud-aio-storage>=9.0.0","grpcio>=1.60.0",+ "cua-bench[otel]",]# API server (for custom environments)diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.pyindex 70e93293b5..a6e66088f4 100644--- a/libs/python/agent/agent/agent.py+++ b/libs/python/agent/agent/agent.py@@ -6,6 +6,8 @@ import asyncioimport hashlibimport inspectimport json+import random+import timefrom pathlib import Pathfrom typing import (Any,@@ -31,7 +33,6 @@ from .adapters import (HuggingFaceLocalAdapter,HumanAdapter,MLXVLMAdapter,- YutoriAdapter,)from .callbacks import (BudgetManagerCallback,@@ -189,6 +190,70 @@ def hash_api_key(api_key: Optional[str]) -> Optional[str]:return hashlib.sha256(api_key.encode()).hexdigest()+def _is_retryable_error(exc: BaseException) -> bool:+ """Return True if the exception is a transient error that warrants a retry."""+ # asyncio / network timeouts+ if isinstance(exc, (asyncio.TimeoutError, TimeoutError)):+ return True+ # liteLLM error types+ try:+ import litellm.exceptions as _le++ retryable_types = (+ _le.RateLimitError,+ _le.ServiceUnavailableError,+ _le.APIConnectionError,+ _le.Timeout,+ _le.InternalServerError,+ )+ if isinstance(exc, retryable_types):+ return True+ except Exception:+ pass+ # Generic heuristic: 429 / 5xx in the message+ msg = str(exc).lower()+ if any(k in msg for k in ("timeout", "rate limit", "503", "502", "429", "connection")):+ return True+ return False+++async def _predict_step_with_retry(+ agent_loop,+ loop_kwargs: dict,+ hooks: dict,+ max_retries: int = 3,+ base_delay: float = 2.0,+) -> Any:+ """+ Call agent_loop.predict_step() with exponential backoff retries on transient errors.++ Args:+ agent_loop: The agent loop instance.+ loop_kwargs: Keyword arguments for predict_step.+ hooks: Dict of lifecycle hook callables (_on_api_start, etc.).+ max_retries: Maximum number of retry attempts (total attempts = max_retries + 1).+ base_delay: Base delay in seconds for the first retry; doubles each attempt.+ """+ if max_retries is None:+ max_retries = 0+ last_exc: Optional[BaseException] = None+ for attempt in range(max_retries + 1):+ try:+ return await agent_loop.predict_step(**loop_kwargs, **hooks)+ except Exception as exc:+ last_exc = exc+ if attempt < max_retries and _is_retryable_error(exc):+ delay = base_delay * (2**attempt) + random.uniform(0, 1)+ print(+ f"[cua-agent] Transient error on step (attempt {attempt + 1}/{max_retries + 1}): "+ f"{type(exc).__name__}: {exc}. Retrying in {delay:.1f}s …"+ )+ await asyncio.sleep(delay)+ else:+ raise+ raise last_exc # unreachable, but satisfies type checkers++class ComputerAgent:"""Main agent class that automatically selects the appropriate agent loop@@ -299,14 +364,12 @@ class ComputerAgent:mlx_adapter = MLXVLMAdapter()cua_adapter = CUAAdapter()azure_ml_adapter = AzureMLAdapter()- yutori_adapter = YutoriAdapter()litellm.custom_provider_map = [{"provider": "huggingface-local", "custom_handler": hf_adapter},{"provider": "human", "custom_handler": human_adapter},{"provider": "mlx", "custom_handler": mlx_adapter},{"provider": "cua", "custom_handler": cua_adapter},{"provider": "azure_ml", "custom_handler": azure_ml_adapter},- {"provider": "yutori", "custom_handler": yutori_adapter},]litellm.suppress_debug_info = True@@ -786,116 +849,6 @@ class ComputerAgent:if item_type == "function_call":await self._on_function_call_start(item)-- # Special handling for "computer" function calls (from GPT 5.4 etc)- # These need to be treated like computer_call items- if item.get("name") == "computer" and computer:- args = json.loads(item.get("arguments", "{}"))- action_type = args.get("action")- if not action_type:- raise ToolError("Computer function call missing 'action' argument")-- # Map action types to their relevant parameters- action_param_map = {- "screenshot": [], # No parameters- "click": ["x", "y", "button"],- "double_click": ["x", "y"],- "right_click": ["x", "y"],- "type": ["text"],- "keypress": ["keys"],- "scroll": ["x", "y", "scroll_x", "scroll_y"],- "move": ["x", "y"],- "drag": ["start_x", "start_y", "end_x", "end_y"],- "wait": ["seconds", "ms"],- "terminate": ["status"],- }-- # Extract only relevant action arguments, filtering out empty values- relevant_params = action_param_map.get(action_type, [])- action_args = {}- for k, v in args.items():- if k == "action":- continue- # Include if it's a relevant param OR if param map doesn't have this action- if k in relevant_params or action_type not in action_param_map:- # Filter out empty/default values but allow numeric zero (valid for coordinates/scroll)- if v is not None and v != "" and v != []:- action_args[k] = v-- # Execute the computer action- computer_method = getattr(computer, action_type, None)- action_result = None- if computer_method:- action_result = await computer_method(**action_args)- else:- raise ToolError(f"Unknown computer action: {action_type}")-- # Track computer action execution- if self.telemetry_enabled and is_telemetry_enabled():- record_event(- "computer_action_executed",- {"action_type": action_type},- )- record_event(- "agent_tool_executed",- {"tool_type": "computer", "tool_name": action_type},- )-- # Check if this was a terminate action- is_terminate = action_type == "terminate" or (- isinstance(action_result, dict) and action_result.get("terminated")- )-- # Take screenshot after action (skip for terminate)- if not is_terminate:- if self.screenshot_delay and self.screenshot_delay > 0:- await asyncio.sleep(self.screenshot_delay)- screenshot_base64 = await computer.screenshot()- await self._on_screenshot(screenshot_base64, "screenshot_after")-- # Create function call output for GPT 5.4 etc- # Note: function_call_output only supports text, so we include the- # screenshot as a separate user message with image content- if is_terminate:- output_content = json.dumps(- action_result if action_result else {"terminated": True}- )- call_output = {- "type": "function_call_output",- "call_id": item.get("call_id"),- "output": output_content,- }- result = [call_output]- else:- # Return both the function output AND a user message with the screenshot- # This allows the model to see the screenshot result- # Preserve actual action_result so failures/errors aren't masked- if action_result is not None:- output_content = json.dumps(action_result)- else:- output_content = json.dumps(- {"success": True, "screenshot_captured": True}- )- call_output = {- "type": "function_call_output",- "call_id": item.get("call_id"),- "output": output_content,- }- # Include screenshot as a user message with image content- image_message = {- "role": "user",- "content": [- {- "type": "input_image",- "image_url": f"data:image/png;base64,{screenshot_base64}",- }- ],- }- result = [call_output, image_message]-- await self._on_function_call_end(item, result)- return result-# Perform function callfunction = self._get_tool(item.get("name"))if not function:@@ -1018,7 +971,9 @@ class ComputerAgent:"tools": self.tool_schemas,"stream": False,"computer_handler": self.computer_handler,- "max_retries": self.max_retries,+ # Inner liteLLM retries are disabled here; _predict_step_with_retry+ # is the sole retry layer so the two don't stack.+ "max_retries": 0,"use_prompt_caching": self.use_prompt_caching,**merged_kwargs,}@@ -1053,13 +1008,17 @@ class ComputerAgent:)# ---------------------------------- # Run agent loop iteration- result = await self.agent_loop.predict_step(- **loop_kwargs,- _on_api_start=self._on_api_start,- _on_api_end=self._on_api_end,- _on_usage=self._on_usage,- _on_screenshot=self._on_screenshot,+ # Run agent loop iteration (with automatic retry on transient errors)+ result = await _predict_step_with_retry(+ self.agent_loop,+ loop_kwargs,+ hooks={+ "_on_api_start": self._on_api_start,+ "_on_api_end": self._on_api_end,+ "_on_usage": self._on_usage,+ "_on_screenshot": self._on_screenshot,+ },+ max_retries=self.max_retries,)result = get_json(result)diff --git a/libs/python/agent/agent/loops/anthropic.py b/libs/python/agent/agent/loops/anthropic.pyindex ec90895483..75ae2db9c3 100644--- a/libs/python/agent/agent/loops/anthropic.py+++ b/libs/python/agent/agent/loops/anthropic.py@@ -1800,6 +1800,9 @@ class AnthropicHostedToolsConfig(AsyncAgentConfig):"tools": anthropic_tools if anthropic_tools else None,"stream": stream,"num_retries": max_retries,+ # Bound each request so a stalled connection doesn't block forever.+ # Callers can override via kwargs (e.g. request_timeout=120).+ "request_timeout": kwargs.pop("request_timeout", 120),**kwargs,}@@ -1919,6 +1922,7 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""","stream": False,"max_tokens": 100, # Keep response short for click prediction"headers": {"anthropic-beta": tool_config["beta_flag"]},+ "request_timeout": kwargs.pop("request_timeout", 120),}# Thread optional API paramsif "api_key" in kwargs and kwargs.get("api_key") is not None:diff --git a/libs/python/agent/agent/loops/openai.py b/libs/python/agent/agent/loops/openai.pyindex 5648e67084..8e7a3fb20e 100644--- a/libs/python/agent/agent/loops/openai.py+++ b/libs/python/agent/agent/loops/openai.py@@ -233,6 +233,7 @@ class OpenAIComputerUseConfig:"reasoning": {"summary": "concise"},"truncation": "auto","num_retries": max_retries,+ "request_timeout": kwargs.pop("request_timeout", 120),**kwargs,}@@ -378,6 +379,7 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""","reasoning": {"summary": "concise"},"truncation": "auto","max_tokens": 200, # Keep response short for click prediction+ "request_timeout": kwargs.pop("request_timeout", 120),**kwargs,}