Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-bench): robust agent runs with retry/backoff and fixed cb run stop (#1203)

ddupont1fdc813d7ff1
diff --git a/libs/cua-bench/cua_bench/agents/base.py b/libs/cua-bench/cua_bench/agents/base.py
index ec3b35f39f..26cb772297 100644
--- a/libs/cua-bench/cua_bench/agents/base.py
+++ b/libs/cua-bench/cua_bench/agents/base.py
@@ -17,6 +17,7 @@ class FailureMode(Enum):
NONE = "none"
UNKNOWN = "unknown"
MAX_STEPS_EXCEEDED = "max_steps_exceeded"
+ API_ERROR = "api_error"
@dataclass
diff --git a/libs/cua-bench/cua_bench/agents/cua_agent.py b/libs/cua-bench/cua_bench/agents/cua_agent.py
index 12c04fe21c..e4a783a4a3 100644
--- a/libs/cua-bench/cua_bench/agents/cua_agent.py
+++ b/libs/cua-bench/cua_bench/agents/cua_agent.py
@@ -1,5 +1,6 @@
"""CUA Agent implementation using the Computer Agent SDK."""
+import asyncio
import base64
import sys
from pathlib import Path
@@ -8,6 +9,31 @@ from typing import TYPE_CHECKING, Any, Dict, List
from . import register_agent
from .base import AgentResult, BaseAgent, FailureMode
+
+def _is_retryable_api_error(exc: BaseException) -> bool:
+ """Return True if the exception is a transient error that can be retried."""
+ if isinstance(exc, (asyncio.TimeoutError, TimeoutError)):
+ return True
+ try:
+ import litellm.exceptions as _le
+
+ if isinstance(
+ exc,
+ (
+ _le.RateLimitError,
+ _le.ServiceUnavailableError,
+ _le.APIConnectionError,
+ _le.Timeout,
+ _le.InternalServerError,
+ ),
+ ):
+ return True
+ except Exception:
+ pass
+ msg = str(exc).lower()
+ return any(k in msg for k in ("timeout", "rate limit", "503", "502", "429", "connection"))
+
+
if TYPE_CHECKING:
from ..computers import DesktopSession
@@ -20,6 +46,11 @@ class CuaAgent(BaseAgent):
super().__init__(**kwargs)
self.model = kwargs.get("model", "anthropic/claude-sonnet-4-20250514")
self.max_steps = kwargs.get("max_steps", 100)
+ # Number of times to retry the entire task when a transient API error occurs.
+ # Task-level retry restarts agent.run() from scratch but does NOT reset the
+ # desktop environment, so it is unsafe by default for evaluations.
+ # Only enable via --agent-kwarg task_retries=N when the task is idempotent.
+ self.task_retries = int(kwargs.get("task_retries", 0))
@staticmethod
def name() -> str:
@@ -200,82 +231,108 @@ class CuaAgent(BaseAgent):
)
print("CUA Agent initialized with model:", self.model)
- # Run the agent and track usage
- try:
- total_usage = {
- "prompt_tokens": 0,
- "completion_tokens": 0,
- "total_tokens": 0,
- "response_cost": 0.0,
- }
-
- step = 0
- task_completed = False
-
- async for result in agent.run(instruction):
- sys.stdout.flush() # Flush output
-
- step += 1
- for k in total_usage:
- total_usage[k] += result["usage"].get(k, 0)
-
- # Record agent step to tracer
- if tracer:
- try:
- # Take screenshot
- screenshot = await session.screenshot()
- # Record the step with metadata
- tracer.record(
- "agent_step",
- {
- "step": step,
- "agent": self.name(),
- "model": self.model,
- "usage": result["usage"],
- "output": result["output"],
- },
- [screenshot],
- )
- except Exception as e:
- print(f"Warning: Failed to record agent step to tracer: {e}")
-
- # Check if we've reached max_steps
- if step >= self.max_steps:
- print(f"\n[Max steps reached] Stopped at step {step}/{self.max_steps}")
- break
-
- # Check if task is completed (agent returned done or similar)
-
- for item in result["output"]:
- if item["type"] == "message":
- if "DONE" in item["content"][0]["text"]:
- print(f"\n[Task completed] Agent indicated completion at step {step}")
- task_completed = True
- break
-
- print(f"\nTotal usage: {total_usage}")
- print(f"Steps completed: {step}/{self.max_steps}")
-
- # Determine failure mode
- if task_completed:
- failure_mode = FailureMode.NONE
- elif step >= self.max_steps:
- failure_mode = FailureMode.MAX_STEPS_EXCEEDED
- else:
- failure_mode = FailureMode.NONE # Completed within max_steps
-
- return AgentResult(
- total_input_tokens=total_usage.get("prompt_tokens", 0),
- total_output_tokens=total_usage.get("completion_tokens", 0),
- failure_mode=failure_mode,
- )
- except Exception as e:
- print(f"Agent execution failed: {e}")
- import traceback
-
- traceback.print_exc()
- return AgentResult(
- total_input_tokens=0,
- total_output_tokens=0,
- failure_mode=FailureMode.UNKNOWN,
- )
+ # Run the agent and track usage (with task-level retry on transient API errors)
+ total_usage = {
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "total_tokens": 0,
+ "response_cost": 0.0,
+ }
+ last_exc: BaseException | None = None
+ for attempt in range(self.task_retries + 1):
+ if attempt > 0:
+ delay = 5.0 * (2 ** (attempt - 1))
+ print(
+ f"[cua-agent] Retrying task (attempt {attempt + 1}/{self.task_retries + 1}) "
+ f"after transient error: {last_exc!r}. Waiting {delay:.0f}s …"
+ )
+ await asyncio.sleep(delay)
+
+ try:
+ step = 0
+ task_completed = False
+
+ async for result in agent.run(instruction):
+ sys.stdout.flush() # Flush output
+
+ step += 1
+ for k in total_usage:
+ total_usage[k] += result["usage"].get(k, 0)
+
+ # Record agent step to tracer
+ if tracer:
+ try:
+ # Take screenshot
+ screenshot = await session.screenshot()
+ # Record the step with metadata
+ usage = result["usage"]
+ if hasattr(usage, "model_dump"):
+ usage = usage.model_dump()
+ elif not isinstance(usage, dict):
+ usage = dict(usage)
+ tracer.record(
+ "agent_step",
+ {
+ "step": step,
+ "agent": self.name(),
+ "model": self.model,
+ "usage": usage,
+ "output": result["output"],
+ },
+ [screenshot],
+ )
+ except Exception as e:
+ print(f"Warning: Failed to record agent step to tracer: {e}")
+
+ # Check if we've reached max_steps
+ if step >= self.max_steps:
+ print(f"\n[Max steps reached] Stopped at step {step}/{self.max_steps}")
+ break
+
+ # Check if task is completed (agent returned done or similar)
+ for item in result["output"]:
+ if item["type"] == "message":
+ if "DONE" in item["content"][0]["text"]:
+ print(
+ f"\n[Task completed] Agent indicated completion at step {step}"
+ )
+ task_completed = True
+ break
+
+ print(f"\nTotal usage: {total_usage}")
+ print(f"Steps completed: {step}/{self.max_steps}")
+
+ # Determine failure mode
+ if task_completed:
+ failure_mode = FailureMode.NONE
+ elif step >= self.max_steps:
+ failure_mode = FailureMode.MAX_STEPS_EXCEEDED
+ else:
+ failure_mode = FailureMode.NONE # Completed within max_steps
+
+ return AgentResult(
+ total_input_tokens=total_usage.get("prompt_tokens", 0),
+ total_output_tokens=total_usage.get("completion_tokens", 0),
+ failure_mode=failure_mode,
+ )
+
+ except Exception as exc:
+ last_exc = exc
+ if attempt < self.task_retries and _is_retryable_api_error(exc):
+ # Will retry in the next loop iteration
+ continue
+
+ # Non-retryable or out of retries
+ import traceback
+
+ print(f"Agent execution failed: {exc}")
+ traceback.print_exc()
+
+ failure_mode = (
+ FailureMode.API_ERROR if _is_retryable_api_error(exc) else FailureMode.UNKNOWN
+ )
+ return AgentResult(
+ total_input_tokens=0,
+ total_output_tokens=0,
+ failure_mode=failure_mode,
+ )
diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py
index 9b04510087..573eee3970 100644
--- a/libs/cua-bench/cua_bench/cli/commands/run.py
+++ b/libs/cua-bench/cua_bench/cli/commands/run.py
@@ -586,6 +586,8 @@ def cmd_stop(args) -> int:
async def _cmd_stop_async(args) -> int:
"""Stop a run asynchronously."""
+ import signal
+
from cua_bench.sessions import list_sessions, manager
from cua_bench.sessions.providers.docker import DockerProvider
@@ -594,12 +596,33 @@ async def _cmd_stop_async(args) -> int:
print(f"{RED}Error: run_id required{RESET}")
return 1
+ # Kill the background orchestrator process first so it can't restart containers
+ pid_file = _get_run_output_dir(run_id) / "run.pid"
+ if pid_file.exists():
+ try:
+ pid = int(pid_file.read_text().strip())
+ os.kill(pid, signal.SIGTERM)
+ print(f"{CYAN}Sent SIGTERM to orchestrator process (pid {pid}){RESET}")
+ # Give it a moment to clean up, then SIGKILL if still alive
+ await asyncio.sleep(2)
+ try:
+ sigkill = getattr(signal, "SIGKILL", None)
+ if sigkill is not None:
+ os.kill(pid, sigkill)
+ except ProcessLookupError:
+ pass # Already exited cleanly
+ except (ValueError, ProcessLookupError):
+ pass # PID file stale or process already gone
+ finally:
+ pid_file.unlink(missing_ok=True)
+
sessions = list_sessions()
run_sessions = [s for s in sessions if s.get("run_id") == run_id]
if not run_sessions:
- print(f"{RED}Error: No sessions found for run: {run_id}{RESET}")
- return 1
+ print(f"{YELLOW}No active sessions found for run: {run_id}{RESET}")
+ print(f"\n{GREEN}✓ Run stopped{RESET}")
+ return 0
print(f"{CYAN}Stopping run: {run_id} ({len(run_sessions)} sessions)...{RESET}")
@@ -1762,12 +1785,18 @@ async def _cmd_run_dataset_async(args) -> int:
log_handle = open(global_log_file, "w", encoding="utf-8", buffering=1)
# Start detached subprocess
- subprocess.Popen(
+ proc = subprocess.Popen(
cmd,
stdout=log_handle,
stderr=subprocess.STDOUT,
env=env_vars,
)
+ # Close the parent's copy of the fd; the subprocess has its own inherited copy.
+ log_handle.close()
+
+ # Save PID so `cb run stop` can kill the process
+ pid_file = output_dir / "run.pid"
+ pid_file.write_text(str(proc.pid))
return 0
diff --git a/libs/cua-bench/pyproject.toml b/libs/cua-bench/pyproject.toml
index 19d0e29c23..399e71bb9b 100644
--- a/libs/cua-bench/pyproject.toml
+++ b/libs/cua-bench/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "cua-bench"
-version = "0.2.6"
+version = "0.2.5"
description = "Toolkit for computer-use RL environments and benchmarks"
readme = "README.md"
license = "MIT"
@@ -59,7 +59,7 @@ dependencies = [
[project.optional-dependencies]
# Full installation with all features
all = [
- "cua-bench[browser,agents,cloud]",
+ "cua-bench[browser,agents,cloud,otel]",
]
# Browser automation (for simulated environments)
@@ -73,6 +73,13 @@ agents = [
"anthropic>=0.26.0",
]
+# OpenTelemetry tracing and metrics export
+otel = [
+ "opentelemetry-api>=1.20.0",
+ "opentelemetry-sdk>=1.20.0",
+ "opentelemetry-exporter-otlp-proto-http>=1.20.0",
+]
+
# Cloud batch execution (Google Cloud)
cloud = [
"google-cloud-batch>=0.17.0",
@@ -80,6 +87,7 @@ cloud = [
"google-cloud-storage>=2.10.0",
"gcloud-aio-storage>=9.0.0",
"grpcio>=1.60.0",
+ "cua-bench[otel]",
]
# API server (for custom environments)
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.py
index 70e93293b5..a6e66088f4 100644
--- a/libs/python/agent/agent/agent.py
+++ b/libs/python/agent/agent/agent.py
@@ -6,6 +6,8 @@ import asyncio
import hashlib
import inspect
import json
+import random
+import time
from pathlib import Path
from typing import (
Any,
@@ -31,7 +33,6 @@ from .adapters import (
HuggingFaceLocalAdapter,
HumanAdapter,
MLXVLMAdapter,
- YutoriAdapter,
)
from .callbacks import (
BudgetManagerCallback,
@@ -189,6 +190,70 @@ def hash_api_key(api_key: Optional[str]) -> Optional[str]:
return hashlib.sha256(api_key.encode()).hexdigest()
+def _is_retryable_error(exc: BaseException) -> bool:
+ """Return True if the exception is a transient error that warrants a retry."""
+ # asyncio / network timeouts
+ if isinstance(exc, (asyncio.TimeoutError, TimeoutError)):
+ return True
+ # liteLLM error types
+ try:
+ import litellm.exceptions as _le
+
+ retryable_types = (
+ _le.RateLimitError,
+ _le.ServiceUnavailableError,
+ _le.APIConnectionError,
+ _le.Timeout,
+ _le.InternalServerError,
+ )
+ if isinstance(exc, retryable_types):
+ return True
+ except Exception:
+ pass
+ # Generic heuristic: 429 / 5xx in the message
+ msg = str(exc).lower()
+ if any(k in msg for k in ("timeout", "rate limit", "503", "502", "429", "connection")):
+ return True
+ return False
+
+
+async def _predict_step_with_retry(
+ agent_loop,
+ loop_kwargs: dict,
+ hooks: dict,
+ max_retries: int = 3,
+ base_delay: float = 2.0,
+) -> Any:
+ """
+ Call agent_loop.predict_step() with exponential backoff retries on transient errors.
+
+ Args:
+ agent_loop: The agent loop instance.
+ loop_kwargs: Keyword arguments for predict_step.
+ hooks: Dict of lifecycle hook callables (_on_api_start, etc.).
+ max_retries: Maximum number of retry attempts (total attempts = max_retries + 1).
+ base_delay: Base delay in seconds for the first retry; doubles each attempt.
+ """
+ if max_retries is None:
+ max_retries = 0
+ last_exc: Optional[BaseException] = None
+ for attempt in range(max_retries + 1):
+ try:
+ return await agent_loop.predict_step(**loop_kwargs, **hooks)
+ except Exception as exc:
+ last_exc = exc
+ if attempt < max_retries and _is_retryable_error(exc):
+ delay = base_delay * (2**attempt) + random.uniform(0, 1)
+ print(
+ f"[cua-agent] Transient error on step (attempt {attempt + 1}/{max_retries + 1}): "
+ f"{type(exc).__name__}: {exc}. Retrying in {delay:.1f}s …"
+ )
+ await asyncio.sleep(delay)
+ else:
+ raise
+ raise last_exc # unreachable, but satisfies type checkers
+
+
class ComputerAgent:
"""
Main agent class that automatically selects the appropriate agent loop
@@ -299,14 +364,12 @@ class ComputerAgent:
mlx_adapter = MLXVLMAdapter()
cua_adapter = CUAAdapter()
azure_ml_adapter = AzureMLAdapter()
- yutori_adapter = YutoriAdapter()
litellm.custom_provider_map = [
{"provider": "huggingface-local", "custom_handler": hf_adapter},
{"provider": "human", "custom_handler": human_adapter},
{"provider": "mlx", "custom_handler": mlx_adapter},
{"provider": "cua", "custom_handler": cua_adapter},
{"provider": "azure_ml", "custom_handler": azure_ml_adapter},
- {"provider": "yutori", "custom_handler": yutori_adapter},
]
litellm.suppress_debug_info = True
@@ -786,116 +849,6 @@ class ComputerAgent:
if item_type == "function_call":
await self._on_function_call_start(item)
-
- # Special handling for "computer" function calls (from GPT 5.4 etc)
- # These need to be treated like computer_call items
- if item.get("name") == "computer" and computer:
- args = json.loads(item.get("arguments", "{}"))
- action_type = args.get("action")
- if not action_type:
- raise ToolError("Computer function call missing 'action' argument")
-
- # Map action types to their relevant parameters
- action_param_map = {
- "screenshot": [], # No parameters
- "click": ["x", "y", "button"],
- "double_click": ["x", "y"],
- "right_click": ["x", "y"],
- "type": ["text"],
- "keypress": ["keys"],
- "scroll": ["x", "y", "scroll_x", "scroll_y"],
- "move": ["x", "y"],
- "drag": ["start_x", "start_y", "end_x", "end_y"],
- "wait": ["seconds", "ms"],
- "terminate": ["status"],
- }
-
- # Extract only relevant action arguments, filtering out empty values
- relevant_params = action_param_map.get(action_type, [])
- action_args = {}
- for k, v in args.items():
- if k == "action":
- continue
- # Include if it's a relevant param OR if param map doesn't have this action
- if k in relevant_params or action_type not in action_param_map:
- # Filter out empty/default values but allow numeric zero (valid for coordinates/scroll)
- if v is not None and v != "" and v != []:
- action_args[k] = v
-
- # Execute the computer action
- computer_method = getattr(computer, action_type, None)
- action_result = None
- if computer_method:
- action_result = await computer_method(**action_args)
- else:
- raise ToolError(f"Unknown computer action: {action_type}")
-
- # Track computer action execution
- if self.telemetry_enabled and is_telemetry_enabled():
- record_event(
- "computer_action_executed",
- {"action_type": action_type},
- )
- record_event(
- "agent_tool_executed",
- {"tool_type": "computer", "tool_name": action_type},
- )
-
- # Check if this was a terminate action
- is_terminate = action_type == "terminate" or (
- isinstance(action_result, dict) and action_result.get("terminated")
- )
-
- # Take screenshot after action (skip for terminate)
- if not is_terminate:
- if self.screenshot_delay and self.screenshot_delay > 0:
- await asyncio.sleep(self.screenshot_delay)
- screenshot_base64 = await computer.screenshot()
- await self._on_screenshot(screenshot_base64, "screenshot_after")
-
- # Create function call output for GPT 5.4 etc
- # Note: function_call_output only supports text, so we include the
- # screenshot as a separate user message with image content
- if is_terminate:
- output_content = json.dumps(
- action_result if action_result else {"terminated": True}
- )
- call_output = {
- "type": "function_call_output",
- "call_id": item.get("call_id"),
- "output": output_content,
- }
- result = [call_output]
- else:
- # Return both the function output AND a user message with the screenshot
- # This allows the model to see the screenshot result
- # Preserve actual action_result so failures/errors aren't masked
- if action_result is not None:
- output_content = json.dumps(action_result)
- else:
- output_content = json.dumps(
- {"success": True, "screenshot_captured": True}
- )
- call_output = {
- "type": "function_call_output",
- "call_id": item.get("call_id"),
- "output": output_content,
- }
- # Include screenshot as a user message with image content
- image_message = {
- "role": "user",
- "content": [
- {
- "type": "input_image",
- "image_url": f"data:image/png;base64,{screenshot_base64}",
- }
- ],
- }
- result = [call_output, image_message]
-
- await self._on_function_call_end(item, result)
- return result
-
# Perform function call
function = self._get_tool(item.get("name"))
if not function:
@@ -1018,7 +971,9 @@ class ComputerAgent:
"tools": self.tool_schemas,
"stream": False,
"computer_handler": self.computer_handler,
- "max_retries": self.max_retries,
+ # Inner liteLLM retries are disabled here; _predict_step_with_retry
+ # is the sole retry layer so the two don't stack.
+ "max_retries": 0,
"use_prompt_caching": self.use_prompt_caching,
**merged_kwargs,
}
@@ -1053,13 +1008,17 @@ class ComputerAgent:
)
# ---------------------------------
- # Run agent loop iteration
- result = await self.agent_loop.predict_step(
- **loop_kwargs,
- _on_api_start=self._on_api_start,
- _on_api_end=self._on_api_end,
- _on_usage=self._on_usage,
- _on_screenshot=self._on_screenshot,
+ # Run agent loop iteration (with automatic retry on transient errors)
+ result = await _predict_step_with_retry(
+ self.agent_loop,
+ loop_kwargs,
+ hooks={
+ "_on_api_start": self._on_api_start,
+ "_on_api_end": self._on_api_end,
+ "_on_usage": self._on_usage,
+ "_on_screenshot": self._on_screenshot,
+ },
+ max_retries=self.max_retries,
)
result = get_json(result)
diff --git a/libs/python/agent/agent/loops/anthropic.py b/libs/python/agent/agent/loops/anthropic.py
index ec90895483..75ae2db9c3 100644
--- a/libs/python/agent/agent/loops/anthropic.py
+++ b/libs/python/agent/agent/loops/anthropic.py
@@ -1800,6 +1800,9 @@ class AnthropicHostedToolsConfig(AsyncAgentConfig):
"tools": anthropic_tools if anthropic_tools else None,
"stream": stream,
"num_retries": max_retries,
+ # Bound each request so a stalled connection doesn't block forever.
+ # Callers can override via kwargs (e.g. request_timeout=120).
+ "request_timeout": kwargs.pop("request_timeout", 120),
**kwargs,
}
@@ -1919,6 +1922,7 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",
"stream": False,
"max_tokens": 100, # Keep response short for click prediction
"headers": {"anthropic-beta": tool_config["beta_flag"]},
+ "request_timeout": kwargs.pop("request_timeout", 120),
}
# Thread optional API params
if "api_key" in kwargs and kwargs.get("api_key") is not None:
diff --git a/libs/python/agent/agent/loops/openai.py b/libs/python/agent/agent/loops/openai.py
index 5648e67084..8e7a3fb20e 100644
--- a/libs/python/agent/agent/loops/openai.py
+++ b/libs/python/agent/agent/loops/openai.py
@@ -233,6 +233,7 @@ class OpenAIComputerUseConfig:
"reasoning": {"summary": "concise"},
"truncation": "auto",
"num_retries": max_retries,
+ "request_timeout": kwargs.pop("request_timeout", 120),
**kwargs,
}
@@ -378,6 +379,7 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",
"reasoning": {"summary": "concise"},
"truncation": "auto",
"max_tokens": 200, # Keep response short for click prediction
+ "request_timeout": kwargs.pop("request_timeout", 120),
**kwargs,
}