Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(agent): Centralized tool resolution in ComputerAgent (#920)

Sarina Licee209d338fb
diff --git a/docs/content/docs/cua/guide/advanced/custom-tools.mdx b/docs/content/docs/cua/guide/advanced/custom-tools.mdx
index 33013e610f..e25806d3bb 100644
--- a/docs/content/docs/cua/guide/advanced/custom-tools.mdx
+++ b/docs/content/docs/cua/guide/advanced/custom-tools.mdx
@@ -3,8 +3,45 @@ title: Custom Tools
description: Extend agent capabilities with your own tools
---
+import { Callout } from 'fumadocs-ui/components/callout';
+
**Custom tools** let you extend what your agent can do beyond the built-in Computer and Browser tools. Define functions that the agent can call to perform specific tasks like calculations, API calls, file operations, or any custom logic.
+## Built-in Tools
+
+Before creating custom tools, understand the built-in tool types:
+
+| Tool | Actions | Use Case |
+| ------------- | ---------------------------------- | -------------------------- |
+| `Computer` | click, type, scroll, screenshot | Generic desktop automation |
+| `BrowserTool` | visit_url, web_search, click, type | Browser-focused tasks |
+
+### Model Tool Requirements
+
+Some models are trained for specific toolsets. For example, FARA requires browser tools:
+
+```python
+# When you pass Computer to FARA, it auto-wraps to BrowserTool
+agent = ComputerAgent(model="cua/microsoft/fara-7b", tools=[computer])
+# Warning: "Model requires browser tools. Auto-wrapping Computer to BrowserTool."
+```
+
+To silence the warning, pass the correct tool explicitly:
+
+```python
+from agent.tools.browser_tool import BrowserTool
+
+browser_tool = BrowserTool(interface=computer.interface)
+agent = ComputerAgent(model="cua/microsoft/fara-7b", tools=[browser_tool])
+# No warning
+```
+
+General models like Claude work with any tools you provide - no auto-wrapping occurs.
+
+<Callout type="info">
+ The Computer is always your base environment. Tools are built on top of it for specific use cases.
+</Callout>
+
## Simple Function Tools
The easiest way to add a tool is to pass a function with a docstring:
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.py
index 2bbd8c9245..4c40f6a30a 100644
--- a/libs/python/agent/agent/agent.py
+++ b/libs/python/agent/agent/agent.py
@@ -309,6 +309,9 @@ class ComputerAgent:
self.agent_loop = config_info.agent_class()
self.agent_config_info = config_info
+ # Note: Tool resolution is deferred to _initialize_computers() because
+ # Computer.interface may not be available until the computer is started
+
# Add telemetry callbacks AFTER agent_loop is set so they can capture the correct agent_type
if self.telemetry_enabled:
# PostHog telemetry (product analytics)
@@ -372,9 +375,93 @@ class ComputerAgent:
record_event("agent_init", event_data)
+ async def _resolve_tools(self, tools: List[Any], required_type: Optional[str]) -> List[Any]:
+ """
+ Resolve tools based on model's required tool_type.
+
+ - If model requires specific type (e.g., "browser"), auto-wrap Computer and warn
+ - If model is flexible (no tool_type), pass through unchanged
+
+ Args:
+ tools: List of tools passed to the agent
+ required_type: The tool type required by the model ("browser", "mobile", or None)
+
+ Returns:
+ List of resolved tools, potentially with Computer wrapped to BrowserTool
+ """
+ import logging
+ import warnings
+
+ from .tools.browser_tool import BrowserTool
+
+ logger = logging.getLogger(__name__)
+
+ if not required_type:
+ return tools # Flexible model, no wrapping
+
+ resolved = []
+ for tool in tools:
+ if required_type == "browser":
+ if isinstance(tool, BrowserTool):
+ # Already correct tool type, no warning needed
+ resolved.append(tool)
+ elif is_agent_computer(tool):
+ # Need to wrap Computer to BrowserTool
+ # Get the interface from the computer object
+ # Use try/except because Computer.interface raises if not initialized
+ interface = None
+ try:
+ interface = tool.interface
+ except (RuntimeError, AttributeError):
+ # Computer not initialized - initialize it now
+ logger.info(
+ "Computer not initialized, initializing for BrowserTool wrapping..."
+ )
+ if hasattr(tool, "__aenter__"):
+ await tool.__aenter__()
+ try:
+ interface = tool.interface
+ except (RuntimeError, AttributeError):
+ pass
+
+ if interface is None:
+ # Try cua_computer for cuaComputerHandler
+ if hasattr(tool, "cua_computer"):
+ interface = tool
+ else:
+ # Fallback: use the tool itself as interface
+ interface = tool
+
+ warnings.warn(
+ "Model requires browser tools. "
+ "Auto-wrapping Computer to BrowserTool. "
+ "Pass BrowserTool explicitly to silence this warning.",
+ UserWarning,
+ stacklevel=3,
+ )
+ logger.info(
+ "Auto-wrapping Computer to BrowserTool for model requiring browser tools"
+ )
+ resolved.append(BrowserTool(interface=interface))
+ else:
+ # Custom tool, pass through unchanged
+ resolved.append(tool)
+ # Future: elif required_type == "mobile": ...
+ else:
+ # Unknown tool type, pass through
+ resolved.append(tool)
+
+ return resolved
+
async def _initialize_computers(self):
- """Initialize computer objects"""
+ """Initialize computer objects and resolve tools based on model requirements."""
if not self.tool_schemas:
+ # Resolve tools based on model's required tool_type
+ # This is done here (not in __init__) because Computer.interface
+ # may not be available until the computer is started
+ tool_type = self.agent_config_info.tool_type if self.agent_config_info else None
+ self.tools = await self._resolve_tools(self.tools, tool_type)
+
# Process tools and create tool schemas
self.tool_schemas = self._process_tools()
diff --git a/libs/python/agent/agent/decorators.py b/libs/python/agent/agent/decorators.py
index a4e5cd936e..4c588a2660 100644
--- a/libs/python/agent/agent/decorators.py
+++ b/libs/python/agent/agent/decorators.py
@@ -10,13 +10,17 @@ from .types import AgentConfigInfo
_agent_configs: List[AgentConfigInfo] = []
-def register_agent(models: str, priority: int = 0):
+def register_agent(models: str, priority: int = 0, tool_type: Optional[str] = None):
"""
Decorator to register an AsyncAgentConfig class.
Args:
models: Regex pattern to match supported models
priority: Priority for agent selection (higher = more priority)
+ tool_type: Required tool type for this model ("browser" | "mobile" | None).
+ Specialized models (like FARA) declare their required tool type,
+ and ComputerAgent will auto-wrap tools accordingly.
+ General models (like Claude) leave this as None for full flexibility.
"""
def decorator(agent_class: type):
@@ -36,7 +40,10 @@ def register_agent(models: str, priority: int = 0):
# Register the agent config
config_info = AgentConfigInfo(
- agent_class=agent_class, models_regex=models, priority=priority
+ agent_class=agent_class,
+ models_regex=models,
+ priority=priority,
+ tool_type=tool_type,
)
_agent_configs.append(config_info)
diff --git a/libs/python/agent/agent/loops/fara/config.py b/libs/python/agent/agent/loops/fara/config.py
index 5f95127769..df36bc6182 100644
--- a/libs/python/agent/agent/loops/fara/config.py
+++ b/libs/python/agent/agent/loops/fara/config.py
@@ -28,7 +28,7 @@ from .helpers import (
)
-@register_agent(models=r"(?i).*fara-7b.*")
+@register_agent(models=r"(?i).*fara-7b.*", tool_type="browser")
class FaraVlmConfig(AsyncAgentConfig):
async def predict_step(
self,
diff --git a/libs/python/agent/agent/proxy/handlers.py b/libs/python/agent/agent/proxy/handlers.py
index 5686c9a082..d7fd4c1779 100644
--- a/libs/python/agent/agent/proxy/handlers.py
+++ b/libs/python/agent/agent/proxy/handlers.py
@@ -5,20 +5,15 @@ Request handlers for the proxy endpoints.
import json
import logging
import os
-import re
from contextlib import contextmanager
from typing import Any, Dict, List, Optional, Union
from computer import Computer
from ..agent import ComputerAgent
-from ..tools.browser_tool import BrowserTool
logger = logging.getLogger(__name__)
-# Pattern to detect FARA models (case-insensitive)
-FARA_MODEL_PATTERN = re.compile(r"(?i).*fara.*")
-
class ResponsesHandler:
"""Handler for /responses endpoint that processes agent requests."""
@@ -95,15 +90,9 @@ class ResponsesHandler:
agent_key_payload = {"model": model, **agent_kwargs_for_key}
agent_key = _stable_key(agent_key_payload)
- # Determine the appropriate tool based on model type
- # FARA models require BrowserTool instead of Computer for browser-specific actions
- # (visit_url, web_search, terminate, history_back, etc.)
- is_fara_model = bool(FARA_MODEL_PATTERN.match(model))
- if is_fara_model and computer is not None:
- tool = BrowserTool(interface=computer.interface)
- logger.info(f"Using BrowserTool for FARA model: {model}")
- else:
- tool = computer
+ # Pass Computer as the tool - ComputerAgent handles wrapping internally
+ # based on the model's required tool_type (e.g., FARA auto-wraps to BrowserTool)
+ tool = computer
agent = self._agent_cache.get(agent_key)
if agent is None:
diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.py
index 2e83266c9e..ccfc5c00f5 100644
--- a/libs/python/agent/agent/tools/browser_tool.py
+++ b/libs/python/agent/agent/tools/browser_tool.py
@@ -33,6 +33,7 @@ class BrowserTool(BaseComputerTool):
"""
self.interface = interface
self._facts = [] # Store memorized facts
+ self._automation = None # Cached automation interface
# Get initial screenshot to determine dimensions
self.viewport_width = None
@@ -56,6 +57,27 @@ class BrowserTool(BaseComputerTool):
super().__init__(cfg)
+ @property
+ def automation(self):
+ """
+ Get the automation interface for keyboard/mouse actions.
+
+ Handles both interface structures:
+ - Nested: interface.interface (wrapper with .interface property)
+ - Direct: interface itself IS the automation handler
+ """
+ if self._automation is not None:
+ return self._automation
+
+ # Try nested structure first (interface.interface)
+ if hasattr(self.interface, "interface") and self.interface.interface is not None:
+ self._automation = self.interface.interface
+ else:
+ # Direct structure - interface IS the automation handler
+ self._automation = self.interface
+
+ return self._automation
+
async def _initialize_dimensions(self):
"""Initialize viewport and resized dimensions from screenshot."""
try:
@@ -274,7 +296,7 @@ class BrowserTool(BaseComputerTool):
# Convert keys to proper format and press via hotkey
try:
- await self.interface.interface.hotkey(*keys)
+ await self.automation.hotkey(*keys)
return {"success": True, "message": f"Pressed keys: {keys}"}
except Exception as e:
return {"success": False, "error": str(e)}
@@ -299,7 +321,7 @@ class BrowserTool(BaseComputerTool):
if not coordinate or len(coordinate) != 2:
return {"success": False, "error": "coordinate parameter [x, y] is required"}
- await self.interface.interface.move_cursor(coordinate[0], coordinate[1])
+ await self.automation.move_cursor(coordinate[0], coordinate[1])
return {"success": True, "message": f"Moved cursor to {coordinate}"}
async def _action_left_click(self, params: dict) -> dict:
@@ -345,7 +367,7 @@ class BrowserTool(BaseComputerTool):
"""Go back in browser history."""
# Press Alt+Left arrow key combination
try:
- await self.interface.interface.hotkey("Alt", "ArrowLeft")
+ await self.automation.hotkey("Alt", "ArrowLeft")
return {"success": True, "message": "Navigated back in history"}
except Exception as e:
return {"success": False, "error": str(e)}
@@ -475,17 +497,17 @@ class BrowserTool(BaseComputerTool):
"""Drag from start to end coordinates. FARA-compatible."""
if start_coordinate and end_coordinate:
# Use start/end coordinates if provided
- await self.interface.interface.move_cursor(start_coordinate[0], start_coordinate[1])
- await self.interface.interface.mouse_down(start_coordinate[0], start_coordinate[1])
- await self.interface.interface.move_cursor(end_coordinate[0], end_coordinate[1])
- await self.interface.interface.mouse_up(end_coordinate[0], end_coordinate[1])
+ await self.automation.move_cursor(start_coordinate[0], start_coordinate[1])
+ await self.automation.mouse_down(start_coordinate[0], start_coordinate[1])
+ await self.automation.move_cursor(end_coordinate[0], end_coordinate[1])
+ await self.automation.mouse_up(end_coordinate[0], end_coordinate[1])
return {
"success": True,
"message": f"Dragged from {start_coordinate} to {end_coordinate}",
}
elif coordinate:
# Just move to coordinate
- await self.interface.interface.move_cursor(coordinate[0], coordinate[1])
+ await self.automation.move_cursor(coordinate[0], coordinate[1])
return {"success": True, "message": f"Moved to {coordinate}"}
return {
"success": False,
diff --git a/libs/python/agent/agent/types.py b/libs/python/agent/agent/types.py
index c28fb6b89e..1a43e2ceaa 100644
--- a/libs/python/agent/agent/types.py
+++ b/libs/python/agent/agent/types.py
@@ -38,6 +38,7 @@ class AgentConfigInfo(BaseModel):
agent_class: type
models_regex: str
priority: int = 0
+ tool_type: Optional[str] = None # "browser" | "mobile" | None (flexible)
def matches_model(self, model: str) -> bool:
"""Check if this agent config matches the given model"""
diff --git a/libs/python/agent/tests/test_tool_resolution.py b/libs/python/agent/tests/test_tool_resolution.py
new file mode 100644
index 0000000000..02bb037cd7
--- /dev/null
+++ b/libs/python/agent/tests/test_tool_resolution.py
@@ -0,0 +1,257 @@
+"""
+Tests for centralized tool resolution in ComputerAgent.
+
+Tests that:
+1. FARA (specialized model) auto-wraps Computer to BrowserTool with warning
+2. FARA accepts explicit BrowserTool without warning
+3. Claude (general model) accepts any tool without wrapping
+4. Custom function tools pass through unchanged
+"""
+
+import warnings
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+from agent.computers import is_agent_computer
+from agent.tools.browser_tool import BrowserTool
+from agent.types import AgentConfigInfo
+
+
+# Mock agent config class for testing
+class MockAgentConfig:
+ """Mock agent config class for testing"""
+
+ async def predict_step(self, **kwargs):
+ return {"output": [], "usage": {}}
+
+ async def predict_click(self, **kwargs):
+ return None
+
+ def get_capabilities(self):
+ return ["step"]
+
+
+class TestToolResolution:
+ """Tests for ComputerAgent._resolve_tools()"""
+
+ @pytest.fixture
+ def mock_computer(self):
+ """Create a mock Computer object"""
+ computer = Mock()
+ computer.interface = Mock()
+ computer.interface.interface = Mock() # For hotkey, etc.
+ computer.interface.playwright_exec = Mock()
+ return computer
+
+ @pytest.fixture
+ def mock_browser_tool(self):
+ """Create a mock BrowserTool"""
+ tool = Mock(spec=BrowserTool)
+ return tool
+
+ def test_fara_auto_wraps_computer_to_browser_tool(self, mock_computer):
+ """FARA auto-wraps Computer to BrowserTool with warning."""
+ from agent.agent import ComputerAgent
+
+ # Patch find_agent_config to return FARA config
+ fara_config = AgentConfigInfo(
+ agent_class=MockAgentConfig,
+ models_regex=r"(?i).*fara.*",
+ priority=0,
+ tool_type="browser",
+ )
+
+ # Patch is_agent_computer to recognize our mock as a computer
+ def mock_is_agent_computer(tool):
+ return tool is mock_computer
+
+ with patch("agent.agent.find_agent_config", return_value=fara_config):
+ with patch("agent.agent.is_agent_computer", side_effect=mock_is_agent_computer):
+ with pytest.warns(UserWarning, match="Auto-wrapping Computer to BrowserTool"):
+ agent = ComputerAgent(model="cua/microsoft/fara-7b", tools=[mock_computer])
+
+ # Should have wrapped to BrowserTool
+ assert len(agent.tools) == 1
+ assert isinstance(agent.tools[0], BrowserTool)
+
+ def test_fara_accepts_explicit_browser_tool_no_warning(self, mock_browser_tool):
+ """FARA accepts explicit BrowserTool without warning."""
+ from agent.agent import ComputerAgent
+
+ # Patch find_agent_config to return FARA config
+ fara_config = AgentConfigInfo(
+ agent_class=MockAgentConfig,
+ models_regex=r"(?i).*fara.*",
+ priority=0,
+ tool_type="browser",
+ )
+
+ with patch("agent.agent.find_agent_config", return_value=fara_config):
+ # Should not raise any warnings
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ agent = ComputerAgent(model="cua/microsoft/fara-7b", tools=[mock_browser_tool])
+
+ # Should keep the original BrowserTool
+ assert len(agent.tools) == 1
+ assert agent.tools[0] is mock_browser_tool
+
+ def test_claude_accepts_any_tool_no_wrapping(self, mock_computer):
+ """Claude (general model) accepts any tool without wrapping."""
+ from agent.agent import ComputerAgent
+
+ # Patch find_agent_config to return Claude config (no tool_type)
+ claude_config = AgentConfigInfo(
+ agent_class=MockAgentConfig,
+ models_regex=r".*claude.*",
+ priority=0,
+ tool_type=None, # General model, no tool type requirement
+ )
+
+ with patch("agent.agent.find_agent_config", return_value=claude_config):
+ with warnings.catch_warnings():
+ warnings.simplefilter("error") # Fail if any warning
+ agent = ComputerAgent(model="claude-sonnet-4", tools=[mock_computer])
+
+ # Should keep original Computer, not wrapped
+ assert len(agent.tools) == 1
+ assert agent.tools[0] is mock_computer
+
+ def test_custom_tools_pass_through(self, mock_computer):
+ """Custom function tools pass through unchanged."""
+ from agent.agent import ComputerAgent
+
+ def custom_tool():
+ """A custom tool function."""
+ pass
+
+ # Patch find_agent_config to return FARA config
+ fara_config = AgentConfigInfo(
+ agent_class=MockAgentConfig,
+ models_regex=r"(?i).*fara.*",
+ priority=0,
+ tool_type="browser",
+ )
+
+ # Patch is_agent_computer to recognize our mock as a computer
+ def mock_is_agent_computer(tool):
+ return tool is mock_computer
+
+ with patch("agent.agent.find_agent_config", return_value=fara_config):
+ with patch("agent.agent.is_agent_computer", side_effect=mock_is_agent_computer):
+ # Suppress the warning for computer wrapping
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ agent = ComputerAgent(
+ model="cua/microsoft/fara-7b", tools=[mock_computer, custom_tool]
+ )
+
+ # Should have wrapped computer but kept custom tool unchanged
+ assert len(agent.tools) == 2
+ assert isinstance(agent.tools[0], BrowserTool)
+ assert agent.tools[1] is custom_tool
+
+
+class TestAgentConfigInfo:
+ """Tests for AgentConfigInfo with tool_type field"""
+
+ def test_agent_config_info_with_tool_type(self):
+ """AgentConfigInfo accepts tool_type parameter"""
+ config = AgentConfigInfo(
+ agent_class=MockAgentConfig,
+ models_regex=r".*fara.*",
+ priority=0,
+ tool_type="browser",
+ )
+ assert config.tool_type == "browser"
+
+ def test_agent_config_info_without_tool_type(self):
+ """AgentConfigInfo defaults tool_type to None"""
+ config = AgentConfigInfo(
+ agent_class=MockAgentConfig,
+ models_regex=r".*claude.*",
+ priority=0,
+ )
+ assert config.tool_type is None
+
+ def test_agent_config_info_matches_model(self):
+ """AgentConfigInfo.matches_model works correctly"""
+ config = AgentConfigInfo(
+ agent_class=MockAgentConfig,
+ models_regex=r"(?i).*fara-7b.*",
+ priority=0,
+ tool_type="browser",
+ )
+ assert config.matches_model("cua/microsoft/fara-7b")
+ assert config.matches_model("FARA-7B")
+ assert not config.matches_model("claude-sonnet-4")
+
+
+class TestRegisterAgentDecorator:
+ """Tests for @register_agent decorator with tool_type"""
+
+ def test_register_agent_with_tool_type(self):
+ """@register_agent accepts tool_type parameter"""
+ from agent.decorators import _agent_configs, register_agent
+
+ # Clear registry for test
+ original_configs = _agent_configs.copy()
+ _agent_configs.clear()
+
+ try:
+
+ @register_agent(models=r"test-model.*", tool_type="browser")
+ class TestAgentConfig:
+ async def predict_step(self, **kwargs):
+ pass
+
+ async def predict_click(self, **kwargs):
+ pass
+
+ def get_capabilities(self):
+ return ["step"]
+
+ # Find the registered config
+ from agent.decorators import find_agent_config
+
+ config = find_agent_config("test-model-123")
+ assert config is not None
+ assert config.tool_type == "browser"
+
+ finally:
+ # Restore original registry
+ _agent_configs.clear()
+ _agent_configs.extend(original_configs)
+
+ def test_register_agent_without_tool_type(self):
+ """@register_agent without tool_type defaults to None"""
+ from agent.decorators import _agent_configs, register_agent
+
+ # Clear registry for test
+ original_configs = _agent_configs.copy()
+ _agent_configs.clear()
+
+ try:
+
+ @register_agent(models=r"general-model.*")
+ class GeneralAgentConfig:
+ async def predict_step(self, **kwargs):
+ pass
+
+ async def predict_click(self, **kwargs):
+ pass
+
+ def get_capabilities(self):
+ return ["step"]
+
+ # Find the registered config
+ from agent.decorators import find_agent_config
+
+ config = find_agent_config("general-model-123")
+ assert config is not None
+ assert config.tool_type is None
+
+ finally:
+ # Restore original registry
+ _agent_configs.clear()
+ _agent_configs.extend(original_configs)
diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.py
index 324e75d194..887fa8e035 100644
--- a/libs/python/computer-server/computer_server/main.py
+++ b/libs/python/computer-server/computer_server/main.py
@@ -724,12 +724,46 @@ async def agent_response_endpoint(
# and delegates to our existing automation/file/accessibility handlers.
from agent.computers import AsyncComputerHandler # runtime-checkable Protocol
+ class DirectComputerInterface:
+ """Interface wrapper providing BrowserTool compatibility.
+
+ Matches the same interface shape as Computer.interface so BrowserTool
+ works identically with both Computer (cloud) and DirectComputer (local).
+ """
+
+ def __init__(self, automation_handler, browser_manager):
+ self._auto = automation_handler
+ self._browser = browser_manager
+
+ @property
+ def interface(self):
+ """Return automation handler for hotkey, move_cursor, etc."""
+ return self._auto
+
+ async def playwright_exec(self, command: str, params: dict) -> dict:
+ """Execute browser command via browser_manager."""
+ return await self._browser.execute_command(command, params)
+
class DirectComputer(AsyncComputerHandler):
def __init__(self):
# use module-scope handler singletons created by HandlerFactory
self._auto = automation_handler
self._file = file_handler
self._access = accessibility_handler
+ # Create interface for BrowserTool compatibility
+ self._interface = DirectComputerInterface(automation_handler, get_browser_manager())
+
+ @property
+ def interface(self):
+ """Return interface compatible with BrowserTool.
+
+ This matches Computer.interface shape so BrowserTool works with either:
+ - computer.interface.interface.hotkey() -> automation
+ - computer.interface.playwright_exec() -> browser commands
+ - direct_computer.interface.interface.hotkey() -> automation
+ - direct_computer.interface.playwright_exec() -> browser commands
+ """
+ return self._interface
async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
sys = platform.system().lower()