Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(agent): fix FARA tool calls for terminate, visit_url, web_search, and click actions (#896)

Sarina Li6309a21ef498
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.py
index f73931e4fe..2f2342093c 100644
--- a/libs/python/agent/agent/agent.py
+++ b/libs/python/agent/agent/agent.py
@@ -537,10 +537,10 @@ class ComputerAgent:
# Perform computer actions
action = item.get("action")
- action_type = action.get("type")
- if action_type is None:
+ action_type = action.get("type") if action else None
+ if not action_type:
print(
- f"Action type cannot be `None`: action={action}, action_type={action_type}"
+ f"Action type is empty or None: action={action}, action_type={action_type}"
)
return []
@@ -551,17 +551,24 @@ class ComputerAgent:
# Execute the computer action
computer_method = getattr(computer, action_type, None)
+ action_result = None
if computer_method:
assert_callable_with(computer_method, **action_args)
- await computer_method(**action_args)
+ action_result = await computer_method(**action_args)
else:
raise ToolError(f"Unknown computer action: {action_type}")
- # Take screenshot after action
- if self.screenshot_delay and self.screenshot_delay > 0:
- await asyncio.sleep(self.screenshot_delay)
- screenshot_base64 = await computer.screenshot()
- await self._on_screenshot(screenshot_base64, "screenshot_after")
+ # Check if this was a terminate action
+ is_terminate = action_type == "terminate" or (
+ isinstance(action_result, dict) and action_result.get("terminated")
+ )
+
+ # Take screenshot after action (skip for terminate)
+ if not is_terminate:
+ if self.screenshot_delay and self.screenshot_delay > 0:
+ await asyncio.sleep(self.screenshot_delay)
+ screenshot_base64 = await computer.screenshot()
+ await self._on_screenshot(screenshot_base64, "screenshot_after")
# Handle safety checks
pending_checks = item.get("pending_safety_checks", [])
@@ -576,15 +583,24 @@ class ComputerAgent:
# raise ValueError(f"Safety check failed: {check_message}")
# Create call output
- call_output = {
- "type": "computer_call_output",
- "call_id": item.get("call_id"),
- "acknowledged_safety_checks": acknowledged_checks,
- "output": {
- "type": "input_image",
- "image_url": f"data:image/png;base64,{screenshot_base64}",
- },
- }
+ if is_terminate:
+ # For terminate action, include the terminated flag
+ call_output = {
+ "type": "computer_call_output",
+ "call_id": item.get("call_id"),
+ "acknowledged_safety_checks": acknowledged_checks,
+ "output": action_result if action_result else {"terminated": True},
+ }
+ else:
+ call_output = {
+ "type": "computer_call_output",
+ "call_id": item.get("call_id"),
+ "acknowledged_safety_checks": acknowledged_checks,
+ "output": {
+ "type": "input_image",
+ "image_url": f"data:image/png;base64,{screenshot_base64}",
+ },
+ }
# # Additional URL safety checks for browser environments
# if await computer.get_environment() == "browser":
diff --git a/libs/python/agent/agent/loops/fara/config.py b/libs/python/agent/agent/loops/fara/config.py
index a31efa6d92..5f95127769 100644
--- a/libs/python/agent/agent/loops/fara/config.py
+++ b/libs/python/agent/agent/loops/fara/config.py
@@ -21,6 +21,7 @@ from ...responses import (
from ...types import AgentCapability
from .helpers import (
build_nous_system,
+ convert_fara_args_to_browser_tool_format,
convert_qwen_tool_args_to_computer_action,
parse_tool_call_from_text,
unnormalize_coordinate,
@@ -89,19 +90,19 @@ class FaraVlmConfig(AsyncAgentConfig):
tool_type = tool.get("type")
if tool_type == "computer":
- # For computer tools, use QWEN3_COMPUTER_TOOL schema
+ # For computer tools, use FARA_COMPUTER_TOOL schema
computer = tool.get("computer")
if computer and is_agent_computer(computer):
- function_schemas.append(QWEN3_COMPUTER_TOOL["function"])
+ function_schemas.append(FARA_COMPUTER_TOOL["function"])
elif tool_type == "function":
# For function tools, use the provided function schema
function_schema = tool.get("function")
if function_schema:
function_schemas.append(function_schema)
- # If no tools provided or no computer tool found, use default QWEN3_COMPUTER_TOOL
+ # If no tools provided or no computer tool found, use default FARA_COMPUTER_TOOL
if not function_schemas:
- function_schemas = [QWEN3_COMPUTER_TOOL["function"]]
+ function_schemas = [FARA_COMPUTER_TOOL["function"]]
# Prepend Nous-generated system if available
nous_system = build_nous_system(function_schemas)
@@ -244,6 +245,9 @@ class FaraVlmConfig(AsyncAgentConfig):
)
args = await unnormalize_coordinate(raw_args, (last_rw, last_rh))
+ # Convert FARA model output format to BrowserTool compatible format
+ args = convert_fara_args_to_browser_tool_format(args)
+
# Extract thoughts (text before <tool_call> tag)
thoughts = ""
if "<tool_call>" in content_text:
@@ -281,11 +285,9 @@ class FaraVlmConfig(AsyncAgentConfig):
if "coordinate" in args and last_rw is not None and last_rh is not None:
args = await unnormalize_coordinate(args, (last_rw, last_rh))
- # Convert Qwen format to Computer Calls format if this is a computer tool
- if fn_name == "computer":
- converted_action = convert_qwen_tool_args_to_computer_action(args)
- if converted_action:
- args = converted_action
+ # Convert FARA model output format to BrowserTool compatible format
+ if fn_name in ("computer", "computer_use"):
+ args = convert_fara_args_to_browser_tool_format(args)
processed_tool_calls.append(
{
@@ -359,7 +361,7 @@ class FaraVlmConfig(AsyncAgentConfig):
reduced_tool = {
"type": "function",
"function": {
- **QWEN3_COMPUTER_TOOL["function"],
+ **FARA_COMPUTER_TOOL["function"],
"parameters": {
"type": "object",
"properties": {
@@ -439,8 +441,9 @@ class FaraVlmConfig(AsyncAgentConfig):
return None
-# ComputerUse tool schema (OpenAI function tool format)
-QWEN3_COMPUTER_TOOL: dict[str, Any] = {
+# FARA-specific ComputerUse tool schema (OpenAI function tool format)
+# This schema is tailored for FARA-7B model and includes browser-specific actions
+FARA_COMPUTER_TOOL: dict[str, Any] = {
"type": "function",
"function": {
"name": "computer",
@@ -451,7 +454,8 @@ QWEN3_COMPUTER_TOOL: dict[str, Any] = {
"* The screen's resolution is 1000x1000.\n"
"* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n"
"* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n"
- "* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges."
+ "* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges.\n"
+ "* Use terminate action when you have completed the task or cannot proceed further."
),
"parameters": {
"type": "object",
@@ -472,6 +476,10 @@ QWEN3_COMPUTER_TOOL: dict[str, Any] = {
"hscroll",
"screenshot",
"wait",
+ "visit_url",
+ "web_search",
+ "history_back",
+ "terminate",
],
"type": "string",
},
@@ -481,7 +489,7 @@ QWEN3_COMPUTER_TOOL: dict[str, Any] = {
"items": {"type": "string"},
},
"text": {
- "description": "Required only by action=type and action=answer.",
+ "description": "Required only by action=type.",
"type": "string",
},
"coordinate": {
@@ -499,6 +507,19 @@ QWEN3_COMPUTER_TOOL: dict[str, Any] = {
"description": "Seconds to wait (action=wait).",
"type": "number",
},
+ "url": {
+ "description": "The URL to visit. Required only by action=visit_url.",
+ "type": "string",
+ },
+ "query": {
+ "description": "The search query. Required only by action=web_search.",
+ "type": "string",
+ },
+ "status": {
+ "description": "Task completion status. Required only by action=terminate.",
+ "type": "string",
+ "enum": ["success", "failure"],
+ },
},
"required": ["action"],
},
diff --git a/libs/python/agent/agent/loops/fara/helpers.py b/libs/python/agent/agent/loops/fara/helpers.py
index 5ffc7a28bb..61a3996d7f 100644
--- a/libs/python/agent/agent/loops/fara/helpers.py
+++ b/libs/python/agent/agent/loops/fara/helpers.py
@@ -355,3 +355,130 @@ def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[
# Non-UI or terminal actions: terminate/answer -> not mapped here
return None
+
+
+def convert_fara_args_to_browser_tool_format(args: Dict[str, Any]) -> Dict[str, Any]:
+ """
+ Convert FARA model output format to BrowserTool compatible format.
+
+ FARA model may output extra parameters that BrowserTool methods don't accept.
+ This function cleans up the arguments and maps them to the correct format.
+
+ Examples:
+ Input: {"action": "click", "button": "left", "x": 378, "y": 144}
+ Output: {"action": "left_click", "coordinate": [378, 144]}
+
+ Input: {"action": "visit_url", "url": "https://...", "text": "..."}
+ Output: {"action": "visit_url", "url": "https://..."}
+
+ Input: {"action": "terminate", "url": "...", "text": "...", "status": "success"}
+ Output: {"action": "terminate", "status": "success"}
+ """
+ if not isinstance(args, dict):
+ return args
+
+ action = args.get("action", "")
+ if not isinstance(action, str):
+ return args
+
+ a = action.lower()
+ result: Dict[str, Any] = {"action": a}
+
+ # Handle coordinate-based actions
+ # Check for both coordinate array and separate x/y fields
+ coord = args.get("coordinate")
+ x = args.get("x")
+ y = args.get("y")
+
+ if coord and isinstance(coord, (list, tuple)) and len(coord) >= 2:
+ x, y = coord[0], coord[1]
+
+ # Click actions - normalize to left_click with coordinate
+ if a in {"click", "left_click"}:
+ if x is not None and y is not None:
+ result["action"] = "left_click"
+ result["coordinate"] = [x, y]
+ return result
+
+ if a in {"right_click", "middle_click", "double_click", "triple_click"}:
+ if x is not None and y is not None:
+ result["coordinate"] = [x, y]
+ return result
+
+ if a == "mouse_move":
+ if x is not None and y is not None:
+ result["coordinate"] = [x, y]
+ return result
+
+ if a == "left_click_drag":
+ if x is not None and y is not None:
+ result["coordinate"] = [x, y]
+ # Also handle start/end coordinates if present
+ start_coord = args.get("start_coordinate")
+ end_coord = args.get("end_coordinate")
+ if start_coord:
+ result["start_coordinate"] = start_coord
+ if end_coord:
+ result["end_coordinate"] = end_coord
+ return result
+
+ # Keyboard actions
+ if a == "key":
+ keys = args.get("keys")
+ if keys:
+ result["keys"] = keys
+ return result
+
+ if a == "type":
+ text = args.get("text")
+ if text:
+ result["text"] = text
+ # Include coordinate if typing at a specific location
+ if x is not None and y is not None:
+ result["coordinate"] = [x, y]
+ return result
+
+ # Scroll actions
+ if a in {"scroll", "hscroll"}:
+ pixels = args.get("pixels")
+ if pixels is not None:
+ result["pixels"] = pixels
+ if x is not None and y is not None:
+ result["coordinate"] = [x, y]
+ return result
+
+ # Browser-specific actions
+ if a == "visit_url":
+ url = args.get("url")
+ if url:
+ result["url"] = url
+ return result
+
+ if a == "web_search":
+ query = args.get("query")
+ if query:
+ result["query"] = query
+ return result
+
+ if a == "history_back":
+ return result
+
+ # Wait action
+ if a == "wait":
+ time_val = args.get("time")
+ if time_val is not None:
+ result["time"] = time_val
+ return result
+
+ # Screenshot action
+ if a == "screenshot":
+ return result
+
+ # Terminate action
+ if a == "terminate":
+ status = args.get("status", "success")
+ result["status"] = status
+ return result
+
+ # For any other action, return cleaned args (just action + known fields)
+ return result
diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.py
index f19e8e59ed..2e83266c9e 100644
--- a/libs/python/agent/agent/tools/browser_tool.py
+++ b/libs/python/agent/agent/tools/browser_tool.py
@@ -421,3 +421,96 @@ class BrowserTool(BaseComputerTool):
else:
error = result.get("error", "Unknown error")
raise RuntimeError(f"Failed to get current URL: {error}")
+
+ # FARA-compatible action methods
+ # These methods accept parameters in the format that FARA model outputs
+ # and agent.py passes via **action_args
+
+ async def left_click(self, coordinate=None, **kwargs) -> dict:
+ """Left click at coordinates. FARA-compatible."""
+ return await self._action_left_click({"coordinate": coordinate})
+
+ async def right_click(self, coordinate=None, **kwargs) -> dict:
+ """Right click at coordinates. FARA-compatible."""
+ if not coordinate or len(coordinate) != 2:
+ return {"success": False, "error": "coordinate parameter [x, y] is required"}
+ # Use playwright_exec for right click
+ result = await self.interface.playwright_exec(
+ "click", {"x": coordinate[0], "y": coordinate[1], "button": "right"}
+ )
+ return result
+
+ async def middle_click(self, coordinate=None, **kwargs) -> dict:
+ """Middle click at coordinates. FARA-compatible."""
+ if not coordinate or len(coordinate) != 2:
+ return {"success": False, "error": "coordinate parameter [x, y] is required"}
+ result = await self.interface.playwright_exec(
+ "click", {"x": coordinate[0], "y": coordinate[1], "button": "middle"}
+ )
+ return result
+
+ async def double_click(self, coordinate=None, **kwargs) -> dict:
+ """Double click at coordinates. FARA-compatible."""
+ if not coordinate or len(coordinate) != 2:
+ return {"success": False, "error": "coordinate parameter [x, y] is required"}
+ result = await self.interface.playwright_exec(
+ "dblclick", {"x": coordinate[0], "y": coordinate[1]}
+ )
+ return result
+
+ async def triple_click(self, coordinate=None, **kwargs) -> dict:
+ """Triple click at coordinates. FARA-compatible."""
+ if not coordinate or len(coordinate) != 2:
+ return {"success": False, "error": "coordinate parameter [x, y] is required"}
+ # Triple click is approximated as double click
+ return await self.double_click(coordinate=coordinate)
+
+ async def mouse_move(self, coordinate=None, **kwargs) -> dict:
+ """Move mouse to coordinates. FARA-compatible."""
+ return await self._action_mouse_move({"coordinate": coordinate})
+
+ async def left_click_drag(
+ self, coordinate=None, start_coordinate=None, end_coordinate=None, **kwargs
+ ) -> dict:
+ """Drag from start to end coordinates. FARA-compatible."""
+ if start_coordinate and end_coordinate:
+ # Use start/end coordinates if provided
+ await self.interface.interface.move_cursor(start_coordinate[0], start_coordinate[1])
+ await self.interface.interface.mouse_down(start_coordinate[0], start_coordinate[1])
+ await self.interface.interface.move_cursor(end_coordinate[0], end_coordinate[1])
+ await self.interface.interface.mouse_up(end_coordinate[0], end_coordinate[1])
+ return {
+ "success": True,
+ "message": f"Dragged from {start_coordinate} to {end_coordinate}",
+ }
+ elif coordinate:
+ # Just move to coordinate
+ await self.interface.interface.move_cursor(coordinate[0], coordinate[1])
+ return {"success": True, "message": f"Moved to {coordinate}"}
+ return {
+ "success": False,
+ "error": "start_coordinate and end_coordinate or coordinate required",
+ }
+
+ async def key(self, keys=None, **kwargs) -> dict:
+ """Press keys. FARA-compatible."""
+ return await self._action_key({"keys": keys})
+
+ async def hscroll(self, pixels=None, coordinate=None, **kwargs) -> dict:
+ """Horizontal scroll. FARA-compatible."""
+ if pixels is None:
+ return {"success": False, "error": "pixels parameter is required"}
+ result = await self.interface.playwright_exec("scroll", {"delta_x": pixels, "delta_y": 0})
+ return result
+
+ async def wait(self, time=None, **kwargs) -> dict:
+ """Wait for specified seconds. FARA-compatible."""
+ return await self._action_wait({"time": time})
+
+ async def history_back(self, **kwargs) -> dict:
+ """Go back in browser history. FARA-compatible."""
+ return await self._action_history_back({})
+
+ async def terminate(self, status=None, **kwargs) -> dict:
+ """Terminate and report status. FARA-compatible."""
+ return await self._action_terminate({"status": status or "success"})