Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(fara): correct coordinate scaling and agent conversion loops (#934)

Sarina Li8759df254931
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.py
index 4c40f6a30a..dbf11adcf6 100644
--- a/libs/python/agent/agent/agent.py
+++ b/libs/python/agent/agent/agent.py
@@ -690,8 +690,6 @@ class ComputerAgent:
# Extract action arguments (all fields except 'type')
action_args = {k: v for k, v in action.items() if k != "type"}
- # print(f"{action_type}({action_args})")
-
# Execute the computer action
computer_method = getattr(computer, action_type, None)
action_result = None
diff --git a/libs/python/agent/agent/loops/fara/config.py b/libs/python/agent/agent/loops/fara/config.py
index df36bc6182..c31d48d05f 100644
--- a/libs/python/agent/agent/loops/fara/config.py
+++ b/libs/python/agent/agent/loops/fara/config.py
@@ -14,20 +14,167 @@ from litellm.responses.litellm_completion_transformation.transformation import (
from ...decorators import register_agent
from ...loops.base import AsyncAgentConfig
from ...responses import (
- convert_completion_messages_to_responses_items,
- convert_responses_items_to_completion_messages,
+ make_click_item,
+ make_double_click_item,
+ make_drag_item,
+ make_keypress_item,
+ make_move_item,
+ make_output_text_item,
make_reasoning_item,
+ make_screenshot_item,
+ make_scroll_item,
+ make_type_item,
+ make_wait_item,
)
from ...types import AgentCapability
from .helpers import (
+ _convert_responses_items_to_fara_messages,
build_nous_system,
- convert_fara_args_to_browser_tool_format,
- convert_qwen_tool_args_to_computer_action,
parse_tool_call_from_text,
- unnormalize_coordinate,
)
+def _scale_fara_coordinates(
+ args: Dict[str, Any],
+ original_dims: Tuple[int, int],
+ resized_dims: Tuple[int, int],
+) -> Dict[str, Any]:
+ """
+ Scale FARA coordinates from resized image space to original viewport space.
+
+ FARA outputs pixel coordinates on the resized image (after smart_resize).
+ This scales them back to the original browser viewport, matching FARA's
+ convert_resized_coords_to_original() in fara_agent.py:
+ scale_x = og_w / rsz_w
+ return [coords[0] * scale_x, coords[1] * scale_y]
+
+ Args:
+ args: Action arguments containing "coordinate" key
+ original_dims: (width, height) of original browser viewport
+ resized_dims: (width, height) after smart_resize
+ """
+ coord = args.get("coordinate")
+ if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+ return args
+
+ x, y = float(coord[0]), float(coord[1])
+ original_w, original_h = float(original_dims[0]), float(original_dims[1])
+ resized_w, resized_h = float(resized_dims[0]), float(resized_dims[1])
+
+ # Scale from resized to original: x_final = x * (original / resized)
+ scale_x = original_w / resized_w
+ scale_y = original_h / resized_h
+
+ x_scaled = max(0.0, min(original_w, x * scale_x))
+ y_scaled = max(0.0, min(original_h, y * scale_y))
+
+ return {**args, "coordinate": [round(x_scaled), round(y_scaled)]}
+
+
+def _fara_args_to_sdk_item(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """
+ Convert FARA model output args to SDK item using make_*_item helpers.
+
+ FARA format: {"action": "left_click", "coordinate": [100, 200]}
+ SDK format: ResponseComputerToolCallParam with action={"type": "click", "x": 100, "y": 200}
+ """
+ action = args.get("action", "")
+ coordinate = args.get("coordinate", [0, 0])
+ x = coordinate[0] if len(coordinate) > 0 else 0
+ y = coordinate[1] if len(coordinate) > 1 else 0
+
+ # Click actions
+ if action in ("left_click", "click"):
+ return make_click_item(x=x, y=y, button="left")
+ if action == "right_click":
+ return make_click_item(x=x, y=y, button="right")
+ if action == "middle_click":
+ return make_click_item(x=x, y=y, button="wheel")
+ if action == "double_click":
+ return make_double_click_item(x=x, y=y)
+
+ # Type action
+ if action == "type":
+ return make_type_item(text=args.get("text", ""))
+
+ # Key action
+ if action in ("key", "keypress"):
+ keys = args.get("keys", [])
+ if isinstance(keys, str):
+ keys = keys.split("+")
+ return make_keypress_item(keys=keys)
+
+ # Move action
+ if action in ("mouse_move", "move"):
+ return make_move_item(x=x, y=y)
+
+ # Scroll action
+ if action == "scroll":
+ pixels = args.get("pixels") or 0 # Handle None explicitly
+ # FARA: positive = up, negative = down
+ scroll_y = -pixels # SDK: positive = down
+ return make_scroll_item(x=x, y=y, scroll_x=0, scroll_y=scroll_y)
+
+ if action == "hscroll":
+ pixels = args.get("pixels") or 0 # Handle None explicitly
+ return make_scroll_item(x=x, y=y, scroll_x=pixels, scroll_y=0)
+
+ # Drag action
+ if action == "left_click_drag":
+ start_coord = args.get("start_coordinate", [0, 0])
+ end_coord = args.get("end_coordinate", [0, 0])
+ return make_drag_item(
+ path=[
+ {"x": start_coord[0], "y": start_coord[1]},
+ {"x": end_coord[0], "y": end_coord[1]},
+ ]
+ )
+
+ # Screenshot
+ if action == "screenshot":
+ return make_screenshot_item()
+
+ # Wait
+ if action == "wait":
+ return make_wait_item()
+
+ # Terminate - return None so no action is executed
+ # The caller checks for terminate action and adds an assistant message to stop the loop
+ if action == "terminate":
+ return None
+
+ # FARA browser-specific actions - create computer_call items directly
+ # agent.py uses getattr(computer, action_type) to call these methods
+ if action == "visit_url":
+ return {
+ "type": "computer_call",
+ "call_id": f"call_{id(args)}",
+ "action": {"type": "visit_url", "url": args.get("url", "")},
+ "pending_safety_checks": [],
+ "status": "completed",
+ }
+
+ if action == "web_search":
+ return {
+ "type": "computer_call",
+ "call_id": f"call_{id(args)}",
+ "action": {"type": "web_search", "query": args.get("query", "")},
+ "pending_safety_checks": [],
+ "status": "completed",
+ }
+
+ if action == "history_back":
+ return {
+ "type": "computer_call",
+ "call_id": f"call_{id(args)}",
+ "action": {"type": "history_back"},
+ "pending_safety_checks": [],
+ "status": "completed",
+ }
+
+ return None
+
+
@register_agent(models=r"(?i).*fara-7b.*", tool_type="browser")
class FaraVlmConfig(AsyncAgentConfig):
async def predict_step(
@@ -75,10 +222,10 @@ class FaraVlmConfig(AsyncAgentConfig):
"usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
}
- # Build messages using NousFnCallPrompt system with tool schema in text
- # Start with converted conversation (images/text preserved)
- converted_msgs = convert_responses_items_to_completion_messages(
- messages, allow_images_in_tool_results=False, use_xml_tools=True
+ # Build messages using FARA's dedicated conversion layer
+ # This converts SDK format to FARA's native format (action + coordinate)
+ converted_msgs = _convert_responses_items_to_fara_messages(
+ messages, allow_images_in_tool_results=False
)
# Build function schemas from tools array
@@ -154,7 +301,9 @@ class FaraVlmConfig(AsyncAgentConfig):
completion_messages.append(screenshot_msg)
# Smart-resize all screenshots and attach min/max pixel hints. Fail fast if deps missing.
- # Also record the last resized width/height to unnormalize coordinates later.
+ # Track both original and resized dimensions for coordinate scaling.
+ last_original_w: Optional[int] = None
+ last_original_h: Optional[int] = None
last_rw: Optional[int] = None
last_rh: Optional[int] = None
MIN_PIXELS = 3136
@@ -189,6 +338,8 @@ class FaraVlmConfig(AsyncAgentConfig):
# Attach hints on this image block
part["min_pixels"] = MIN_PIXELS
part["max_pixels"] = MAX_PIXELS
+ # Track both original and resized dimensions
+ last_original_w, last_original_h = w, h
last_rw, last_rh = rw, rh
api_kwargs: Dict[str, Any] = {
@@ -227,52 +378,55 @@ class FaraVlmConfig(AsyncAgentConfig):
reasoning_text = message.get("reasoning") or ""
output_items: List[Dict[str, Any]] = []
+ has_terminate = False
# Add reasoning if present (Ollama Cloud format)
if reasoning_text:
output_items.append(make_reasoning_item(reasoning_text))
+ # Extract thoughts (text before <tool_call> tag)
+ thoughts = ""
+ if "<tool_call>" in content_text:
+ thoughts = content_text.split("<tool_call>")[0].strip()
+
+ # Add thoughts as assistant message if present
+ if thoughts:
+ output_items.append(make_output_text_item(thoughts))
+
# Priority 1: Try to parse tool call from content text (OpenRouter format)
tool_call = parse_tool_call_from_text(content_text)
if tool_call and isinstance(tool_call, dict):
fn_name = tool_call.get("name") or "computer"
raw_args = tool_call.get("arguments") or {}
- # Unnormalize coordinates to actual screen size using last resized dims
- if last_rw is None or last_rh is None:
+
+ # Scale coordinates from resized image space to original viewport
+ if (
+ last_rw is None
+ or last_rh is None
+ or last_original_w is None
+ or last_original_h is None
+ ):
raise RuntimeError(
- "No screenshots found to derive dimensions for coordinate unnormalization."
+ "No screenshots found to derive dimensions for coordinate scaling."
)
- args = await unnormalize_coordinate(raw_args, (last_rw, last_rh))
-
- # Convert FARA model output format to BrowserTool compatible format
- args = convert_fara_args_to_browser_tool_format(args)
+ args = _scale_fara_coordinates(
+ raw_args,
+ original_dims=(last_original_w, last_original_h),
+ resized_dims=(last_rw, last_rh),
+ )
- # Extract thoughts (text before <tool_call> tag)
- thoughts = ""
- if "<tool_call>" in content_text:
- thoughts = content_text.split("<tool_call>")[0].strip()
+ # Convert FARA output to SDK format using make_*_item helpers
+ if fn_name in ("computer", "computer_use"):
+ item = _fara_args_to_sdk_item(args)
+ if item:
+ output_items.append(item)
+ # Check for terminate (even if item is None)
+ if args.get("action") == "terminate":
+ has_terminate = True
- # Build an OpenAI-style tool call so we can reuse the converter
- fake_cm = {
- "role": "assistant",
- "content": thoughts, # Preserve thoughts before tool call
- "tool_calls": [
- {
- "type": "function",
- "id": "call_0",
- "function": {
- "name": fn_name,
- "arguments": json.dumps(args),
- },
- }
- ],
- }
- output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
elif tool_calls_array:
# Priority 2: Use tool_calls field if present (Ollama Cloud format)
- # Process and unnormalize coordinates in tool calls
- processed_tool_calls = []
for tc in tool_calls_array:
function = tc.get("function", {})
fn_name = function.get("name", "computer")
@@ -281,56 +435,30 @@ class FaraVlmConfig(AsyncAgentConfig):
try:
args = json.loads(args_str)
- # Unnormalize coordinates if present
+ # Scale coordinates from resized image space to original viewport
if "coordinate" in args and last_rw is not None and last_rh is not None:
- args = await unnormalize_coordinate(args, (last_rw, last_rh))
-
- # Convert FARA model output format to BrowserTool compatible format
+ if last_original_w is not None and last_original_h is not None:
+ args = _scale_fara_coordinates(
+ args,
+ original_dims=(last_original_w, last_original_h),
+ resized_dims=(last_rw, last_rh),
+ )
+
+ # Convert FARA output to SDK format
if fn_name in ("computer", "computer_use"):
- args = convert_fara_args_to_browser_tool_format(args)
-
- processed_tool_calls.append(
- {
- "type": tc.get("type", "function"),
- "id": tc.get("id", "call_0"),
- "function": {
- "name": fn_name,
- "arguments": json.dumps(args),
- },
- }
- )
+ item = _fara_args_to_sdk_item(args)
+ if item:
+ output_items.append(item)
+ # Check for terminate (even if item is None)
+ if args.get("action") == "terminate":
+ has_terminate = True
except json.JSONDecodeError:
- # Keep original if parsing fails
- processed_tool_calls.append(tc)
-
- fake_cm = {
- "role": "assistant",
- "content": content_text if content_text else "",
- "tool_calls": processed_tool_calls,
- }
- output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
- else:
- # No tool calls found in either format, return text response
- fake_cm = {"role": "assistant", "content": content_text}
- output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
-
- # Check if this is a terminate action - if so, add a final assistant message to stop the loop
- has_terminate = False
- for item in output_items:
- if item.get("type") == "computer_call":
- action = item.get("action", {})
- if action.get("type") == "terminate":
- has_terminate = True
- break
- elif item.get("type") == "function_call":
- try:
- args = json.loads(item.get("arguments", "{}"))
- if args.get("action") == "terminate":
- has_terminate = True
- break
- except:
pass
+ elif content_text:
+ # No tool calls found, return text response
+ output_items.append(make_output_text_item(content_text))
+
# If terminate detected, ensure LAST item is an assistant message to exit the loop
# The generic agent loop checks: while new_items[-1].get("role") != "assistant"
if has_terminate:
@@ -434,7 +562,12 @@ class FaraVlmConfig(AsyncAgentConfig):
content_text = ((choice.get("message") or {}).get("content")) or ""
tool_call = parse_tool_call_from_text(content_text) or {}
args = tool_call.get("arguments") or {}
- args = await unnormalize_coordinate(args, (rh, rw))
+ # Scale from resized image space to original viewport
+ args = _scale_fara_coordinates(
+ args,
+ original_dims=(w, h),
+ resized_dims=(rw, rh),
+ )
coord = args.get("coordinate")
if isinstance(coord, (list, tuple)) and len(coord) >= 2:
return int(coord[0]), int(coord[1])
@@ -443,10 +576,11 @@ class FaraVlmConfig(AsyncAgentConfig):
# FARA-specific ComputerUse tool schema (OpenAI function tool format)
# This schema is tailored for FARA-7B model and includes browser-specific actions
+# NOTE: Tool name MUST be "computer_use" to match what FARA-7B was trained on
FARA_COMPUTER_TOOL: dict[str, Any] = {
"type": "function",
"function": {
- "name": "computer",
+ "name": "computer_use",
"description": (
"Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
"* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n"
diff --git a/libs/python/agent/agent/loops/fara/helpers.py b/libs/python/agent/agent/loops/fara/helpers.py
index 61a3996d7f..99bc22a5d7 100644
--- a/libs/python/agent/agent/loops/fara/helpers.py
+++ b/libs/python/agent/agent/loops/fara/helpers.py
@@ -225,6 +225,103 @@ def build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any
return {"role": "system", "content": content}
+def fix_fara_tool_call_format(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """
+ Fix tool call format in conversation history for FARA compatibility.
+
+ The shared `convert_responses_items_to_completion_messages` function outputs:
+ - Tool name as "computer" (should be "computer_use")
+ - Action key as "type" (should be "action")
+
+ This function post-processes assistant messages to fix these issues.
+ """
+ import re
+
+ # Valid FARA action types
+ valid_actions = {
+ "left_click",
+ "right_click",
+ "middle_click",
+ "double_click",
+ "triple_click",
+ "click",
+ "type",
+ "key",
+ "scroll",
+ "hscroll",
+ "mouse_move",
+ "wait",
+ "visit_url",
+ "web_search",
+ "history_back",
+ "screenshot",
+ "terminate",
+ }
+
+ fixed_messages = []
+ for msg in messages:
+ if msg.get("role") != "assistant":
+ fixed_messages.append(msg)
+ continue
+
+ content = msg.get("content", "")
+ if not isinstance(content, str) or "<tool_call>" not in content:
+ fixed_messages.append(msg)
+ continue
+
+ # Find and fix all tool calls in the content
+ def fix_tool_call(match):
+ tool_call_content = match.group(1)
+ try:
+ tool_call = json.loads(tool_call_content)
+
+ # Fix tool name: "computer" -> "computer_use"
+ if tool_call.get("name") == "computer":
+ tool_call["name"] = "computer_use"
+
+ # Fix arguments: "type" -> "action" and x/y -> coordinate
+ args = tool_call.get("arguments", {})
+ if isinstance(args, dict):
+ # If "type" contains a valid action, rename to "action"
+ if "type" in args and args["type"] in valid_actions:
+ args["action"] = args.pop("type")
+
+ # Convert internal x/y format back to FARA coordinate format
+ if "x" in args and "y" in args and "coordinate" not in args:
+ args["coordinate"] = [args.pop("x"), args.pop("y")]
+
+ # Normalize action names: "click" -> "left_click"
+ if args.get("action") == "click":
+ args["action"] = "left_click"
+
+ # Remove "button" field - FARA doesn't use it (action name implies button)
+ args.pop("button", None)
+
+ # If "action" is empty but we can infer from other keys
+ if args.get("action") == "" and "coordinate" in args:
+ args["action"] = "left_click"
+
+ tool_call["arguments"] = args
+
+ return f"<tool_call>\n{json.dumps(tool_call)}\n</tool_call>"
+ except (json.JSONDecodeError, TypeError):
+ return match.group(0) # Return original if parsing fails
+
+ # Match <tool_call>...</tool_call> or <tool_call>...</tool_call>
+ fixed_content = re.sub(
+ r"<tool_call>\s*(\{.*?\})\s*</tool_call>", fix_tool_call, content, flags=re.DOTALL
+ )
+
+ # Also handle malformed closing tags like <tool_call> used as closing
+ fixed_content = re.sub(
+ r"<tool_call>(\{.*?\})<tool_call>", fix_tool_call, fixed_content, flags=re.DOTALL
+ )
+
+ fixed_messages.append({**msg, "content": fixed_content})
+
+ return fixed_messages
+
+
def parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
"""Extract JSON object within <tool_call>...</tool_call> from model text.
@@ -482,3 +579,239 @@ def convert_fara_args_to_browser_tool_format(args: Dict[str, Any]) -> Dict[str,
# For any other action, return cleaned args (just action + known fields)
return result
+
+
+def _convert_responses_items_to_fara_messages(
+ messages: List[Dict[str, Any]],
+ allow_images_in_tool_results: bool = False,
+) -> List[Dict[str, Any]]:
+ """
+ Convert SDK responses_items format to FARA-compatible completion messages.
+
+ This is FARA's dedicated conversion layer (similar to Anthropic's pattern).
+ It handles the conversion from SDK's OpenAI-style format to FARA's native format:
+
+ SDK format:
+ {"type": "click", "x": 100, "y": 200, "button": "left"}
+
+ FARA format (in XML tool_call):
+ {"name": "computer_use", "arguments": {"action": "left_click", "coordinate": [100, 200]}}
+ """
+ completion_messages: List[Dict[str, Any]] = []
+
+ for message in messages:
+ msg_type = message.get("type")
+ role = message.get("role")
+
+ # Handle user messages
+ if role == "user" or msg_type == "user":
+ content = message.get("content", "")
+ if isinstance(content, list):
+ converted_content = []
+ for item in content:
+ if isinstance(item, dict):
+ item_type = item.get("type")
+ if item_type == "input_image":
+ image_url = item.get("image_url", "")
+ if image_url and image_url != "[omitted]":
+ converted_content.append(
+ {"type": "image_url", "image_url": {"url": image_url}}
+ )
+ elif item_type == "input_text":
+ converted_content.append({"type": "text", "text": item.get("text", "")})
+ elif item_type == "image_url":
+ # Already in correct format
+ converted_content.append(item)
+ elif item_type == "text":
+ converted_content.append(item)
+ else:
+ converted_content.append(item)
+ else:
+ converted_content.append({"type": "text", "text": str(item)})
+ completion_messages.append({"role": "user", "content": converted_content})
+ else:
+ completion_messages.append({"role": "user", "content": content})
+
+ # Handle assistant messages
+ elif role == "assistant" and msg_type == "message":
+ content = message.get("content", [])
+ if isinstance(content, str):
+ completion_messages.append({"role": "assistant", "content": content})
+ elif isinstance(content, list):
+ text_parts = []
+ for item in content:
+ if isinstance(item, dict) and item.get("type") == "output_text":
+ text_parts.append(item.get("text", ""))
+ completion_messages.append({"role": "assistant", "content": "\n".join(text_parts)})
+
+ # Handle reasoning
+ elif msg_type == "reasoning":
+ summary = message.get("summary", [])
+ reasoning_text = ""
+ if isinstance(summary, list) and summary:
+ for item in summary:
+ if isinstance(item, dict) and item.get("type") == "summary_text":
+ reasoning_text = item.get("text", "")
+ break
+ if reasoning_text:
+ completion_messages.append({"role": "assistant", "content": reasoning_text})
+
+ # Handle computer_call - convert SDK format to FARA's XML tool_call format
+ elif msg_type == "computer_call":
+ action = message.get("action", {})
+ action_type = action.get("type")
+
+ # Convert SDK action to FARA format
+ fara_args = _sdk_action_to_fara_args(action)
+
+ # Build FARA's XML tool_call format
+ tool_call_json = json.dumps({"name": "computer_use", "arguments": fara_args})
+ tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"
+
+ # Append to last assistant message or create new one
+ if completion_messages and completion_messages[-1].get("role") == "assistant":
+ prev_content = completion_messages[-1].get("content", "")
+ completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()
+ else:
+ completion_messages.append({"role": "assistant", "content": tool_call_text})
+
+ # Handle computer_call_output - convert to FARA's tool_response format
+ elif msg_type == "computer_call_output":
+ output = message.get("output", {})
+
+ # Build response content
+ if isinstance(output, dict) and output.get("type") == "input_image":
+ image_url = output.get("image_url", "")
+ response_text = "<tool_response>\nAction executed successfully. Here is the next screenshot.\n</tool_response>"
+
+ # Add as user message with image
+ if allow_images_in_tool_results and image_url and image_url != "[omitted]":
+ completion_messages.append(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": response_text},
+ {"type": "image_url", "image_url": {"url": image_url}},
+ ],
+ }
+ )
+ else:
+ completion_messages.append(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": response_text},
+ ],
+ }
+ )
+ elif isinstance(output, dict) and output.get("terminated"):
+ response_text = "<tool_response>\nTask terminated.\n</tool_response>"
+ completion_messages.append({"role": "user", "content": response_text})
+ else:
+ response_text = f"<tool_response>\n{json.dumps(output) if isinstance(output, dict) else str(output)}\n</tool_response>"
+ completion_messages.append({"role": "user", "content": response_text})
+
+ # Handle function_call (non-computer tools)
+ elif msg_type == "function_call":
+ fn_name = message.get("name", "")
+ fn_args = message.get("arguments", "{}")
+
+ tool_call_json = json.dumps(
+ {
+ "name": fn_name,
+ "arguments": json.loads(fn_args) if isinstance(fn_args, str) else fn_args,
+ }
+ )
+ tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"
+
+ if completion_messages and completion_messages[-1].get("role") == "assistant":
+ prev_content = completion_messages[-1].get("content", "")
+ completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()
+ else:
+ completion_messages.append({"role": "assistant", "content": tool_call_text})
+
+ # Handle function_call_output
+ elif msg_type == "function_call_output":
+ output = message.get("output", "")
+ response_text = f"<tool_response>\n{output}\n</tool_response>"
+ completion_messages.append({"role": "user", "content": response_text})
+
+ return completion_messages
+
+
+def _sdk_action_to_fara_args(action: Dict[str, Any]) -> Dict[str, Any]:
+ """
+ Convert SDK action format to FARA arguments format.
+
+ SDK format: {"type": "click", "x": 100, "y": 200, "button": "left"}
+ FARA format: {"action": "left_click", "coordinate": [100, 200]}
+ """
+ action_type = action.get("type", "")
+
+ # Click actions
+ if action_type == "click":
+ button = action.get("button", "left")
+ action_name = {
+ "left": "left_click",
+ "right": "right_click",
+ "wheel": "middle_click",
+ "middle": "middle_click",
+ }.get(button, "left_click")
+ return {"action": action_name, "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+ if action_type == "double_click":
+ return {"action": "double_click", "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+ # Type action
+ if action_type == "type":
+ result = {"action": "type", "text": action.get("text", "")}
+ # Include coordinate if present (for click-then-type)
+ if "x" in action and "y" in action:
+ result["coordinate"] = [action.get("x", 0), action.get("y", 0)]
+ return result
+
+ # Keypress action
+ if action_type == "keypress":
+ keys = action.get("keys", [])
+ return {"action": "key", "keys": keys}
+
+ # Move action
+ if action_type in ("move", "mouse_move"):
+ return {"action": "mouse_move", "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+ # Scroll action
+ if action_type == "scroll":
+ scroll_x = action.get("scroll_x", 0)
+ scroll_y = action.get("scroll_y", 0)
+ # FARA uses pixels (positive = up/left, negative = down/right)
+ pixels = scroll_y if scroll_y != 0 else scroll_x
+ result = {"action": "scroll", "pixels": pixels}
+ if "x" in action and "y" in action:
+ result["coordinate"] = [action.get("x", 0), action.get("y", 0)]
+ return result
+
+ # Drag action
+ if action_type == "drag":
+ path = action.get("path", [])
+ if len(path) >= 2:
+ return {
+ "action": "left_click_drag",
+ "start_coordinate": [path[0].get("x", 0), path[0].get("y", 0)],
+ "end_coordinate": [path[-1].get("x", 0), path[-1].get("y", 0)],
+ }
+ return {"action": "left_click_drag"}
+
+ # Screenshot
+ if action_type == "screenshot":
+ return {"action": "screenshot"}
+
+ # Wait
+ if action_type == "wait":
+ return {"action": "wait"}
+
+ # Terminate
+ if action_type == "terminate":
+ return {"action": "terminate", "status": action.get("status", "success")}
+
+ # Fallback - return as-is with type renamed to action
+ return {"action": action_type, **{k: v for k, v in action.items() if k != "type"}}
diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.py
index ccfc5c00f5..6163579585 100644
--- a/libs/python/agent/agent/tools/browser_tool.py
+++ b/libs/python/agent/agent/tools/browser_tool.py
@@ -337,8 +337,9 @@ class BrowserTool(BaseComputerTool):
async def _action_scroll(self, params: dict) -> dict:
"""Scroll the page."""
- pixels = params.get("pixels", 0)
- if pixels == 0:
+ pixels = params.get("pixels")
+ # Handle None explicitly - default to 0 means "no scroll requested"
+ if pixels is None or pixels == 0:
return {"success": False, "error": "pixels parameter is required"}
# Positive = up (negative delta_y), Negative = down (positive delta_y)
@@ -387,7 +388,10 @@ class BrowserTool(BaseComputerTool):
async def _action_wait(self, params: dict) -> dict:
"""Wait for specified seconds."""
- time = params.get("time", 0)
+ time = params.get("time")
+ # Handle None or missing time - default to 3 seconds (matches FARA behavior)
+ if time is None:
+ time = 3
if time <= 0:
return {"success": False, "error": "time parameter must be positive"}
@@ -396,7 +400,10 @@ class BrowserTool(BaseComputerTool):
async def _action_terminate(self, params: dict) -> dict:
"""Terminate and report status."""
- status = params.get("status", "success")
+ status = params.get("status")
+ # Handle None or missing status - default to "success"
+ if status is None:
+ status = "success"
message = f"Task terminated with status: {status}"
if self._facts:
@@ -409,17 +416,54 @@ class BrowserTool(BaseComputerTool):
"""Navigate to a URL."""
return await self._action_visit_url({"url": url})
- async def click(self, x: int, y: int) -> dict:
- """Click at coordinates."""
- return await self._action_left_click({"coordinate": [x, y]})
+ async def click(self, x: int = None, y: int = None, button: str = "left", **kwargs) -> dict:
+ """Click at coordinates. Supports both positional (x, y) and kwargs (button, x, y).
+
+ This is compatible with the normalized format from OperatorNormalizerCallback
+ which transforms actions like {"type": "left_click", "coordinate": [x, y]}
+ into {"type": "click", "button": "left", "x": x, "y": y}.
+ """
+ if x is None or y is None:
+ return {"success": False, "error": "x and y coordinates are required"}
+ if button == "right":
+ return await self.interface.playwright_exec(
+ "click", {"x": x, "y": y, "button": "right"}
+ )
+ elif button == "middle" or button == "wheel":
+ return await self.interface.playwright_exec(
+ "click", {"x": x, "y": y, "button": "middle"}
+ )
+ else:
+ # Default to left click
+ return await self._action_left_click({"coordinate": [x, y]})
async def type(self, text: str) -> dict:
"""Type text into the focused element."""
return await self._action_type({"text": text})
- async def scroll(self, delta_x: int, delta_y: int) -> dict:
- """Scroll the page."""
- return await self._action_scroll({"pixels": -delta_y})
+ async def scroll(
+ self,
+ delta_x: int = None,
+ delta_y: int = None,
+ scroll_x: int = None,
+ scroll_y: int = None,
+ x: int = None,
+ y: int = None,
+ pixels: int = None,
+ coordinate=None,
+ **kwargs,
+ ) -> dict:
+ """Scroll the page. Supports multiple formats:
+ - Legacy: scroll(delta_x, delta_y)
+ - Normalized: scroll(scroll_x=0, scroll_y=100, x=500, y=300)
+ - FARA: scroll(pixels=100, coordinate=[500, 300])
+ """
+ # Determine scroll amounts from various input formats
+ dx = scroll_x or delta_x or 0
+ dy = scroll_y or delta_y or (-(pixels or 0)) # pixels: positive=up, negative=down
+
+ result = await self.interface.playwright_exec("scroll", {"delta_x": dx, "delta_y": dy})
+ return result
async def web_search(self, query: str) -> dict:
"""Navigate to a Google search for the query."""
@@ -448,48 +492,69 @@ class BrowserTool(BaseComputerTool):
# These methods accept parameters in the format that FARA model outputs
# and agent.py passes via **action_args
- async def left_click(self, coordinate=None, **kwargs) -> dict:
- """Left click at coordinates. FARA-compatible."""
- return await self._action_left_click({"coordinate": coordinate})
+ async def left_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:
+ """Left click at coordinates. Supports coordinate array or x/y kwargs."""
+ # Accept either coordinate array or x/y kwargs
+ if coordinate and len(coordinate) >= 2:
+ x, y = coordinate[0], coordinate[1]
+ if x is None or y is None:
+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}
+ return await self._action_left_click({"coordinate": [x, y]})
- async def right_click(self, coordinate=None, **kwargs) -> dict:
- """Right click at coordinates. FARA-compatible."""
- if not coordinate or len(coordinate) != 2:
- return {"success": False, "error": "coordinate parameter [x, y] is required"}
- # Use playwright_exec for right click
- result = await self.interface.playwright_exec(
- "click", {"x": coordinate[0], "y": coordinate[1], "button": "right"}
- )
+ async def right_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:
+ """Right click at coordinates. Supports coordinate array or x/y kwargs."""
+ # Accept either coordinate array or x/y kwargs
+ if coordinate and len(coordinate) >= 2:
+ x, y = coordinate[0], coordinate[1]
+ if x is None or y is None:
+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}
+ result = await self.interface.playwright_exec("click", {"x": x, "y": y, "button": "right"})
return result
- async def middle_click(self, coordinate=None, **kwargs) -> dict:
- """Middle click at coordinates. FARA-compatible."""
- if not coordinate or len(coordinate) != 2:
- return {"success": False, "error": "coordinate parameter [x, y] is required"}
- result = await self.interface.playwright_exec(
- "click", {"x": coordinate[0], "y": coordinate[1], "button": "middle"}
- )
+ async def middle_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:
+ """Middle click at coordinates. Supports coordinate array or x/y kwargs."""
+ # Accept either coordinate array or x/y kwargs
+ if coordinate and len(coordinate) >= 2:
+ x, y = coordinate[0], coordinate[1]
+ if x is None or y is None:
+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}
+ result = await self.interface.playwright_exec("click", {"x": x, "y": y, "button": "middle"})
return result
- async def double_click(self, coordinate=None, **kwargs) -> dict:
- """Double click at coordinates. FARA-compatible."""
- if not coordinate or len(coordinate) != 2:
- return {"success": False, "error": "coordinate parameter [x, y] is required"}
- result = await self.interface.playwright_exec(
- "dblclick", {"x": coordinate[0], "y": coordinate[1]}
- )
+ async def double_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:
+ """Double click at coordinates. Supports coordinate array or x/y kwargs."""
+ # Accept either coordinate array or x/y kwargs
+ if coordinate and len(coordinate) >= 2:
+ x, y = coordinate[0], coordinate[1]
+ if x is None or y is None:
+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}
+ result = await self.interface.playwright_exec("dblclick", {"x": x, "y": y})
return result
- async def triple_click(self, coordinate=None, **kwargs) -> dict:
- """Triple click at coordinates. FARA-compatible."""
- if not coordinate or len(coordinate) != 2:
- return {"success": False, "error": "coordinate parameter [x, y] is required"}
+ async def triple_click(
+ self, coordinate=None, x: int = None, y: int = None, button: str = None, **kwargs
+ ) -> dict:
+ """Triple click at coordinates. Supports coordinate array or x/y kwargs."""
+ # Accept either coordinate array or x/y kwargs
+ if coordinate and len(coordinate) >= 2:
+ x, y = coordinate[0], coordinate[1]
+ if x is None or y is None:
+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}
# Triple click is approximated as double click
- return await self.double_click(coordinate=coordinate)
+ return await self.double_click(x=x, y=y)
- async def mouse_move(self, coordinate=None, **kwargs) -> dict:
- """Move mouse to coordinates. FARA-compatible."""
- return await self._action_mouse_move({"coordinate": coordinate})
+ async def mouse_move(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:
+ """Move mouse to coordinates. Supports coordinate array or x/y kwargs."""
+ # Accept either coordinate array or x/y kwargs
+ if coordinate and len(coordinate) >= 2:
+ x, y = coordinate[0], coordinate[1]
+ if x is None or y is None:
+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}
+ return await self._action_mouse_move({"coordinate": [x, y]})
+
+ async def move(self, x: int = None, y: int = None, **kwargs) -> dict:
+ """Move mouse to coordinates. Alias for mouse_move with x/y kwargs."""
+ return await self.mouse_move(x=x, y=y)
async def left_click_drag(
self, coordinate=None, start_coordinate=None, end_coordinate=None, **kwargs
@@ -518,6 +583,10 @@ class BrowserTool(BaseComputerTool):
"""Press keys. FARA-compatible."""
return await self._action_key({"keys": keys})
+ async def keypress(self, keys=None, **kwargs) -> dict:
+ """Press keys. Alias for key() - used by OperatorNormalizerCallback."""
+ return await self._action_key({"keys": keys})
+
async def hscroll(self, pixels=None, coordinate=None, **kwargs) -> dict:
"""Horizontal scroll. FARA-compatible."""
if pixels is None: