Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(fara): correct coordinate scaling and agent conversion loops (#934)
Sarina Li8759df254931
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.pyindex 4c40f6a30a..dbf11adcf6 100644--- a/libs/python/agent/agent/agent.py+++ b/libs/python/agent/agent/agent.py@@ -690,8 +690,6 @@ class ComputerAgent:# Extract action arguments (all fields except 'type')action_args = {k: v for k, v in action.items() if k != "type"}- # print(f"{action_type}({action_args})")-# Execute the computer actioncomputer_method = getattr(computer, action_type, None)action_result = Nonediff --git a/libs/python/agent/agent/loops/fara/config.py b/libs/python/agent/agent/loops/fara/config.pyindex df36bc6182..c31d48d05f 100644--- a/libs/python/agent/agent/loops/fara/config.py+++ b/libs/python/agent/agent/loops/fara/config.py@@ -14,20 +14,167 @@ from litellm.responses.litellm_completion_transformation.transformation import (from ...decorators import register_agentfrom ...loops.base import AsyncAgentConfigfrom ...responses import (- convert_completion_messages_to_responses_items,- convert_responses_items_to_completion_messages,+ make_click_item,+ make_double_click_item,+ make_drag_item,+ make_keypress_item,+ make_move_item,+ make_output_text_item,make_reasoning_item,+ make_screenshot_item,+ make_scroll_item,+ make_type_item,+ make_wait_item,)from ...types import AgentCapabilityfrom .helpers import (+ _convert_responses_items_to_fara_messages,build_nous_system,- convert_fara_args_to_browser_tool_format,- convert_qwen_tool_args_to_computer_action,parse_tool_call_from_text,- unnormalize_coordinate,)+def _scale_fara_coordinates(+ args: Dict[str, Any],+ original_dims: Tuple[int, int],+ resized_dims: Tuple[int, int],+) -> Dict[str, Any]:+ """+ Scale FARA coordinates from resized image space to original viewport space.++ FARA outputs pixel coordinates on the resized image (after smart_resize).+ This scales them back to the original browser viewport, matching FARA's+ convert_resized_coords_to_original() in fara_agent.py:+ scale_x = og_w / rsz_w+ return [coords[0] * scale_x, coords[1] * scale_y]++ Args:+ args: Action arguments containing "coordinate" key+ original_dims: (width, height) of original browser viewport+ resized_dims: (width, height) after smart_resize+ """+ coord = args.get("coordinate")+ if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:+ return args++ x, y = float(coord[0]), float(coord[1])+ original_w, original_h = float(original_dims[0]), float(original_dims[1])+ resized_w, resized_h = float(resized_dims[0]), float(resized_dims[1])++ # Scale from resized to original: x_final = x * (original / resized)+ scale_x = original_w / resized_w+ scale_y = original_h / resized_h++ x_scaled = max(0.0, min(original_w, x * scale_x))+ y_scaled = max(0.0, min(original_h, y * scale_y))++ return {**args, "coordinate": [round(x_scaled), round(y_scaled)]}+++def _fara_args_to_sdk_item(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:+ """+ Convert FARA model output args to SDK item using make_*_item helpers.++ FARA format: {"action": "left_click", "coordinate": [100, 200]}+ SDK format: ResponseComputerToolCallParam with action={"type": "click", "x": 100, "y": 200}+ """+ action = args.get("action", "")+ coordinate = args.get("coordinate", [0, 0])+ x = coordinate[0] if len(coordinate) > 0 else 0+ y = coordinate[1] if len(coordinate) > 1 else 0++ # Click actions+ if action in ("left_click", "click"):+ return make_click_item(x=x, y=y, button="left")+ if action == "right_click":+ return make_click_item(x=x, y=y, button="right")+ if action == "middle_click":+ return make_click_item(x=x, y=y, button="wheel")+ if action == "double_click":+ return make_double_click_item(x=x, y=y)++ # Type action+ if action == "type":+ return make_type_item(text=args.get("text", ""))++ # Key action+ if action in ("key", "keypress"):+ keys = args.get("keys", [])+ if isinstance(keys, str):+ keys = keys.split("+")+ return make_keypress_item(keys=keys)++ # Move action+ if action in ("mouse_move", "move"):+ return make_move_item(x=x, y=y)++ # Scroll action+ if action == "scroll":+ pixels = args.get("pixels") or 0 # Handle None explicitly+ # FARA: positive = up, negative = down+ scroll_y = -pixels # SDK: positive = down+ return make_scroll_item(x=x, y=y, scroll_x=0, scroll_y=scroll_y)++ if action == "hscroll":+ pixels = args.get("pixels") or 0 # Handle None explicitly+ return make_scroll_item(x=x, y=y, scroll_x=pixels, scroll_y=0)++ # Drag action+ if action == "left_click_drag":+ start_coord = args.get("start_coordinate", [0, 0])+ end_coord = args.get("end_coordinate", [0, 0])+ return make_drag_item(+ path=[+ {"x": start_coord[0], "y": start_coord[1]},+ {"x": end_coord[0], "y": end_coord[1]},+ ]+ )++ # Screenshot+ if action == "screenshot":+ return make_screenshot_item()++ # Wait+ if action == "wait":+ return make_wait_item()++ # Terminate - return None so no action is executed+ # The caller checks for terminate action and adds an assistant message to stop the loop+ if action == "terminate":+ return None++ # FARA browser-specific actions - create computer_call items directly+ # agent.py uses getattr(computer, action_type) to call these methods+ if action == "visit_url":+ return {+ "type": "computer_call",+ "call_id": f"call_{id(args)}",+ "action": {"type": "visit_url", "url": args.get("url", "")},+ "pending_safety_checks": [],+ "status": "completed",+ }++ if action == "web_search":+ return {+ "type": "computer_call",+ "call_id": f"call_{id(args)}",+ "action": {"type": "web_search", "query": args.get("query", "")},+ "pending_safety_checks": [],+ "status": "completed",+ }++ if action == "history_back":+ return {+ "type": "computer_call",+ "call_id": f"call_{id(args)}",+ "action": {"type": "history_back"},+ "pending_safety_checks": [],+ "status": "completed",+ }++ return None++@register_agent(models=r"(?i).*fara-7b.*", tool_type="browser")class FaraVlmConfig(AsyncAgentConfig):async def predict_step(@@ -75,10 +222,10 @@ class FaraVlmConfig(AsyncAgentConfig):"usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},}- # Build messages using NousFnCallPrompt system with tool schema in text- # Start with converted conversation (images/text preserved)- converted_msgs = convert_responses_items_to_completion_messages(- messages, allow_images_in_tool_results=False, use_xml_tools=True+ # Build messages using FARA's dedicated conversion layer+ # This converts SDK format to FARA's native format (action + coordinate)+ converted_msgs = _convert_responses_items_to_fara_messages(+ messages, allow_images_in_tool_results=False)# Build function schemas from tools array@@ -154,7 +301,9 @@ class FaraVlmConfig(AsyncAgentConfig):completion_messages.append(screenshot_msg)# Smart-resize all screenshots and attach min/max pixel hints. Fail fast if deps missing.- # Also record the last resized width/height to unnormalize coordinates later.+ # Track both original and resized dimensions for coordinate scaling.+ last_original_w: Optional[int] = None+ last_original_h: Optional[int] = Nonelast_rw: Optional[int] = Nonelast_rh: Optional[int] = NoneMIN_PIXELS = 3136@@ -189,6 +338,8 @@ class FaraVlmConfig(AsyncAgentConfig):# Attach hints on this image blockpart["min_pixels"] = MIN_PIXELSpart["max_pixels"] = MAX_PIXELS+ # Track both original and resized dimensions+ last_original_w, last_original_h = w, hlast_rw, last_rh = rw, rhapi_kwargs: Dict[str, Any] = {@@ -227,52 +378,55 @@ class FaraVlmConfig(AsyncAgentConfig):reasoning_text = message.get("reasoning") or ""output_items: List[Dict[str, Any]] = []+ has_terminate = False# Add reasoning if present (Ollama Cloud format)if reasoning_text:output_items.append(make_reasoning_item(reasoning_text))+ # Extract thoughts (text before <tool_call> tag)+ thoughts = ""+ if "<tool_call>" in content_text:+ thoughts = content_text.split("<tool_call>")[0].strip()++ # Add thoughts as assistant message if present+ if thoughts:+ output_items.append(make_output_text_item(thoughts))+# Priority 1: Try to parse tool call from content text (OpenRouter format)tool_call = parse_tool_call_from_text(content_text)if tool_call and isinstance(tool_call, dict):fn_name = tool_call.get("name") or "computer"raw_args = tool_call.get("arguments") or {}- # Unnormalize coordinates to actual screen size using last resized dims- if last_rw is None or last_rh is None:++ # Scale coordinates from resized image space to original viewport+ if (+ last_rw is None+ or last_rh is None+ or last_original_w is None+ or last_original_h is None+ ):raise RuntimeError(- "No screenshots found to derive dimensions for coordinate unnormalization."+ "No screenshots found to derive dimensions for coordinate scaling.")- args = await unnormalize_coordinate(raw_args, (last_rw, last_rh))-- # Convert FARA model output format to BrowserTool compatible format- args = convert_fara_args_to_browser_tool_format(args)+ args = _scale_fara_coordinates(+ raw_args,+ original_dims=(last_original_w, last_original_h),+ resized_dims=(last_rw, last_rh),+ )- # Extract thoughts (text before <tool_call> tag)- thoughts = ""- if "<tool_call>" in content_text:- thoughts = content_text.split("<tool_call>")[0].strip()+ # Convert FARA output to SDK format using make_*_item helpers+ if fn_name in ("computer", "computer_use"):+ item = _fara_args_to_sdk_item(args)+ if item:+ output_items.append(item)+ # Check for terminate (even if item is None)+ if args.get("action") == "terminate":+ has_terminate = True- # Build an OpenAI-style tool call so we can reuse the converter- fake_cm = {- "role": "assistant",- "content": thoughts, # Preserve thoughts before tool call- "tool_calls": [- {- "type": "function",- "id": "call_0",- "function": {- "name": fn_name,- "arguments": json.dumps(args),- },- }- ],- }- output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))elif tool_calls_array:# Priority 2: Use tool_calls field if present (Ollama Cloud format)- # Process and unnormalize coordinates in tool calls- processed_tool_calls = []for tc in tool_calls_array:function = tc.get("function", {})fn_name = function.get("name", "computer")@@ -281,56 +435,30 @@ class FaraVlmConfig(AsyncAgentConfig):try:args = json.loads(args_str)- # Unnormalize coordinates if present+ # Scale coordinates from resized image space to original viewportif "coordinate" in args and last_rw is not None and last_rh is not None:- args = await unnormalize_coordinate(args, (last_rw, last_rh))-- # Convert FARA model output format to BrowserTool compatible format+ if last_original_w is not None and last_original_h is not None:+ args = _scale_fara_coordinates(+ args,+ original_dims=(last_original_w, last_original_h),+ resized_dims=(last_rw, last_rh),+ )++ # Convert FARA output to SDK formatif fn_name in ("computer", "computer_use"):- args = convert_fara_args_to_browser_tool_format(args)-- processed_tool_calls.append(- {- "type": tc.get("type", "function"),- "id": tc.get("id", "call_0"),- "function": {- "name": fn_name,- "arguments": json.dumps(args),- },- }- )+ item = _fara_args_to_sdk_item(args)+ if item:+ output_items.append(item)+ # Check for terminate (even if item is None)+ if args.get("action") == "terminate":+ has_terminate = Trueexcept json.JSONDecodeError:- # Keep original if parsing fails- processed_tool_calls.append(tc)-- fake_cm = {- "role": "assistant",- "content": content_text if content_text else "",- "tool_calls": processed_tool_calls,- }- output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))- else:- # No tool calls found in either format, return text response- fake_cm = {"role": "assistant", "content": content_text}- output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))-- # Check if this is a terminate action - if so, add a final assistant message to stop the loop- has_terminate = False- for item in output_items:- if item.get("type") == "computer_call":- action = item.get("action", {})- if action.get("type") == "terminate":- has_terminate = True- break- elif item.get("type") == "function_call":- try:- args = json.loads(item.get("arguments", "{}"))- if args.get("action") == "terminate":- has_terminate = True- break- except:pass+ elif content_text:+ # No tool calls found, return text response+ output_items.append(make_output_text_item(content_text))+# If terminate detected, ensure LAST item is an assistant message to exit the loop# The generic agent loop checks: while new_items[-1].get("role") != "assistant"if has_terminate:@@ -434,7 +562,12 @@ class FaraVlmConfig(AsyncAgentConfig):content_text = ((choice.get("message") or {}).get("content")) or ""tool_call = parse_tool_call_from_text(content_text) or {}args = tool_call.get("arguments") or {}- args = await unnormalize_coordinate(args, (rh, rw))+ # Scale from resized image space to original viewport+ args = _scale_fara_coordinates(+ args,+ original_dims=(w, h),+ resized_dims=(rw, rh),+ )coord = args.get("coordinate")if isinstance(coord, (list, tuple)) and len(coord) >= 2:return int(coord[0]), int(coord[1])@@ -443,10 +576,11 @@ class FaraVlmConfig(AsyncAgentConfig):# FARA-specific ComputerUse tool schema (OpenAI function tool format)# This schema is tailored for FARA-7B model and includes browser-specific actions+# NOTE: Tool name MUST be "computer_use" to match what FARA-7B was trained onFARA_COMPUTER_TOOL: dict[str, Any] = {"type": "function","function": {- "name": "computer",+ "name": "computer_use","description": ("Use a mouse and keyboard to interact with a computer, and take screenshots.\n""* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n"diff --git a/libs/python/agent/agent/loops/fara/helpers.py b/libs/python/agent/agent/loops/fara/helpers.pyindex 61a3996d7f..99bc22a5d7 100644--- a/libs/python/agent/agent/loops/fara/helpers.py+++ b/libs/python/agent/agent/loops/fara/helpers.py@@ -225,6 +225,103 @@ def build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Anyreturn {"role": "system", "content": content}+def fix_fara_tool_call_format(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:+ """+ Fix tool call format in conversation history for FARA compatibility.++ The shared `convert_responses_items_to_completion_messages` function outputs:+ - Tool name as "computer" (should be "computer_use")+ - Action key as "type" (should be "action")++ This function post-processes assistant messages to fix these issues.+ """+ import re++ # Valid FARA action types+ valid_actions = {+ "left_click",+ "right_click",+ "middle_click",+ "double_click",+ "triple_click",+ "click",+ "type",+ "key",+ "scroll",+ "hscroll",+ "mouse_move",+ "wait",+ "visit_url",+ "web_search",+ "history_back",+ "screenshot",+ "terminate",+ }++ fixed_messages = []+ for msg in messages:+ if msg.get("role") != "assistant":+ fixed_messages.append(msg)+ continue++ content = msg.get("content", "")+ if not isinstance(content, str) or "<tool_call>" not in content:+ fixed_messages.append(msg)+ continue++ # Find and fix all tool calls in the content+ def fix_tool_call(match):+ tool_call_content = match.group(1)+ try:+ tool_call = json.loads(tool_call_content)++ # Fix tool name: "computer" -> "computer_use"+ if tool_call.get("name") == "computer":+ tool_call["name"] = "computer_use"++ # Fix arguments: "type" -> "action" and x/y -> coordinate+ args = tool_call.get("arguments", {})+ if isinstance(args, dict):+ # If "type" contains a valid action, rename to "action"+ if "type" in args and args["type"] in valid_actions:+ args["action"] = args.pop("type")++ # Convert internal x/y format back to FARA coordinate format+ if "x" in args and "y" in args and "coordinate" not in args:+ args["coordinate"] = [args.pop("x"), args.pop("y")]++ # Normalize action names: "click" -> "left_click"+ if args.get("action") == "click":+ args["action"] = "left_click"++ # Remove "button" field - FARA doesn't use it (action name implies button)+ args.pop("button", None)++ # If "action" is empty but we can infer from other keys+ if args.get("action") == "" and "coordinate" in args:+ args["action"] = "left_click"++ tool_call["arguments"] = args++ return f"<tool_call>\n{json.dumps(tool_call)}\n</tool_call>"+ except (json.JSONDecodeError, TypeError):+ return match.group(0) # Return original if parsing fails++ # Match <tool_call>...</tool_call> or <tool_call>...</tool_call>+ fixed_content = re.sub(+ r"<tool_call>\s*(\{.*?\})\s*</tool_call>", fix_tool_call, content, flags=re.DOTALL+ )++ # Also handle malformed closing tags like <tool_call> used as closing+ fixed_content = re.sub(+ r"<tool_call>(\{.*?\})<tool_call>", fix_tool_call, fixed_content, flags=re.DOTALL+ )++ fixed_messages.append({**msg, "content": fixed_content})++ return fixed_messages++def parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:"""Extract JSON object within <tool_call>...</tool_call> from model text.@@ -482,3 +579,239 @@ def convert_fara_args_to_browser_tool_format(args: Dict[str, Any]) -> Dict[str,# For any other action, return cleaned args (just action + known fields)return result+++def _convert_responses_items_to_fara_messages(+ messages: List[Dict[str, Any]],+ allow_images_in_tool_results: bool = False,+) -> List[Dict[str, Any]]:+ """+ Convert SDK responses_items format to FARA-compatible completion messages.++ This is FARA's dedicated conversion layer (similar to Anthropic's pattern).+ It handles the conversion from SDK's OpenAI-style format to FARA's native format:++ SDK format:+ {"type": "click", "x": 100, "y": 200, "button": "left"}++ FARA format (in XML tool_call):+ {"name": "computer_use", "arguments": {"action": "left_click", "coordinate": [100, 200]}}+ """+ completion_messages: List[Dict[str, Any]] = []++ for message in messages:+ msg_type = message.get("type")+ role = message.get("role")++ # Handle user messages+ if role == "user" or msg_type == "user":+ content = message.get("content", "")+ if isinstance(content, list):+ converted_content = []+ for item in content:+ if isinstance(item, dict):+ item_type = item.get("type")+ if item_type == "input_image":+ image_url = item.get("image_url", "")+ if image_url and image_url != "[omitted]":+ converted_content.append(+ {"type": "image_url", "image_url": {"url": image_url}}+ )+ elif item_type == "input_text":+ converted_content.append({"type": "text", "text": item.get("text", "")})+ elif item_type == "image_url":+ # Already in correct format+ converted_content.append(item)+ elif item_type == "text":+ converted_content.append(item)+ else:+ converted_content.append(item)+ else:+ converted_content.append({"type": "text", "text": str(item)})+ completion_messages.append({"role": "user", "content": converted_content})+ else:+ completion_messages.append({"role": "user", "content": content})++ # Handle assistant messages+ elif role == "assistant" and msg_type == "message":+ content = message.get("content", [])+ if isinstance(content, str):+ completion_messages.append({"role": "assistant", "content": content})+ elif isinstance(content, list):+ text_parts = []+ for item in content:+ if isinstance(item, dict) and item.get("type") == "output_text":+ text_parts.append(item.get("text", ""))+ completion_messages.append({"role": "assistant", "content": "\n".join(text_parts)})++ # Handle reasoning+ elif msg_type == "reasoning":+ summary = message.get("summary", [])+ reasoning_text = ""+ if isinstance(summary, list) and summary:+ for item in summary:+ if isinstance(item, dict) and item.get("type") == "summary_text":+ reasoning_text = item.get("text", "")+ break+ if reasoning_text:+ completion_messages.append({"role": "assistant", "content": reasoning_text})++ # Handle computer_call - convert SDK format to FARA's XML tool_call format+ elif msg_type == "computer_call":+ action = message.get("action", {})+ action_type = action.get("type")++ # Convert SDK action to FARA format+ fara_args = _sdk_action_to_fara_args(action)++ # Build FARA's XML tool_call format+ tool_call_json = json.dumps({"name": "computer_use", "arguments": fara_args})+ tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"++ # Append to last assistant message or create new one+ if completion_messages and completion_messages[-1].get("role") == "assistant":+ prev_content = completion_messages[-1].get("content", "")+ completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()+ else:+ completion_messages.append({"role": "assistant", "content": tool_call_text})++ # Handle computer_call_output - convert to FARA's tool_response format+ elif msg_type == "computer_call_output":+ output = message.get("output", {})++ # Build response content+ if isinstance(output, dict) and output.get("type") == "input_image":+ image_url = output.get("image_url", "")+ response_text = "<tool_response>\nAction executed successfully. Here is the next screenshot.\n</tool_response>"++ # Add as user message with image+ if allow_images_in_tool_results and image_url and image_url != "[omitted]":+ completion_messages.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": response_text},+ {"type": "image_url", "image_url": {"url": image_url}},+ ],+ }+ )+ else:+ completion_messages.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": response_text},+ ],+ }+ )+ elif isinstance(output, dict) and output.get("terminated"):+ response_text = "<tool_response>\nTask terminated.\n</tool_response>"+ completion_messages.append({"role": "user", "content": response_text})+ else:+ response_text = f"<tool_response>\n{json.dumps(output) if isinstance(output, dict) else str(output)}\n</tool_response>"+ completion_messages.append({"role": "user", "content": response_text})++ # Handle function_call (non-computer tools)+ elif msg_type == "function_call":+ fn_name = message.get("name", "")+ fn_args = message.get("arguments", "{}")++ tool_call_json = json.dumps(+ {+ "name": fn_name,+ "arguments": json.loads(fn_args) if isinstance(fn_args, str) else fn_args,+ }+ )+ tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"++ if completion_messages and completion_messages[-1].get("role") == "assistant":+ prev_content = completion_messages[-1].get("content", "")+ completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()+ else:+ completion_messages.append({"role": "assistant", "content": tool_call_text})++ # Handle function_call_output+ elif msg_type == "function_call_output":+ output = message.get("output", "")+ response_text = f"<tool_response>\n{output}\n</tool_response>"+ completion_messages.append({"role": "user", "content": response_text})++ return completion_messages+++def _sdk_action_to_fara_args(action: Dict[str, Any]) -> Dict[str, Any]:+ """+ Convert SDK action format to FARA arguments format.++ SDK format: {"type": "click", "x": 100, "y": 200, "button": "left"}+ FARA format: {"action": "left_click", "coordinate": [100, 200]}+ """+ action_type = action.get("type", "")++ # Click actions+ if action_type == "click":+ button = action.get("button", "left")+ action_name = {+ "left": "left_click",+ "right": "right_click",+ "wheel": "middle_click",+ "middle": "middle_click",+ }.get(button, "left_click")+ return {"action": action_name, "coordinate": [action.get("x", 0), action.get("y", 0)]}++ if action_type == "double_click":+ return {"action": "double_click", "coordinate": [action.get("x", 0), action.get("y", 0)]}++ # Type action+ if action_type == "type":+ result = {"action": "type", "text": action.get("text", "")}+ # Include coordinate if present (for click-then-type)+ if "x" in action and "y" in action:+ result["coordinate"] = [action.get("x", 0), action.get("y", 0)]+ return result++ # Keypress action+ if action_type == "keypress":+ keys = action.get("keys", [])+ return {"action": "key", "keys": keys}++ # Move action+ if action_type in ("move", "mouse_move"):+ return {"action": "mouse_move", "coordinate": [action.get("x", 0), action.get("y", 0)]}++ # Scroll action+ if action_type == "scroll":+ scroll_x = action.get("scroll_x", 0)+ scroll_y = action.get("scroll_y", 0)+ # FARA uses pixels (positive = up/left, negative = down/right)+ pixels = scroll_y if scroll_y != 0 else scroll_x+ result = {"action": "scroll", "pixels": pixels}+ if "x" in action and "y" in action:+ result["coordinate"] = [action.get("x", 0), action.get("y", 0)]+ return result++ # Drag action+ if action_type == "drag":+ path = action.get("path", [])+ if len(path) >= 2:+ return {+ "action": "left_click_drag",+ "start_coordinate": [path[0].get("x", 0), path[0].get("y", 0)],+ "end_coordinate": [path[-1].get("x", 0), path[-1].get("y", 0)],+ }+ return {"action": "left_click_drag"}++ # Screenshot+ if action_type == "screenshot":+ return {"action": "screenshot"}++ # Wait+ if action_type == "wait":+ return {"action": "wait"}++ # Terminate+ if action_type == "terminate":+ return {"action": "terminate", "status": action.get("status", "success")}++ # Fallback - return as-is with type renamed to action+ return {"action": action_type, **{k: v for k, v in action.items() if k != "type"}}diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.pyindex ccfc5c00f5..6163579585 100644--- a/libs/python/agent/agent/tools/browser_tool.py+++ b/libs/python/agent/agent/tools/browser_tool.py@@ -337,8 +337,9 @@ class BrowserTool(BaseComputerTool):async def _action_scroll(self, params: dict) -> dict:"""Scroll the page."""- pixels = params.get("pixels", 0)- if pixels == 0:+ pixels = params.get("pixels")+ # Handle None explicitly - default to 0 means "no scroll requested"+ if pixels is None or pixels == 0:return {"success": False, "error": "pixels parameter is required"}# Positive = up (negative delta_y), Negative = down (positive delta_y)@@ -387,7 +388,10 @@ class BrowserTool(BaseComputerTool):async def _action_wait(self, params: dict) -> dict:"""Wait for specified seconds."""- time = params.get("time", 0)+ time = params.get("time")+ # Handle None or missing time - default to 3 seconds (matches FARA behavior)+ if time is None:+ time = 3if time <= 0:return {"success": False, "error": "time parameter must be positive"}@@ -396,7 +400,10 @@ class BrowserTool(BaseComputerTool):async def _action_terminate(self, params: dict) -> dict:"""Terminate and report status."""- status = params.get("status", "success")+ status = params.get("status")+ # Handle None or missing status - default to "success"+ if status is None:+ status = "success"message = f"Task terminated with status: {status}"if self._facts:@@ -409,17 +416,54 @@ class BrowserTool(BaseComputerTool):"""Navigate to a URL."""return await self._action_visit_url({"url": url})- async def click(self, x: int, y: int) -> dict:- """Click at coordinates."""- return await self._action_left_click({"coordinate": [x, y]})+ async def click(self, x: int = None, y: int = None, button: str = "left", **kwargs) -> dict:+ """Click at coordinates. Supports both positional (x, y) and kwargs (button, x, y).++ This is compatible with the normalized format from OperatorNormalizerCallback+ which transforms actions like {"type": "left_click", "coordinate": [x, y]}+ into {"type": "click", "button": "left", "x": x, "y": y}.+ """+ if x is None or y is None:+ return {"success": False, "error": "x and y coordinates are required"}+ if button == "right":+ return await self.interface.playwright_exec(+ "click", {"x": x, "y": y, "button": "right"}+ )+ elif button == "middle" or button == "wheel":+ return await self.interface.playwright_exec(+ "click", {"x": x, "y": y, "button": "middle"}+ )+ else:+ # Default to left click+ return await self._action_left_click({"coordinate": [x, y]})async def type(self, text: str) -> dict:"""Type text into the focused element."""return await self._action_type({"text": text})- async def scroll(self, delta_x: int, delta_y: int) -> dict:- """Scroll the page."""- return await self._action_scroll({"pixels": -delta_y})+ async def scroll(+ self,+ delta_x: int = None,+ delta_y: int = None,+ scroll_x: int = None,+ scroll_y: int = None,+ x: int = None,+ y: int = None,+ pixels: int = None,+ coordinate=None,+ **kwargs,+ ) -> dict:+ """Scroll the page. Supports multiple formats:+ - Legacy: scroll(delta_x, delta_y)+ - Normalized: scroll(scroll_x=0, scroll_y=100, x=500, y=300)+ - FARA: scroll(pixels=100, coordinate=[500, 300])+ """+ # Determine scroll amounts from various input formats+ dx = scroll_x or delta_x or 0+ dy = scroll_y or delta_y or (-(pixels or 0)) # pixels: positive=up, negative=down++ result = await self.interface.playwright_exec("scroll", {"delta_x": dx, "delta_y": dy})+ return resultasync def web_search(self, query: str) -> dict:"""Navigate to a Google search for the query."""@@ -448,48 +492,69 @@ class BrowserTool(BaseComputerTool):# These methods accept parameters in the format that FARA model outputs# and agent.py passes via **action_args- async def left_click(self, coordinate=None, **kwargs) -> dict:- """Left click at coordinates. FARA-compatible."""- return await self._action_left_click({"coordinate": coordinate})+ async def left_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:+ """Left click at coordinates. Supports coordinate array or x/y kwargs."""+ # Accept either coordinate array or x/y kwargs+ if coordinate and len(coordinate) >= 2:+ x, y = coordinate[0], coordinate[1]+ if x is None or y is None:+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}+ return await self._action_left_click({"coordinate": [x, y]})- async def right_click(self, coordinate=None, **kwargs) -> dict:- """Right click at coordinates. FARA-compatible."""- if not coordinate or len(coordinate) != 2:- return {"success": False, "error": "coordinate parameter [x, y] is required"}- # Use playwright_exec for right click- result = await self.interface.playwright_exec(- "click", {"x": coordinate[0], "y": coordinate[1], "button": "right"}- )+ async def right_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:+ """Right click at coordinates. Supports coordinate array or x/y kwargs."""+ # Accept either coordinate array or x/y kwargs+ if coordinate and len(coordinate) >= 2:+ x, y = coordinate[0], coordinate[1]+ if x is None or y is None:+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}+ result = await self.interface.playwright_exec("click", {"x": x, "y": y, "button": "right"})return result- async def middle_click(self, coordinate=None, **kwargs) -> dict:- """Middle click at coordinates. FARA-compatible."""- if not coordinate or len(coordinate) != 2:- return {"success": False, "error": "coordinate parameter [x, y] is required"}- result = await self.interface.playwright_exec(- "click", {"x": coordinate[0], "y": coordinate[1], "button": "middle"}- )+ async def middle_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:+ """Middle click at coordinates. Supports coordinate array or x/y kwargs."""+ # Accept either coordinate array or x/y kwargs+ if coordinate and len(coordinate) >= 2:+ x, y = coordinate[0], coordinate[1]+ if x is None or y is None:+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}+ result = await self.interface.playwright_exec("click", {"x": x, "y": y, "button": "middle"})return result- async def double_click(self, coordinate=None, **kwargs) -> dict:- """Double click at coordinates. FARA-compatible."""- if not coordinate or len(coordinate) != 2:- return {"success": False, "error": "coordinate parameter [x, y] is required"}- result = await self.interface.playwright_exec(- "dblclick", {"x": coordinate[0], "y": coordinate[1]}- )+ async def double_click(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:+ """Double click at coordinates. Supports coordinate array or x/y kwargs."""+ # Accept either coordinate array or x/y kwargs+ if coordinate and len(coordinate) >= 2:+ x, y = coordinate[0], coordinate[1]+ if x is None or y is None:+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}+ result = await self.interface.playwright_exec("dblclick", {"x": x, "y": y})return result- async def triple_click(self, coordinate=None, **kwargs) -> dict:- """Triple click at coordinates. FARA-compatible."""- if not coordinate or len(coordinate) != 2:- return {"success": False, "error": "coordinate parameter [x, y] is required"}+ async def triple_click(+ self, coordinate=None, x: int = None, y: int = None, button: str = None, **kwargs+ ) -> dict:+ """Triple click at coordinates. Supports coordinate array or x/y kwargs."""+ # Accept either coordinate array or x/y kwargs+ if coordinate and len(coordinate) >= 2:+ x, y = coordinate[0], coordinate[1]+ if x is None or y is None:+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}# Triple click is approximated as double click- return await self.double_click(coordinate=coordinate)+ return await self.double_click(x=x, y=y)- async def mouse_move(self, coordinate=None, **kwargs) -> dict:- """Move mouse to coordinates. FARA-compatible."""- return await self._action_mouse_move({"coordinate": coordinate})+ async def mouse_move(self, coordinate=None, x: int = None, y: int = None, **kwargs) -> dict:+ """Move mouse to coordinates. Supports coordinate array or x/y kwargs."""+ # Accept either coordinate array or x/y kwargs+ if coordinate and len(coordinate) >= 2:+ x, y = coordinate[0], coordinate[1]+ if x is None or y is None:+ return {"success": False, "error": "coordinate parameter [x, y] or x/y kwargs required"}+ return await self._action_mouse_move({"coordinate": [x, y]})++ async def move(self, x: int = None, y: int = None, **kwargs) -> dict:+ """Move mouse to coordinates. Alias for mouse_move with x/y kwargs."""+ return await self.mouse_move(x=x, y=y)async def left_click_drag(self, coordinate=None, start_coordinate=None, end_coordinate=None, **kwargs@@ -518,6 +583,10 @@ class BrowserTool(BaseComputerTool):"""Press keys. FARA-compatible."""return await self._action_key({"keys": keys})+ async def keypress(self, keys=None, **kwargs) -> dict:+ """Press keys. Alias for key() - used by OperatorNormalizerCallback."""+ return await self._action_key({"keys": keys})+async def hscroll(self, pixels=None, coordinate=None, **kwargs) -> dict:"""Horizontal scroll. FARA-compatible."""if pixels is None: