Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(agent): Add GPT 5.4 support for Computer tool (#1150)

Sarina Liff38603826b4
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.py
index 2d790d2ed2..40014da478 100644
--- a/libs/python/agent/agent/agent.py
+++ b/libs/python/agent/agent/agent.py
@@ -782,6 +782,112 @@ class ComputerAgent:
if item_type == "function_call":
await self._on_function_call_start(item)
+
+ # Special handling for "computer" function calls (from GPT 5.4 etc)
+ # These need to be treated like computer_call items
+ if item.get("name") == "computer" and computer:
+ args = json.loads(item.get("arguments", "{}"))
+ action_type = args.get("action")
+ if not action_type:
+ raise ToolError("Computer function call missing 'action' argument")
+
+ # Map action types to their relevant parameters
+ action_param_map = {
+ "screenshot": [], # No parameters
+ "click": ["x", "y", "button"],
+ "double_click": ["x", "y"],
+ "right_click": ["x", "y"],
+ "type": ["text"],
+ "keypress": ["keys"],
+ "scroll": ["x", "y", "scroll_x", "scroll_y"],
+ "move": ["x", "y"],
+ "drag": ["start_x", "start_y", "end_x", "end_y"],
+ "wait": ["seconds", "ms"],
+ "terminate": ["status"],
+ }
+
+ # Extract only relevant action arguments, filtering out empty values
+ relevant_params = action_param_map.get(action_type, [])
+ action_args = {}
+ for k, v in args.items():
+ if k == "action":
+ continue
+ # Include if it's a relevant param OR if param map doesn't have this action
+ if k in relevant_params or action_type not in action_param_map:
+ # Filter out empty/default values but allow numeric zero (valid for coordinates/scroll)
+ if v is not None and v != "" and v != []:
+ action_args[k] = v
+
+ # Execute the computer action
+ computer_method = getattr(computer, action_type, None)
+ action_result = None
+ if computer_method:
+ action_result = await computer_method(**action_args)
+ else:
+ raise ToolError(f"Unknown computer action: {action_type}")
+
+ # Track computer action execution
+ if self.telemetry_enabled and is_telemetry_enabled():
+ record_event(
+ "computer_action_executed",
+ {"action_type": action_type},
+ )
+ record_event(
+ "agent_tool_executed",
+ {"tool_type": "computer", "tool_name": action_type},
+ )
+
+ # Check if this was a terminate action
+ is_terminate = action_type == "terminate" or (
+ isinstance(action_result, dict) and action_result.get("terminated")
+ )
+
+ # Take screenshot after action (skip for terminate)
+ if not is_terminate:
+ if self.screenshot_delay and self.screenshot_delay > 0:
+ await asyncio.sleep(self.screenshot_delay)
+ screenshot_base64 = await computer.screenshot()
+ await self._on_screenshot(screenshot_base64, "screenshot_after")
+
+ # Create function call output for GPT 5.4 etc
+ # Note: function_call_output only supports text, so we include the
+ # screenshot as a separate user message with image content
+ if is_terminate:
+ output_content = json.dumps(action_result if action_result else {"terminated": True})
+ call_output = {
+ "type": "function_call_output",
+ "call_id": item.get("call_id"),
+ "output": output_content,
+ }
+ result = [call_output]
+ else:
+ # Return both the function output AND a user message with the screenshot
+ # This allows the model to see the screenshot result
+ # Preserve actual action_result so failures/errors aren't masked
+ if action_result is not None:
+ output_content = json.dumps(action_result)
+ else:
+ output_content = json.dumps({"success": True, "screenshot_captured": True})
+ call_output = {
+ "type": "function_call_output",
+ "call_id": item.get("call_id"),
+ "output": output_content,
+ }
+ # Include screenshot as a user message with image content
+ image_message = {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_image",
+ "image_url": f"data:image/png;base64,{screenshot_base64}",
+ }
+ ],
+ }
+ result = [call_output, image_message]
+
+ await self._on_function_call_end(item, result)
+ return result
+
# Perform function call
function = self._get_tool(item.get("name"))
if not function:
diff --git a/libs/python/agent/agent/callbacks/trajectory_saver.py b/libs/python/agent/agent/callbacks/trajectory_saver.py
index c73276bf0e..81627882f7 100644
--- a/libs/python/agent/agent/callbacks/trajectory_saver.py
+++ b/libs/python/agent/agent/callbacks/trajectory_saver.py
@@ -51,13 +51,14 @@ def extract_computer_call_outputs(
items: List[Dict[str, Any]], screenshot_dir: Optional[Path]
) -> List[Dict[str, Any]]:
"""
- Save any base64-encoded screenshots from computer_call_output entries to files and
- replace their image_url with the saved file path when a call_id is present.
+ Save any base64-encoded screenshots from computer_call_output or function_call_output
+ entries to files and replace their image_url with the saved file path when a call_id is present.
Only operates if screenshot_dir is provided and exists; otherwise returns items unchanged.
Args:
- items: List of message/result dicts potentially containing computer_call_output entries
+ items: List of message/result dicts potentially containing computer_call_output
+ or function_call_output entries
screenshot_dir: Directory to write screenshots into
Returns:
@@ -101,6 +102,94 @@ def extract_computer_call_outputs(
new_output = dict(output)
new_output["image_url"] = str(out_path)
msg["output"] = new_output
+
+ elif msg.get("type") == "function_call_output":
+ # Handle function_call_output from GPT 5.4 / BrowserTool
+ call_id = msg.get("call_id")
+ output = msg.get("output", "")
+
+ # Parse output if it's a string
+ if isinstance(output, str):
+ try:
+ output_dict = json.loads(output)
+ except (json.JSONDecodeError, TypeError):
+ output_dict = None
+ else:
+ output_dict = output
+
+ if isinstance(output_dict, dict) and call_id:
+ image_data = None
+ image_key = None
+
+ # Format 1: {"type": "input_image", "image_url": "data:image/png;base64,..."}
+ if output_dict.get("type") == "input_image":
+ image_url = output_dict.get("image_url", "")
+ if isinstance(image_url, str) and image_url.startswith("data:"):
+ image_data = image_url.split(",", 1)[1] if "," in image_url else None
+ image_key = "image_url"
+
+ # Format 2: {"success": True, "screenshot": "base64data"}
+ elif output_dict.get("screenshot"):
+ image_data = output_dict.get("screenshot")
+ image_key = "screenshot"
+
+ if image_data and image_key:
+ out_path = screenshot_dir / f"{call_id}.png"
+ if not out_path.exists():
+ try:
+ img_bytes = base64.b64decode(image_data)
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(out_path, "wb") as f:
+ f.write(img_bytes)
+ except Exception:
+ pass
+ # Update output to reference file path
+ new_output_dict = dict(output_dict)
+ new_output_dict[image_key] = str(out_path)
+ msg["output"] = json.dumps(new_output_dict)
+
+ elif msg.get("role") == "user":
+ # Handle user messages with input_image content (GPT-5.4 sibling screenshot messages)
+ # These accompany function_call_output for computer calls
+ content = msg.get("content", [])
+ if isinstance(content, list):
+ new_content = []
+ content_modified = False
+ for content_item in content:
+ if isinstance(content_item, dict) and content_item.get("type") == "input_image":
+ image_url = content_item.get("image_url", "")
+ if isinstance(image_url, str) and image_url.startswith("data:"):
+ # Generate a unique ID for this screenshot
+ screenshot_id = str(uuid.uuid4())[:8]
+ try:
+ ext = image_url.split(";", 1)[0].split("/")[-1]
+ if not ext:
+ ext = "png"
+ except Exception:
+ ext = "png"
+ out_path = screenshot_dir / f"user_screenshot_{screenshot_id}.{ext}"
+ if not out_path.exists():
+ try:
+ b64_payload = image_url.split(",", 1)[1]
+ img_bytes = base64.b64decode(b64_payload)
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(out_path, "wb") as f:
+ f.write(img_bytes)
+ except Exception:
+ new_content.append(content_item)
+ continue
+ # Update image_url to file path
+ new_item = dict(content_item)
+ new_item["image_url"] = str(out_path)
+ new_content.append(new_item)
+ content_modified = True
+ else:
+ new_content.append(content_item)
+ else:
+ new_content.append(content_item)
+ if content_modified:
+ msg["content"] = new_content
+
except Exception:
# do not block on malformed entries; keep original
pass
@@ -420,3 +509,130 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
# Increment turn counter
self.current_turn += 1
+
+ @override
+ async def on_function_call_end(
+ self, item: Dict[str, Any], result: List[Dict[str, Any]]
+ ) -> None:
+ """
+ Called when a function call has completed.
+ Saves screenshots and function call output for GPT 5.4 / BrowserTool.
+ """
+ if not self.trajectory_id:
+ return
+
+ self._save_artifact("function_call_result", {"item": item, "result": result})
+
+ # Extract coordinates from function call arguments if present
+ x_coord, y_coord = None, None
+ try:
+ arguments = item.get("arguments", "{}")
+ if isinstance(arguments, str):
+ args_dict = json.loads(arguments)
+ else:
+ args_dict = arguments
+
+ # Check for coordinate array format (BrowserTool style)
+ coord = args_dict.get("coordinate")
+ if coord and isinstance(coord, list) and len(coord) >= 2:
+ x_coord, y_coord = coord[0], coord[1]
+ # Check for x/y format (computer_use style)
+ elif "x" in args_dict and "y" in args_dict:
+ x_coord, y_coord = args_dict.get("x"), args_dict.get("y")
+ except (json.JSONDecodeError, TypeError):
+ pass
+
+ # Look for screenshot in the result
+ screenshot_found = False
+ for result_item in result:
+ if screenshot_found:
+ break
+
+ if result_item.get("type") == "function_call_output":
+ output = result_item.get("output", "")
+
+ # Parse output if it's a string
+ if isinstance(output, str):
+ try:
+ output_dict = json.loads(output)
+ except (json.JSONDecodeError, TypeError):
+ # Try to evaluate as Python literal (for stringified dicts)
+ try:
+ import ast
+ output_dict = ast.literal_eval(output)
+ except (ValueError, SyntaxError):
+ continue
+ else:
+ output_dict = output
+
+ if not isinstance(output_dict, dict):
+ continue
+
+ # Extract screenshot from various formats
+ image_data = None
+
+ # Format 1: {"type": "input_image", "image_url": "data:image/png;base64,..."}
+ if output_dict.get("type") == "input_image":
+ image_url = output_dict.get("image_url", "")
+ if image_url.startswith("data:image/"):
+ image_data = image_url.split(",", 1)[1]
+ elif image_url:
+ image_data = image_url
+
+ # Format 2: {"success": True, "screenshot": "base64data"}
+ elif output_dict.get("screenshot"):
+ image_data = output_dict.get("screenshot")
+
+ if image_data:
+ try:
+ # Decode the image
+ image_bytes = base64.b64decode(image_data)
+
+ # If we have coordinates, draw crosshair annotation
+ if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:
+ annotated_image = self._draw_crosshair_on_image(
+ image_bytes, int(x_coord), int(y_coord)
+ )
+ self._save_artifact("screenshot_action", annotated_image)
+ else:
+ # Save plain screenshot without crosshair
+ self._save_artifact("screenshot", image_bytes)
+
+ screenshot_found = True
+
+ except Exception as e:
+ # If processing fails, just log and continue
+ print(f"Failed to process screenshot from function call: {e}")
+
+ # Handle sibling user messages with input_image content (GPT-5.4 computer calls)
+ # These accompany function_call_output and contain the actual screenshot
+ elif result_item.get("role") == "user":
+ content = result_item.get("content", [])
+ if isinstance(content, list):
+ for content_item in content:
+ if isinstance(content_item, dict) and content_item.get("type") == "input_image":
+ image_url = content_item.get("image_url", "")
+ if isinstance(image_url, str) and image_url.startswith("data:"):
+ try:
+ b64_payload = image_url.split(",", 1)[1]
+ image_bytes = base64.b64decode(b64_payload)
+
+ # If we have coordinates, draw crosshair annotation
+ if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:
+ annotated_image = self._draw_crosshair_on_image(
+ image_bytes, int(x_coord), int(y_coord)
+ )
+ self._save_artifact("screenshot_action", annotated_image)
+ else:
+ # Save plain screenshot without crosshair
+ self._save_artifact("screenshot", image_bytes)
+
+ screenshot_found = True
+ break
+
+ except Exception as e:
+ # If processing fails, just log and continue
+ print(f"Failed to process screenshot from user message: {e}")
+
+ # Increment turn counter
+ self.current_turn += 1
diff --git a/libs/python/agent/agent/computers/cua.py b/libs/python/agent/agent/computers/cua.py
index 6f928402cb..3b5d97e610 100644
--- a/libs/python/agent/agent/computers/cua.py
+++ b/libs/python/agent/agent/computers/cua.py
@@ -62,6 +62,11 @@ class cuaComputerHandler(AsyncComputerHandler):
assert self.interface is not None
await self.interface.double_click(x, y)
+ async def right_click(self, x: int, y: int) -> None:
+ """Right click at coordinates."""
+ assert self.interface is not None
+ await self.interface.right_click(x, y)
+
async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
"""Scroll at coordinates with specified scroll amounts."""
assert self.interface is not None
@@ -96,9 +101,26 @@ class cuaComputerHandler(AsyncComputerHandler):
# Handle key combinations
await self.interface.hotkey(*keys)
- async def drag(self, path: List[Dict[str, int]]) -> None:
- """Drag along specified path."""
+ async def drag(
+ self,
+ path: Optional[List[Dict[str, int]]] = None,
+ start_x: Optional[int] = None,
+ start_y: Optional[int] = None,
+ end_x: Optional[int] = None,
+ end_y: Optional[int] = None,
+ ) -> None:
+ """Drag along specified path or from start to end coordinates.
+
+ Supports two formats:
+ - path: List of {x, y} points to drag through
+ - start_x, start_y, end_x, end_y: Simple drag from start to end
+ """
assert self.interface is not None
+
+ # If start/end coordinates provided, convert to path format
+ if start_x is not None and start_y is not None and end_x is not None and end_y is not None:
+ path = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]
+
if not path:
return
@@ -114,6 +136,17 @@ class cuaComputerHandler(AsyncComputerHandler):
end = path[-1]
await self.interface.mouse_up(end["x"], end["y"])
+ async def terminate(self, status: str = "success") -> Dict[str, Any]:
+ """Terminate the current task and report its completion status.
+
+ Args:
+ status: Status of the task ("success" or "failure")
+
+ Returns:
+ Dict with terminated flag and status
+ """
+ return {"success": True, "status": status, "terminated": True}
+
async def get_current_url(self) -> str:
"""Get current URL (for browser environments)."""
# This would need to be implemented based on the specific browser interface
diff --git a/libs/python/agent/agent/loops/openai.py b/libs/python/agent/agent/loops/openai.py
index 573d09eb60..e013e746e8 100644
--- a/libs/python/agent/agent/loops/openai.py
+++ b/libs/python/agent/agent/loops/openai.py
@@ -15,8 +15,14 @@ from ..decorators import register_agent
from ..types import AgentCapability, AgentResponse, Messages, Tools
-async def _map_computer_tool_to_openai(computer_handler: Any) -> Dict[str, Any]:
- """Map a computer tool to OpenAI's computer-use-preview tool schema"""
+async def _map_computer_tool_to_openai(computer_handler: Any, use_native_tool: bool = True) -> Dict[str, Any]:
+ """Map a computer tool to OpenAI's tool schema.
+
+ Args:
+ computer_handler: The computer handler instance
+ use_native_tool: If True, use native computer_use_preview format (for computer-use-preview model).
+ If False, use standard function calling format (for GPT-5.4 etc).
+ """
# Get dimensions from the computer handler
try:
width, height = await computer_handler.get_dimensions()
@@ -31,32 +37,140 @@ async def _map_computer_tool_to_openai(computer_handler: Any) -> Dict[str, Any]:
# Fallback to default environment if method fails
environment = "linux"
- return {
- "type": "computer_use_preview",
- "display_width": width,
- "display_height": height,
- "environment": environment, # mac, windows, linux, browser
- }
+ if use_native_tool:
+ # Native computer_use_preview format (for computer-use-preview model)
+ return {
+ "type": "computer_use_preview",
+ "display_width": width,
+ "display_height": height,
+ "environment": environment, # mac, windows, linux, browser
+ }
+ else:
+ # Standard function calling format (for GPT-5.4 etc)
+ # Responses API requires: {type, name, description, parameters} at root level
+ return {
+ "type": "function",
+ "name": "computer",
+ "description": (
+ f"Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+ f"Screen resolution: {width}x{height} pixels.\n"
+ f"Environment: {environment}."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "action": {
+ "description": "The action to perform.",
+ "type": "string",
+ "enum": [
+ "click",
+ "double_click",
+ "right_click",
+ "type",
+ "keypress",
+ "scroll",
+ "move",
+ "drag",
+ "screenshot",
+ "wait",
+ "terminate",
+ ],
+ },
+ "x": {
+ "description": "X coordinate for click/move/scroll actions.",
+ "type": "integer",
+ },
+ "y": {
+ "description": "Y coordinate for click/move/scroll actions.",
+ "type": "integer",
+ },
+ "text": {
+ "description": "Text to type (for action=type).",
+ "type": "string",
+ },
+ "keys": {
+ "description": "Keys to press (for action=keypress). Example: ['ctrl', 'c']",
+ "type": "array",
+ "items": {"type": "string"},
+ },
+ "scroll_x": {
+ "description": "Horizontal scroll amount. Positive=right, negative=left.",
+ "type": "integer",
+ },
+ "scroll_y": {
+ "description": "Vertical scroll amount. Positive=down, negative=up.",
+ "type": "integer",
+ },
+ "button": {
+ "description": "Mouse button for click action.",
+ "type": "string",
+ "enum": ["left", "right", "middle"],
+ },
+ "start_x": {
+ "description": "Starting X coordinate for drag action.",
+ "type": "integer",
+ },
+ "start_y": {
+ "description": "Starting Y coordinate for drag action.",
+ "type": "integer",
+ },
+ "end_x": {
+ "description": "Ending X coordinate for drag action.",
+ "type": "integer",
+ },
+ "end_y": {
+ "description": "Ending Y coordinate for drag action.",
+ "type": "integer",
+ },
+ "status": {
+ "description": "Status for terminate action.",
+ "type": "string",
+ "enum": ["success", "failure"],
+ },
+ },
+ "required": ["action"],
+ },
+ }
+
+
+def _is_native_computer_use_model(model: str) -> bool:
+ """Check if the model supports native computer_use_preview tool format."""
+ import re
+ # Only computer-use-preview models support native computer_use_preview tool
+ # GPT 5.4 does NOT support computer_use_preview - it uses function calling
+ return bool(re.search(r"computer-use-preview", model, re.IGNORECASE))
+
+async def _prepare_tools_for_openai(tool_schemas: List[Dict[str, Any]], model: str = "") -> Tools:
+ """Prepare tools for OpenAI API format.
-async def _prepare_tools_for_openai(tool_schemas: List[Dict[str, Any]]) -> Tools:
- """Prepare tools for OpenAI API format"""
+ Args:
+ tool_schemas: List of tool schemas to prepare
+ model: Model name to determine tool format
+ """
openai_tools = []
+ use_native = _is_native_computer_use_model(model)
for schema in tool_schemas:
if schema["type"] == "computer":
- # Map computer tool to OpenAI format
- computer_tool = await _map_computer_tool_to_openai(schema["computer"])
+ # Map computer tool to OpenAI format (native or function based on model)
+ computer_tool = await _map_computer_tool_to_openai(schema["computer"], use_native_tool=use_native)
openai_tools.append(computer_tool)
elif schema["type"] == "function":
- # Function tools use OpenAI-compatible schema directly (liteLLM expects this format)
- # Schema should be: {type, name, description, parameters}
- openai_tools.append({"type": "function", **schema["function"]})
+ # Function tools for Responses API need: {type, name, description, parameters}
+ # Note: parameters are at the root level, NOT nested under 'function'
+ func = schema["function"]
+ openai_tools.append({
+ "type": "function",
+ "name": func["name"],
+ "description": func.get("description", ""),
+ "parameters": func.get("parameters", {}),
+ })
return openai_tools
-@register_agent(models=r".*(^|/)computer-use-preview")
+@register_agent(models=r".*(computer-use-preview|gpt-?5\.?4)")
class OpenAIComputerUseConfig:
"""
OpenAI computer-use-preview agent configuration using liteLLM responses.
@@ -101,7 +215,7 @@ class OpenAIComputerUseConfig:
tools = tools or []
# Prepare tools for OpenAI API
- openai_tools = await _prepare_tools_for_openai(tools)
+ openai_tools = await _prepare_tools_for_openai(tools, model=model)
# Prepare API call kwargs
api_kwargs = {
@@ -126,16 +240,32 @@ class OpenAIComputerUseConfig:
if _on_api_end:
await _on_api_end(api_kwargs, response)
- # Extract usage information
- usage = {
- **response.usage.model_dump(),
- "response_cost": response._hidden_params.get("response_cost", 0.0),
- }
+ # Extract usage information - handle both dict and Pydantic model responses
+ if isinstance(response, dict):
+ response_usage = response.get("usage", {})
+ usage = response_usage if isinstance(response_usage, dict) else {}
+ output_dict = response
+ else:
+ # Response is a Pydantic model - but usage might be dict or model
+ response_usage = response.usage
+ if hasattr(response_usage, "model_dump"):
+ usage = response_usage.model_dump()
+ elif isinstance(response_usage, dict):
+ usage = response_usage
+ else:
+ usage = {}
+ output_dict = response.model_dump()
+
+ # Add response cost if available
+ if hasattr(response, "_hidden_params"):
+ usage["response_cost"] = response._hidden_params.get("response_cost", 0.0)
+ elif isinstance(response, dict):
+ usage["response_cost"] = response.get("_hidden_params", {}).get("response_cost", 0.0)
+
if _on_usage:
await _on_usage(usage)
# Return in the expected format
- output_dict = response.model_dump()
output_dict["usage"] = usage
return output_dict
@@ -191,13 +321,46 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",
# Fallback to default dimensions if image parsing fails
display_width, display_height = 1024, 768
- # Prepare computer tool for click actions
- computer_tool = {
- "type": "computer_use_preview",
- "display_width": display_width,
- "display_height": display_height,
- "environment": "windows",
- }
+ # Prepare computer tool for click actions - use native format only for models that support it
+ use_native = _is_native_computer_use_model(model)
+ if use_native:
+ # Native computer_use_preview format (for computer-use-preview model)
+ computer_tool = {
+ "type": "computer_use_preview",
+ "display_width": display_width,
+ "display_height": display_height,
+ "environment": "windows",
+ }
+ else:
+ # Standard function calling format (for GPT-5.4 etc)
+ computer_tool = {
+ "type": "function",
+ "name": "computer",
+ "description": (
+ f"Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+ f"Screen resolution: {display_width}x{display_height} pixels.\n"
+ f"Environment: windows."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "action": {
+ "description": "The action to perform.",
+ "type": "string",
+ "enum": ["click"],
+ },
+ "x": {
+ "description": "X coordinate for click action.",
+ "type": "integer",
+ },
+ "y": {
+ "description": "Y coordinate for click action.",
+ "type": "integer",
+ },
+ },
+ "required": ["action", "x", "y"],
+ },
+ }
# Prepare API call kwargs
api_kwargs = {
@@ -214,22 +377,37 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",
# Use liteLLM responses
response = await litellm.aresponses(**api_kwargs)
- # Extract click coordinates from response output
- output_dict = response.model_dump()
+ # Extract click coordinates from response output - handle both dict and Pydantic model
+ output_dict = response if isinstance(response, dict) else response.model_dump()
output_items = output_dict.get("output", [])
- # Look for computer_call with click action
+ # Look for click coordinates in the response
for item in output_items:
+ if not isinstance(item, dict):
+ continue
+
+ # Native format: computer_call with action dict
if (
- isinstance(item, dict)
- and item.get("type") == "computer_call"
+ item.get("type") == "computer_call"
and isinstance(item.get("action"), dict)
):
-
action = item["action"]
if action.get("x") is not None and action.get("y") is not None:
return (int(action.get("x")), int(action.get("y")))
+ # Function calling format: function_call with arguments
+ if item.get("type") == "function_call" and item.get("name") == "computer":
+ try:
+ arguments = item.get("arguments", "{}")
+ if isinstance(arguments, str):
+ args = json.loads(arguments)
+ else:
+ args = arguments
+ if args.get("x") is not None and args.get("y") is not None:
+ return (int(args.get("x")), int(args.get("y")))
+ except (json.JSONDecodeError, TypeError):
+ continue
+
return None
def get_capabilities(self) -> List[AgentCapability]:
diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.py
index 6163579585..95ff43fcb7 100644
--- a/libs/python/agent/agent/tools/browser_tool.py
+++ b/libs/python/agent/agent/tools/browser_tool.py
@@ -170,7 +170,8 @@ class BrowserTool(BaseComputerTool):
* history_back: Go back to the previous page in the browser history.
* pause_and_memorize_fact: Pause and memorize a fact for future reference.
* wait: Wait specified seconds for the change to happen.
-* terminate: Terminate the current task and report its completion status.""",
+* terminate: Terminate the current task and report its completion status.
+* screenshot: Take a screenshot of the current screen.""",
"enum": [
"key",
"type",
@@ -183,14 +184,16 @@ class BrowserTool(BaseComputerTool):
"pause_and_memorize_fact",
"wait",
"terminate",
+ "screenshot",
],
"type": "string",
},
- "keys": {"description": "Required only by action=key.", "type": "array"},
+ "keys": {"description": "Required only by action=key.", "type": "array", "items": {"type": "string"}},
"text": {"description": "Required only by action=type.", "type": "string"},
"coordinate": {
"description": "(x, y) coordinates for mouse actions. Required only by action=left_click, action=mouse_move, and action=type.",
"type": "array",
+ "items": {"type": "number"},
},
"pixels": {
"description": "Amount of scrolling. Positive = up, Negative = down. Required only by action=scroll.",
@@ -282,6 +285,8 @@ class BrowserTool(BaseComputerTool):
return await self._action_wait(params)
elif action == "terminate":
return await self._action_terminate(params)
+ elif action == "screenshot":
+ return await self._action_screenshot(params)
else:
return {"success": False, "error": f"Unknown action: {action}"}
except Exception as e:
@@ -411,6 +416,14 @@ class BrowserTool(BaseComputerTool):
return {"success": True, "status": status, "message": message, "terminated": True}
+ async def _action_screenshot(self, params: dict) -> dict:
+ """Take a screenshot of the current screen."""
+ try:
+ screenshot_b64 = await self.screenshot()
+ return {"success": True, "screenshot": screenshot_b64}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
# Legacy methods for backward compatibility
async def visit_url(self, url: str) -> dict:
"""Navigate to a URL."""