Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(agent): Add GPT 5.4 support for Computer tool (#1150)
Sarina Liff38603826b4
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.pyindex 2d790d2ed2..40014da478 100644--- a/libs/python/agent/agent/agent.py+++ b/libs/python/agent/agent/agent.py@@ -782,6 +782,112 @@ class ComputerAgent:if item_type == "function_call":await self._on_function_call_start(item)++ # Special handling for "computer" function calls (from GPT 5.4 etc)+ # These need to be treated like computer_call items+ if item.get("name") == "computer" and computer:+ args = json.loads(item.get("arguments", "{}"))+ action_type = args.get("action")+ if not action_type:+ raise ToolError("Computer function call missing 'action' argument")++ # Map action types to their relevant parameters+ action_param_map = {+ "screenshot": [], # No parameters+ "click": ["x", "y", "button"],+ "double_click": ["x", "y"],+ "right_click": ["x", "y"],+ "type": ["text"],+ "keypress": ["keys"],+ "scroll": ["x", "y", "scroll_x", "scroll_y"],+ "move": ["x", "y"],+ "drag": ["start_x", "start_y", "end_x", "end_y"],+ "wait": ["seconds", "ms"],+ "terminate": ["status"],+ }++ # Extract only relevant action arguments, filtering out empty values+ relevant_params = action_param_map.get(action_type, [])+ action_args = {}+ for k, v in args.items():+ if k == "action":+ continue+ # Include if it's a relevant param OR if param map doesn't have this action+ if k in relevant_params or action_type not in action_param_map:+ # Filter out empty/default values but allow numeric zero (valid for coordinates/scroll)+ if v is not None and v != "" and v != []:+ action_args[k] = v++ # Execute the computer action+ computer_method = getattr(computer, action_type, None)+ action_result = None+ if computer_method:+ action_result = await computer_method(**action_args)+ else:+ raise ToolError(f"Unknown computer action: {action_type}")++ # Track computer action execution+ if self.telemetry_enabled and is_telemetry_enabled():+ record_event(+ "computer_action_executed",+ {"action_type": action_type},+ )+ record_event(+ "agent_tool_executed",+ {"tool_type": "computer", "tool_name": action_type},+ )++ # Check if this was a terminate action+ is_terminate = action_type == "terminate" or (+ isinstance(action_result, dict) and action_result.get("terminated")+ )++ # Take screenshot after action (skip for terminate)+ if not is_terminate:+ if self.screenshot_delay and self.screenshot_delay > 0:+ await asyncio.sleep(self.screenshot_delay)+ screenshot_base64 = await computer.screenshot()+ await self._on_screenshot(screenshot_base64, "screenshot_after")++ # Create function call output for GPT 5.4 etc+ # Note: function_call_output only supports text, so we include the+ # screenshot as a separate user message with image content+ if is_terminate:+ output_content = json.dumps(action_result if action_result else {"terminated": True})+ call_output = {+ "type": "function_call_output",+ "call_id": item.get("call_id"),+ "output": output_content,+ }+ result = [call_output]+ else:+ # Return both the function output AND a user message with the screenshot+ # This allows the model to see the screenshot result+ # Preserve actual action_result so failures/errors aren't masked+ if action_result is not None:+ output_content = json.dumps(action_result)+ else:+ output_content = json.dumps({"success": True, "screenshot_captured": True})+ call_output = {+ "type": "function_call_output",+ "call_id": item.get("call_id"),+ "output": output_content,+ }+ # Include screenshot as a user message with image content+ image_message = {+ "role": "user",+ "content": [+ {+ "type": "input_image",+ "image_url": f"data:image/png;base64,{screenshot_base64}",+ }+ ],+ }+ result = [call_output, image_message]++ await self._on_function_call_end(item, result)+ return result+# Perform function callfunction = self._get_tool(item.get("name"))if not function:diff --git a/libs/python/agent/agent/callbacks/trajectory_saver.py b/libs/python/agent/agent/callbacks/trajectory_saver.pyindex c73276bf0e..81627882f7 100644--- a/libs/python/agent/agent/callbacks/trajectory_saver.py+++ b/libs/python/agent/agent/callbacks/trajectory_saver.py@@ -51,13 +51,14 @@ def extract_computer_call_outputs(items: List[Dict[str, Any]], screenshot_dir: Optional[Path]) -> List[Dict[str, Any]]:"""- Save any base64-encoded screenshots from computer_call_output entries to files and- replace their image_url with the saved file path when a call_id is present.+ Save any base64-encoded screenshots from computer_call_output or function_call_output+ entries to files and replace their image_url with the saved file path when a call_id is present.Only operates if screenshot_dir is provided and exists; otherwise returns items unchanged.Args:- items: List of message/result dicts potentially containing computer_call_output entries+ items: List of message/result dicts potentially containing computer_call_output+ or function_call_output entriesscreenshot_dir: Directory to write screenshots intoReturns:@@ -101,6 +102,94 @@ def extract_computer_call_outputs(new_output = dict(output)new_output["image_url"] = str(out_path)msg["output"] = new_output++ elif msg.get("type") == "function_call_output":+ # Handle function_call_output from GPT 5.4 / BrowserTool+ call_id = msg.get("call_id")+ output = msg.get("output", "")++ # Parse output if it's a string+ if isinstance(output, str):+ try:+ output_dict = json.loads(output)+ except (json.JSONDecodeError, TypeError):+ output_dict = None+ else:+ output_dict = output++ if isinstance(output_dict, dict) and call_id:+ image_data = None+ image_key = None++ # Format 1: {"type": "input_image", "image_url": "data:image/png;base64,..."}+ if output_dict.get("type") == "input_image":+ image_url = output_dict.get("image_url", "")+ if isinstance(image_url, str) and image_url.startswith("data:"):+ image_data = image_url.split(",", 1)[1] if "," in image_url else None+ image_key = "image_url"++ # Format 2: {"success": True, "screenshot": "base64data"}+ elif output_dict.get("screenshot"):+ image_data = output_dict.get("screenshot")+ image_key = "screenshot"++ if image_data and image_key:+ out_path = screenshot_dir / f"{call_id}.png"+ if not out_path.exists():+ try:+ img_bytes = base64.b64decode(image_data)+ out_path.parent.mkdir(parents=True, exist_ok=True)+ with open(out_path, "wb") as f:+ f.write(img_bytes)+ except Exception:+ pass+ # Update output to reference file path+ new_output_dict = dict(output_dict)+ new_output_dict[image_key] = str(out_path)+ msg["output"] = json.dumps(new_output_dict)++ elif msg.get("role") == "user":+ # Handle user messages with input_image content (GPT-5.4 sibling screenshot messages)+ # These accompany function_call_output for computer calls+ content = msg.get("content", [])+ if isinstance(content, list):+ new_content = []+ content_modified = False+ for content_item in content:+ if isinstance(content_item, dict) and content_item.get("type") == "input_image":+ image_url = content_item.get("image_url", "")+ if isinstance(image_url, str) and image_url.startswith("data:"):+ # Generate a unique ID for this screenshot+ screenshot_id = str(uuid.uuid4())[:8]+ try:+ ext = image_url.split(";", 1)[0].split("/")[-1]+ if not ext:+ ext = "png"+ except Exception:+ ext = "png"+ out_path = screenshot_dir / f"user_screenshot_{screenshot_id}.{ext}"+ if not out_path.exists():+ try:+ b64_payload = image_url.split(",", 1)[1]+ img_bytes = base64.b64decode(b64_payload)+ out_path.parent.mkdir(parents=True, exist_ok=True)+ with open(out_path, "wb") as f:+ f.write(img_bytes)+ except Exception:+ new_content.append(content_item)+ continue+ # Update image_url to file path+ new_item = dict(content_item)+ new_item["image_url"] = str(out_path)+ new_content.append(new_item)+ content_modified = True+ else:+ new_content.append(content_item)+ else:+ new_content.append(content_item)+ if content_modified:+ msg["content"] = new_content+except Exception:# do not block on malformed entries; keep originalpass@@ -420,3 +509,130 @@ class TrajectorySaverCallback(AsyncCallbackHandler):# Increment turn counterself.current_turn += 1++ @override+ async def on_function_call_end(+ self, item: Dict[str, Any], result: List[Dict[str, Any]]+ ) -> None:+ """+ Called when a function call has completed.+ Saves screenshots and function call output for GPT 5.4 / BrowserTool.+ """+ if not self.trajectory_id:+ return++ self._save_artifact("function_call_result", {"item": item, "result": result})++ # Extract coordinates from function call arguments if present+ x_coord, y_coord = None, None+ try:+ arguments = item.get("arguments", "{}")+ if isinstance(arguments, str):+ args_dict = json.loads(arguments)+ else:+ args_dict = arguments++ # Check for coordinate array format (BrowserTool style)+ coord = args_dict.get("coordinate")+ if coord and isinstance(coord, list) and len(coord) >= 2:+ x_coord, y_coord = coord[0], coord[1]+ # Check for x/y format (computer_use style)+ elif "x" in args_dict and "y" in args_dict:+ x_coord, y_coord = args_dict.get("x"), args_dict.get("y")+ except (json.JSONDecodeError, TypeError):+ pass++ # Look for screenshot in the result+ screenshot_found = False+ for result_item in result:+ if screenshot_found:+ break++ if result_item.get("type") == "function_call_output":+ output = result_item.get("output", "")++ # Parse output if it's a string+ if isinstance(output, str):+ try:+ output_dict = json.loads(output)+ except (json.JSONDecodeError, TypeError):+ # Try to evaluate as Python literal (for stringified dicts)+ try:+ import ast+ output_dict = ast.literal_eval(output)+ except (ValueError, SyntaxError):+ continue+ else:+ output_dict = output++ if not isinstance(output_dict, dict):+ continue++ # Extract screenshot from various formats+ image_data = None++ # Format 1: {"type": "input_image", "image_url": "data:image/png;base64,..."}+ if output_dict.get("type") == "input_image":+ image_url = output_dict.get("image_url", "")+ if image_url.startswith("data:image/"):+ image_data = image_url.split(",", 1)[1]+ elif image_url:+ image_data = image_url++ # Format 2: {"success": True, "screenshot": "base64data"}+ elif output_dict.get("screenshot"):+ image_data = output_dict.get("screenshot")++ if image_data:+ try:+ # Decode the image+ image_bytes = base64.b64decode(image_data)++ # If we have coordinates, draw crosshair annotation+ if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:+ annotated_image = self._draw_crosshair_on_image(+ image_bytes, int(x_coord), int(y_coord)+ )+ self._save_artifact("screenshot_action", annotated_image)+ else:+ # Save plain screenshot without crosshair+ self._save_artifact("screenshot", image_bytes)++ screenshot_found = True++ except Exception as e:+ # If processing fails, just log and continue+ print(f"Failed to process screenshot from function call: {e}")++ # Handle sibling user messages with input_image content (GPT-5.4 computer calls)+ # These accompany function_call_output and contain the actual screenshot+ elif result_item.get("role") == "user":+ content = result_item.get("content", [])+ if isinstance(content, list):+ for content_item in content:+ if isinstance(content_item, dict) and content_item.get("type") == "input_image":+ image_url = content_item.get("image_url", "")+ if isinstance(image_url, str) and image_url.startswith("data:"):+ try:+ b64_payload = image_url.split(",", 1)[1]+ image_bytes = base64.b64decode(b64_payload)++ # If we have coordinates, draw crosshair annotation+ if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:+ annotated_image = self._draw_crosshair_on_image(+ image_bytes, int(x_coord), int(y_coord)+ )+ self._save_artifact("screenshot_action", annotated_image)+ else:+ # Save plain screenshot without crosshair+ self._save_artifact("screenshot", image_bytes)++ screenshot_found = True+ break++ except Exception as e:+ # If processing fails, just log and continue+ print(f"Failed to process screenshot from user message: {e}")++ # Increment turn counter+ self.current_turn += 1diff --git a/libs/python/agent/agent/computers/cua.py b/libs/python/agent/agent/computers/cua.pyindex 6f928402cb..3b5d97e610 100644--- a/libs/python/agent/agent/computers/cua.py+++ b/libs/python/agent/agent/computers/cua.py@@ -62,6 +62,11 @@ class cuaComputerHandler(AsyncComputerHandler):assert self.interface is not Noneawait self.interface.double_click(x, y)+ async def right_click(self, x: int, y: int) -> None:+ """Right click at coordinates."""+ assert self.interface is not None+ await self.interface.right_click(x, y)+async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:"""Scroll at coordinates with specified scroll amounts."""assert self.interface is not None@@ -96,9 +101,26 @@ class cuaComputerHandler(AsyncComputerHandler):# Handle key combinationsawait self.interface.hotkey(*keys)- async def drag(self, path: List[Dict[str, int]]) -> None:- """Drag along specified path."""+ async def drag(+ self,+ path: Optional[List[Dict[str, int]]] = None,+ start_x: Optional[int] = None,+ start_y: Optional[int] = None,+ end_x: Optional[int] = None,+ end_y: Optional[int] = None,+ ) -> None:+ """Drag along specified path or from start to end coordinates.++ Supports two formats:+ - path: List of {x, y} points to drag through+ - start_x, start_y, end_x, end_y: Simple drag from start to end+ """assert self.interface is not None++ # If start/end coordinates provided, convert to path format+ if start_x is not None and start_y is not None and end_x is not None and end_y is not None:+ path = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]+if not path:return@@ -114,6 +136,17 @@ class cuaComputerHandler(AsyncComputerHandler):end = path[-1]await self.interface.mouse_up(end["x"], end["y"])+ async def terminate(self, status: str = "success") -> Dict[str, Any]:+ """Terminate the current task and report its completion status.++ Args:+ status: Status of the task ("success" or "failure")++ Returns:+ Dict with terminated flag and status+ """+ return {"success": True, "status": status, "terminated": True}+async def get_current_url(self) -> str:"""Get current URL (for browser environments)."""# This would need to be implemented based on the specific browser interfacediff --git a/libs/python/agent/agent/loops/openai.py b/libs/python/agent/agent/loops/openai.pyindex 573d09eb60..e013e746e8 100644--- a/libs/python/agent/agent/loops/openai.py+++ b/libs/python/agent/agent/loops/openai.py@@ -15,8 +15,14 @@ from ..decorators import register_agentfrom ..types import AgentCapability, AgentResponse, Messages, Tools-async def _map_computer_tool_to_openai(computer_handler: Any) -> Dict[str, Any]:- """Map a computer tool to OpenAI's computer-use-preview tool schema"""+async def _map_computer_tool_to_openai(computer_handler: Any, use_native_tool: bool = True) -> Dict[str, Any]:+ """Map a computer tool to OpenAI's tool schema.++ Args:+ computer_handler: The computer handler instance+ use_native_tool: If True, use native computer_use_preview format (for computer-use-preview model).+ If False, use standard function calling format (for GPT-5.4 etc).+ """# Get dimensions from the computer handlertry:width, height = await computer_handler.get_dimensions()@@ -31,32 +37,140 @@ async def _map_computer_tool_to_openai(computer_handler: Any) -> Dict[str, Any]:# Fallback to default environment if method failsenvironment = "linux"- return {- "type": "computer_use_preview",- "display_width": width,- "display_height": height,- "environment": environment, # mac, windows, linux, browser- }+ if use_native_tool:+ # Native computer_use_preview format (for computer-use-preview model)+ return {+ "type": "computer_use_preview",+ "display_width": width,+ "display_height": height,+ "environment": environment, # mac, windows, linux, browser+ }+ else:+ # Standard function calling format (for GPT-5.4 etc)+ # Responses API requires: {type, name, description, parameters} at root level+ return {+ "type": "function",+ "name": "computer",+ "description": (+ f"Use a mouse and keyboard to interact with a computer, and take screenshots.\n"+ f"Screen resolution: {width}x{height} pixels.\n"+ f"Environment: {environment}."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "action": {+ "description": "The action to perform.",+ "type": "string",+ "enum": [+ "click",+ "double_click",+ "right_click",+ "type",+ "keypress",+ "scroll",+ "move",+ "drag",+ "screenshot",+ "wait",+ "terminate",+ ],+ },+ "x": {+ "description": "X coordinate for click/move/scroll actions.",+ "type": "integer",+ },+ "y": {+ "description": "Y coordinate for click/move/scroll actions.",+ "type": "integer",+ },+ "text": {+ "description": "Text to type (for action=type).",+ "type": "string",+ },+ "keys": {+ "description": "Keys to press (for action=keypress). Example: ['ctrl', 'c']",+ "type": "array",+ "items": {"type": "string"},+ },+ "scroll_x": {+ "description": "Horizontal scroll amount. Positive=right, negative=left.",+ "type": "integer",+ },+ "scroll_y": {+ "description": "Vertical scroll amount. Positive=down, negative=up.",+ "type": "integer",+ },+ "button": {+ "description": "Mouse button for click action.",+ "type": "string",+ "enum": ["left", "right", "middle"],+ },+ "start_x": {+ "description": "Starting X coordinate for drag action.",+ "type": "integer",+ },+ "start_y": {+ "description": "Starting Y coordinate for drag action.",+ "type": "integer",+ },+ "end_x": {+ "description": "Ending X coordinate for drag action.",+ "type": "integer",+ },+ "end_y": {+ "description": "Ending Y coordinate for drag action.",+ "type": "integer",+ },+ "status": {+ "description": "Status for terminate action.",+ "type": "string",+ "enum": ["success", "failure"],+ },+ },+ "required": ["action"],+ },+ }+++def _is_native_computer_use_model(model: str) -> bool:+ """Check if the model supports native computer_use_preview tool format."""+ import re+ # Only computer-use-preview models support native computer_use_preview tool+ # GPT 5.4 does NOT support computer_use_preview - it uses function calling+ return bool(re.search(r"computer-use-preview", model, re.IGNORECASE))++async def _prepare_tools_for_openai(tool_schemas: List[Dict[str, Any]], model: str = "") -> Tools:+ """Prepare tools for OpenAI API format.-async def _prepare_tools_for_openai(tool_schemas: List[Dict[str, Any]]) -> Tools:- """Prepare tools for OpenAI API format"""+ Args:+ tool_schemas: List of tool schemas to prepare+ model: Model name to determine tool format+ """openai_tools = []+ use_native = _is_native_computer_use_model(model)for schema in tool_schemas:if schema["type"] == "computer":- # Map computer tool to OpenAI format- computer_tool = await _map_computer_tool_to_openai(schema["computer"])+ # Map computer tool to OpenAI format (native or function based on model)+ computer_tool = await _map_computer_tool_to_openai(schema["computer"], use_native_tool=use_native)openai_tools.append(computer_tool)elif schema["type"] == "function":- # Function tools use OpenAI-compatible schema directly (liteLLM expects this format)- # Schema should be: {type, name, description, parameters}- openai_tools.append({"type": "function", **schema["function"]})+ # Function tools for Responses API need: {type, name, description, parameters}+ # Note: parameters are at the root level, NOT nested under 'function'+ func = schema["function"]+ openai_tools.append({+ "type": "function",+ "name": func["name"],+ "description": func.get("description", ""),+ "parameters": func.get("parameters", {}),+ })return openai_tools-@register_agent(models=r".*(^|/)computer-use-preview")+@register_agent(models=r".*(computer-use-preview|gpt-?5\.?4)")class OpenAIComputerUseConfig:"""OpenAI computer-use-preview agent configuration using liteLLM responses.@@ -101,7 +215,7 @@ class OpenAIComputerUseConfig:tools = tools or []# Prepare tools for OpenAI API- openai_tools = await _prepare_tools_for_openai(tools)+ openai_tools = await _prepare_tools_for_openai(tools, model=model)# Prepare API call kwargsapi_kwargs = {@@ -126,16 +240,32 @@ class OpenAIComputerUseConfig:if _on_api_end:await _on_api_end(api_kwargs, response)- # Extract usage information- usage = {- **response.usage.model_dump(),- "response_cost": response._hidden_params.get("response_cost", 0.0),- }+ # Extract usage information - handle both dict and Pydantic model responses+ if isinstance(response, dict):+ response_usage = response.get("usage", {})+ usage = response_usage if isinstance(response_usage, dict) else {}+ output_dict = response+ else:+ # Response is a Pydantic model - but usage might be dict or model+ response_usage = response.usage+ if hasattr(response_usage, "model_dump"):+ usage = response_usage.model_dump()+ elif isinstance(response_usage, dict):+ usage = response_usage+ else:+ usage = {}+ output_dict = response.model_dump()++ # Add response cost if available+ if hasattr(response, "_hidden_params"):+ usage["response_cost"] = response._hidden_params.get("response_cost", 0.0)+ elif isinstance(response, dict):+ usage["response_cost"] = response.get("_hidden_params", {}).get("response_cost", 0.0)+if _on_usage:await _on_usage(usage)# Return in the expected format- output_dict = response.model_dump()output_dict["usage"] = usagereturn output_dict@@ -191,13 +321,46 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",# Fallback to default dimensions if image parsing failsdisplay_width, display_height = 1024, 768- # Prepare computer tool for click actions- computer_tool = {- "type": "computer_use_preview",- "display_width": display_width,- "display_height": display_height,- "environment": "windows",- }+ # Prepare computer tool for click actions - use native format only for models that support it+ use_native = _is_native_computer_use_model(model)+ if use_native:+ # Native computer_use_preview format (for computer-use-preview model)+ computer_tool = {+ "type": "computer_use_preview",+ "display_width": display_width,+ "display_height": display_height,+ "environment": "windows",+ }+ else:+ # Standard function calling format (for GPT-5.4 etc)+ computer_tool = {+ "type": "function",+ "name": "computer",+ "description": (+ f"Use a mouse and keyboard to interact with a computer, and take screenshots.\n"+ f"Screen resolution: {display_width}x{display_height} pixels.\n"+ f"Environment: windows."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "action": {+ "description": "The action to perform.",+ "type": "string",+ "enum": ["click"],+ },+ "x": {+ "description": "X coordinate for click action.",+ "type": "integer",+ },+ "y": {+ "description": "Y coordinate for click action.",+ "type": "integer",+ },+ },+ "required": ["action", "x", "y"],+ },+ }# Prepare API call kwargsapi_kwargs = {@@ -214,22 +377,37 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",# Use liteLLM responsesresponse = await litellm.aresponses(**api_kwargs)- # Extract click coordinates from response output- output_dict = response.model_dump()+ # Extract click coordinates from response output - handle both dict and Pydantic model+ output_dict = response if isinstance(response, dict) else response.model_dump()output_items = output_dict.get("output", [])- # Look for computer_call with click action+ # Look for click coordinates in the responsefor item in output_items:+ if not isinstance(item, dict):+ continue++ # Native format: computer_call with action dictif (- isinstance(item, dict)- and item.get("type") == "computer_call"+ item.get("type") == "computer_call"and isinstance(item.get("action"), dict)):-action = item["action"]if action.get("x") is not None and action.get("y") is not None:return (int(action.get("x")), int(action.get("y")))+ # Function calling format: function_call with arguments+ if item.get("type") == "function_call" and item.get("name") == "computer":+ try:+ arguments = item.get("arguments", "{}")+ if isinstance(arguments, str):+ args = json.loads(arguments)+ else:+ args = arguments+ if args.get("x") is not None and args.get("y") is not None:+ return (int(args.get("x")), int(args.get("y")))+ except (json.JSONDecodeError, TypeError):+ continue+return Nonedef get_capabilities(self) -> List[AgentCapability]:diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.pyindex 6163579585..95ff43fcb7 100644--- a/libs/python/agent/agent/tools/browser_tool.py+++ b/libs/python/agent/agent/tools/browser_tool.py@@ -170,7 +170,8 @@ class BrowserTool(BaseComputerTool):* history_back: Go back to the previous page in the browser history.* pause_and_memorize_fact: Pause and memorize a fact for future reference.* wait: Wait specified seconds for the change to happen.-* terminate: Terminate the current task and report its completion status.""",+* terminate: Terminate the current task and report its completion status.+* screenshot: Take a screenshot of the current screen.""","enum": ["key","type",@@ -183,14 +184,16 @@ class BrowserTool(BaseComputerTool):"pause_and_memorize_fact","wait","terminate",+ "screenshot",],"type": "string",},- "keys": {"description": "Required only by action=key.", "type": "array"},+ "keys": {"description": "Required only by action=key.", "type": "array", "items": {"type": "string"}},"text": {"description": "Required only by action=type.", "type": "string"},"coordinate": {"description": "(x, y) coordinates for mouse actions. Required only by action=left_click, action=mouse_move, and action=type.","type": "array",+ "items": {"type": "number"},},"pixels": {"description": "Amount of scrolling. Positive = up, Negative = down. Required only by action=scroll.",@@ -282,6 +285,8 @@ class BrowserTool(BaseComputerTool):return await self._action_wait(params)elif action == "terminate":return await self._action_terminate(params)+ elif action == "screenshot":+ return await self._action_screenshot(params)else:return {"success": False, "error": f"Unknown action: {action}"}except Exception as e:@@ -411,6 +416,14 @@ class BrowserTool(BaseComputerTool):return {"success": True, "status": status, "message": message, "terminated": True}+ async def _action_screenshot(self, params: dict) -> dict:+ """Take a screenshot of the current screen."""+ try:+ screenshot_b64 = await self.screenshot()+ return {"success": True, "screenshot": screenshot_b64}+ except Exception as e:+ return {"success": False, "error": str(e)}+# Legacy methods for backward compatibilityasync def visit_url(self, url: str) -> dict:"""Navigate to a URL."""