Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(computer-server): improve Android MCP usability (#880)
Francesco Bonacci0d6a79b5e9cf
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.pyindex 1bd9adc830..26350cea1d 100644--- a/libs/python/computer-server/computer_server/handlers/android.py+++ b/libs/python/computer-server/computer_server/handlers/android.py@@ -37,13 +37,122 @@ class AndroidAccessibilityHandler(BaseAccessibilityHandler):async def get_accessibility_tree(self) -> Dict[str, Any]:"""Get the accessibility tree using uiautomator dump."""- raise NotImplementedError("get_accessibility_tree not yet implemented for Android")+ import xml.etree.ElementTree as ET++ # Dump UI hierarchy to file and read it+ success, output = await adb_exec.run(+ "shell", "uiautomator dump /sdcard/ui_dump.xml && cat /sdcard/ui_dump.xml",+ decode=True+ )+ if not success or not output:+ raise RuntimeError(f"Failed to dump UI hierarchy: {output}")++ # Clean up the dump file+ await adb_exec.run("shell", "rm", "/sdcard/ui_dump.xml", decode=True)++ try:+ # Parse XML+ root = ET.fromstring(output.strip())+ return {"tree": self._parse_accessibility_node(root)}+ except ET.ParseError as e:+ raise RuntimeError(f"Failed to parse UI hierarchy XML: {e}")++ def _parse_accessibility_node(self, node) -> Dict[str, Any]:+ """Parse a UI Automator XML node into accessibility tree format."""+ result: Dict[str, Any] = {}++ # Extract attributes+ attrs = node.attrib+ if attrs.get("text"):+ result["text"] = attrs["text"]+ if attrs.get("resource-id"):+ result["resource_id"] = attrs["resource-id"]+ if attrs.get("class"):+ result["class"] = attrs["class"]+ if attrs.get("content-desc"):+ result["description"] = attrs["content-desc"]+ if attrs.get("package"):+ result["package"] = attrs["package"]++ # Parse bounds "[x1,y1][x2,y2]" format+ bounds_str = attrs.get("bounds", "")+ if bounds_str:+ import re+ match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)+ if match:+ x1, y1, x2, y2 = map(int, match.groups())+ result["bounds"] = {+ "x": x1,+ "y": y1,+ "width": x2 - x1,+ "height": y2 - y1,+ "center_x": (x1 + x2) // 2,+ "center_y": (y1 + y2) // 2,+ }++ # Boolean attributes+ for attr in ["clickable", "enabled", "focused", "scrollable", "checkable", "checked"]:+ if attrs.get(attr) == "true":+ result[attr] = True++ # Parse children recursively+ children = [self._parse_accessibility_node(child) for child in node]+ if children:+ result["children"] = children++ return resultasync def find_element(self, role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None) -> Dict[str, Any]:- """Find an element in the UI hierarchy."""- raise NotImplementedError("find_element not yet implemented for Android")+ """Find an element in the UI hierarchy by role (class), title (text/content-desc), or value (resource-id)."""+ tree_result = await self.get_accessibility_tree()+ tree = tree_result.get("tree", {})++ matches = []+ self._find_elements_recursive(tree, role, title, value, matches)++ if not matches:+ return {"found": False, "elements": []}++ return {"found": True, "elements": matches}++ def _find_elements_recursive(+ self,+ node: Dict[str, Any],+ role: Optional[str],+ title: Optional[str],+ value: Optional[str],+ matches: List[Dict[str, Any]],+ ):+ """Recursively search for elements matching criteria."""+ is_match = True++ # Match by role (class name)+ if role:+ node_class = node.get("class", "")+ if role.lower() not in node_class.lower():+ is_match = False++ # Match by title (text or content-desc)+ if title and is_match:+ node_text = node.get("text", "")+ node_desc = node.get("description", "")+ if title.lower() not in node_text.lower() and title.lower() not in node_desc.lower():+ is_match = False++ # Match by value (resource-id)+ if value and is_match:+ node_id = node.get("resource_id", "")+ if value.lower() not in node_id.lower():+ is_match = False++ if is_match and (role or title or value):+ matches.append(node)++ # Search children+ for child in node.get("children", []):+ self._find_elements_recursive(child, role, title, value, matches)class AndroidAutomationHandler(BaseAutomationHandler):diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.pyindex 4aefd1d751..e4a4f4adbe 100644--- a/libs/python/computer-server/computer_server/main.py+++ b/libs/python/computer-server/computer_server/main.py@@ -108,6 +108,44 @@ except Exception:desktop_handler,window_handler,) = HandlerFactory.create_handlers()+# Helper function for direction-based scrolling+async def _scroll_direction_handler(direction: str, clicks: int = 1) -> Dict[str, Any]:+ """+ Scroll in a specified direction.++ Args:+ direction: One of 'up', 'down', 'left', 'right'+ clicks: Number of scroll clicks (default: 1)+ """+ direction = direction.lower()+ if direction == "down":+ return await automation_handler.scroll_down(clicks)+ elif direction == "up":+ return await automation_handler.scroll_up(clicks)+ elif direction in ("left", "right"):+ # For horizontal scrolling, use scroll with x amount+ x_amount = -300 if direction == "left" else 300+ return await automation_handler.scroll(x_amount * clicks, 0)+ else:+ raise ValueError(f"Invalid direction: {direction}. Use 'up', 'down', 'left', or 'right'.")+++# Command aliases for common alternative names+COMMAND_ALIASES = {+ "type": "type_text",+ "key": "press_key",+ "tap": "left_click",+ "click": "left_click",+ "shell": "run_command",+ "exec": "run_command",+ "read_file": "read_text",+ "write_file": "write_text",+ "ls": "list_dir",+ "mkdir": "create_dir",+ "rm": "delete_file",+ "rmdir": "delete_dir",+}+handlers = {"version": lambda: {"protocol": protocol_version, "package": package_version},# App-Use commands@@ -165,6 +203,7 @@ handlers = {"scroll": automation_handler.scroll,"scroll_down": automation_handler.scroll_down,"scroll_up": automation_handler.scroll_up,+ "scroll_direction": _scroll_direction_handler,# Screen actions"screenshot": automation_handler.screenshot,"get_cursor_position": automation_handler.get_cursor_position,@@ -284,6 +323,23 @@ manager = ConnectionManager()auth_manager = AuthenticationManager()+def _resolve_command(command: str) -> str:+ """Resolve command aliases to their canonical names."""+ return COMMAND_ALIASES.get(command, command)+++def _suggest_similar_commands(command: str) -> List[str]:+ """Suggest similar commands for typos."""+ all_commands = list(handlers.keys()) + list(COMMAND_ALIASES.keys())+ suggestions = []+ command_lower = command.lower()+ for cmd in all_commands:+ # Simple similarity: starts with same letter or contains the input+ if cmd.lower().startswith(command_lower[:2]) or command_lower in cmd.lower():+ suggestions.append(cmd)+ return suggestions[:5]++@app.get("/status")async def status():# get computer-server features@@ -295,6 +351,42 @@ async def status():return {"status": "ok", "os_type": OS_TYPE, "features": features}[email protected]("/commands")+async def list_commands():+ """List all available commands and their aliases."""+ commands_info = {}+ for cmd_name, handler_func in handlers.items():+ try:+ sig = inspect.signature(handler_func)+ params = [+ {+ "name": p.name,+ "required": p.default == inspect.Parameter.empty,+ "default": None if p.default == inspect.Parameter.empty else p.default,+ }+ for p in sig.parameters.values()+ ]+ except (ValueError, TypeError):+ params = []+ commands_info[cmd_name] = {"params": params}++ # Add alias information+ aliases_by_command: Dict[str, List[str]] = {}+ for alias, canonical in COMMAND_ALIASES.items():+ if canonical not in aliases_by_command:+ aliases_by_command[canonical] = []+ aliases_by_command[canonical].append(alias)++ for cmd_name in commands_info:+ if cmd_name in aliases_by_command:+ commands_info[cmd_name]["aliases"] = aliases_by_command[cmd_name]++ return {+ "commands": commands_info,+ "aliases": COMMAND_ALIASES,+ }++@app.websocket("/ws", name="websocket_endpoint")async def websocket_endpoint(websocket: WebSocket):global handlers@@ -366,10 +458,15 @@ async def websocket_endpoint(websocket: WebSocket):command = data.get("command")params = data.get("params", {})+ # Resolve command aliases+ command = _resolve_command(command)+if command not in handlers:- await websocket.send_json(- {"success": False, "error": f"Unknown command: {command}"}- )+ suggestions = _suggest_similar_commands(command)+ error_msg = f"Unknown command: {command}"+ if suggestions:+ error_msg += f". Did you mean: {', '.join(suggestions)}?"+ await websocket.send_json({"success": False, "error": error_msg})continuetry:@@ -448,6 +545,9 @@ async def cmd_endpoint(if not command:raise HTTPException(status_code=400, detail="Command is required")+ # Resolve command aliases+ command = _resolve_command(command)+# Check if CONTAINER_NAME is set (indicating cloud provider)server_container_name = os.environ.get("CONTAINER_NAME")@@ -470,7 +570,11 @@ async def cmd_endpoint(raise HTTPException(status_code=401, detail="Authentication failed")if command not in handlers:- raise HTTPException(status_code=400, detail=f"Unknown command: {command}")+ suggestions = _suggest_similar_commands(command)+ error_msg = f"Unknown command: {command}"+ if suggestions:+ error_msg += f". Did you mean: {', '.join(suggestions)}?"+ raise HTTPException(status_code=400, detail=error_msg)async def generate_response():"""Generate streaming response for the command execution"""