Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(computer-server): improve Android MCP usability (#880)

Francesco Bonacci0d6a79b5e9cf
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.py
index 1bd9adc830..26350cea1d 100644
--- a/libs/python/computer-server/computer_server/handlers/android.py
+++ b/libs/python/computer-server/computer_server/handlers/android.py
@@ -37,13 +37,122 @@ class AndroidAccessibilityHandler(BaseAccessibilityHandler):
async def get_accessibility_tree(self) -> Dict[str, Any]:
"""Get the accessibility tree using uiautomator dump."""
- raise NotImplementedError("get_accessibility_tree not yet implemented for Android")
+ import xml.etree.ElementTree as ET
+
+ # Dump UI hierarchy to file and read it
+ success, output = await adb_exec.run(
+ "shell", "uiautomator dump /sdcard/ui_dump.xml && cat /sdcard/ui_dump.xml",
+ decode=True
+ )
+ if not success or not output:
+ raise RuntimeError(f"Failed to dump UI hierarchy: {output}")
+
+ # Clean up the dump file
+ await adb_exec.run("shell", "rm", "/sdcard/ui_dump.xml", decode=True)
+
+ try:
+ # Parse XML
+ root = ET.fromstring(output.strip())
+ return {"tree": self._parse_accessibility_node(root)}
+ except ET.ParseError as e:
+ raise RuntimeError(f"Failed to parse UI hierarchy XML: {e}")
+
+ def _parse_accessibility_node(self, node) -> Dict[str, Any]:
+ """Parse a UI Automator XML node into accessibility tree format."""
+ result: Dict[str, Any] = {}
+
+ # Extract attributes
+ attrs = node.attrib
+ if attrs.get("text"):
+ result["text"] = attrs["text"]
+ if attrs.get("resource-id"):
+ result["resource_id"] = attrs["resource-id"]
+ if attrs.get("class"):
+ result["class"] = attrs["class"]
+ if attrs.get("content-desc"):
+ result["description"] = attrs["content-desc"]
+ if attrs.get("package"):
+ result["package"] = attrs["package"]
+
+ # Parse bounds "[x1,y1][x2,y2]" format
+ bounds_str = attrs.get("bounds", "")
+ if bounds_str:
+ import re
+ match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
+ if match:
+ x1, y1, x2, y2 = map(int, match.groups())
+ result["bounds"] = {
+ "x": x1,
+ "y": y1,
+ "width": x2 - x1,
+ "height": y2 - y1,
+ "center_x": (x1 + x2) // 2,
+ "center_y": (y1 + y2) // 2,
+ }
+
+ # Boolean attributes
+ for attr in ["clickable", "enabled", "focused", "scrollable", "checkable", "checked"]:
+ if attrs.get(attr) == "true":
+ result[attr] = True
+
+ # Parse children recursively
+ children = [self._parse_accessibility_node(child) for child in node]
+ if children:
+ result["children"] = children
+
+ return result
async def find_element(
self, role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None
) -> Dict[str, Any]:
- """Find an element in the UI hierarchy."""
- raise NotImplementedError("find_element not yet implemented for Android")
+ """Find an element in the UI hierarchy by role (class), title (text/content-desc), or value (resource-id)."""
+ tree_result = await self.get_accessibility_tree()
+ tree = tree_result.get("tree", {})
+
+ matches = []
+ self._find_elements_recursive(tree, role, title, value, matches)
+
+ if not matches:
+ return {"found": False, "elements": []}
+
+ return {"found": True, "elements": matches}
+
+ def _find_elements_recursive(
+ self,
+ node: Dict[str, Any],
+ role: Optional[str],
+ title: Optional[str],
+ value: Optional[str],
+ matches: List[Dict[str, Any]],
+ ):
+ """Recursively search for elements matching criteria."""
+ is_match = True
+
+ # Match by role (class name)
+ if role:
+ node_class = node.get("class", "")
+ if role.lower() not in node_class.lower():
+ is_match = False
+
+ # Match by title (text or content-desc)
+ if title and is_match:
+ node_text = node.get("text", "")
+ node_desc = node.get("description", "")
+ if title.lower() not in node_text.lower() and title.lower() not in node_desc.lower():
+ is_match = False
+
+ # Match by value (resource-id)
+ if value and is_match:
+ node_id = node.get("resource_id", "")
+ if value.lower() not in node_id.lower():
+ is_match = False
+
+ if is_match and (role or title or value):
+ matches.append(node)
+
+ # Search children
+ for child in node.get("children", []):
+ self._find_elements_recursive(child, role, title, value, matches)
class AndroidAutomationHandler(BaseAutomationHandler):
diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.py
index 4aefd1d751..e4a4f4adbe 100644
--- a/libs/python/computer-server/computer_server/main.py
+++ b/libs/python/computer-server/computer_server/main.py
@@ -108,6 +108,44 @@ except Exception:
desktop_handler,
window_handler,
) = HandlerFactory.create_handlers()
+# Helper function for direction-based scrolling
+async def _scroll_direction_handler(direction: str, clicks: int = 1) -> Dict[str, Any]:
+ """
+ Scroll in a specified direction.
+
+ Args:
+ direction: One of 'up', 'down', 'left', 'right'
+ clicks: Number of scroll clicks (default: 1)
+ """
+ direction = direction.lower()
+ if direction == "down":
+ return await automation_handler.scroll_down(clicks)
+ elif direction == "up":
+ return await automation_handler.scroll_up(clicks)
+ elif direction in ("left", "right"):
+ # For horizontal scrolling, use scroll with x amount
+ x_amount = -300 if direction == "left" else 300
+ return await automation_handler.scroll(x_amount * clicks, 0)
+ else:
+ raise ValueError(f"Invalid direction: {direction}. Use 'up', 'down', 'left', or 'right'.")
+
+
+# Command aliases for common alternative names
+COMMAND_ALIASES = {
+ "type": "type_text",
+ "key": "press_key",
+ "tap": "left_click",
+ "click": "left_click",
+ "shell": "run_command",
+ "exec": "run_command",
+ "read_file": "read_text",
+ "write_file": "write_text",
+ "ls": "list_dir",
+ "mkdir": "create_dir",
+ "rm": "delete_file",
+ "rmdir": "delete_dir",
+}
+
handlers = {
"version": lambda: {"protocol": protocol_version, "package": package_version},
# App-Use commands
@@ -165,6 +203,7 @@ handlers = {
"scroll": automation_handler.scroll,
"scroll_down": automation_handler.scroll_down,
"scroll_up": automation_handler.scroll_up,
+ "scroll_direction": _scroll_direction_handler,
# Screen actions
"screenshot": automation_handler.screenshot,
"get_cursor_position": automation_handler.get_cursor_position,
@@ -284,6 +323,23 @@ manager = ConnectionManager()
auth_manager = AuthenticationManager()
+def _resolve_command(command: str) -> str:
+ """Resolve command aliases to their canonical names."""
+ return COMMAND_ALIASES.get(command, command)
+
+
+def _suggest_similar_commands(command: str) -> List[str]:
+ """Suggest similar commands for typos."""
+ all_commands = list(handlers.keys()) + list(COMMAND_ALIASES.keys())
+ suggestions = []
+ command_lower = command.lower()
+ for cmd in all_commands:
+ # Simple similarity: starts with same letter or contains the input
+ if cmd.lower().startswith(command_lower[:2]) or command_lower in cmd.lower():
+ suggestions.append(cmd)
+ return suggestions[:5]
+
+
@app.get("/status")
async def status():
# get computer-server features
@@ -295,6 +351,42 @@ async def status():
return {"status": "ok", "os_type": OS_TYPE, "features": features}
[email protected]("/commands")
+async def list_commands():
+ """List all available commands and their aliases."""
+ commands_info = {}
+ for cmd_name, handler_func in handlers.items():
+ try:
+ sig = inspect.signature(handler_func)
+ params = [
+ {
+ "name": p.name,
+ "required": p.default == inspect.Parameter.empty,
+ "default": None if p.default == inspect.Parameter.empty else p.default,
+ }
+ for p in sig.parameters.values()
+ ]
+ except (ValueError, TypeError):
+ params = []
+ commands_info[cmd_name] = {"params": params}
+
+ # Add alias information
+ aliases_by_command: Dict[str, List[str]] = {}
+ for alias, canonical in COMMAND_ALIASES.items():
+ if canonical not in aliases_by_command:
+ aliases_by_command[canonical] = []
+ aliases_by_command[canonical].append(alias)
+
+ for cmd_name in commands_info:
+ if cmd_name in aliases_by_command:
+ commands_info[cmd_name]["aliases"] = aliases_by_command[cmd_name]
+
+ return {
+ "commands": commands_info,
+ "aliases": COMMAND_ALIASES,
+ }
+
+
@app.websocket("/ws", name="websocket_endpoint")
async def websocket_endpoint(websocket: WebSocket):
global handlers
@@ -366,10 +458,15 @@ async def websocket_endpoint(websocket: WebSocket):
command = data.get("command")
params = data.get("params", {})
+ # Resolve command aliases
+ command = _resolve_command(command)
+
if command not in handlers:
- await websocket.send_json(
- {"success": False, "error": f"Unknown command: {command}"}
- )
+ suggestions = _suggest_similar_commands(command)
+ error_msg = f"Unknown command: {command}"
+ if suggestions:
+ error_msg += f". Did you mean: {', '.join(suggestions)}?"
+ await websocket.send_json({"success": False, "error": error_msg})
continue
try:
@@ -448,6 +545,9 @@ async def cmd_endpoint(
if not command:
raise HTTPException(status_code=400, detail="Command is required")
+ # Resolve command aliases
+ command = _resolve_command(command)
+
# Check if CONTAINER_NAME is set (indicating cloud provider)
server_container_name = os.environ.get("CONTAINER_NAME")
@@ -470,7 +570,11 @@ async def cmd_endpoint(
raise HTTPException(status_code=401, detail="Authentication failed")
if command not in handlers:
- raise HTTPException(status_code=400, detail=f"Unknown command: {command}")
+ suggestions = _suggest_similar_commands(command)
+ error_msg = f"Unknown command: {command}"
+ if suggestions:
+ error_msg += f". Did you mean: {', '.join(suggestions)}?"
+ raise HTTPException(status_code=400, detail=error_msg)
async def generate_response():
"""Generate streaming response for the command execution"""