Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(computer-server): MCP handler consistency and uiautomator fix (#881)
Francesco Bonacci09e18ec2342f
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.pyindex 26350cea1d..90c582fc2a 100644--- a/libs/python/computer-server/computer_server/handlers/android.py+++ b/libs/python/computer-server/computer_server/handlers/android.py@@ -51,8 +51,14 @@ class AndroidAccessibilityHandler(BaseAccessibilityHandler):await adb_exec.run("shell", "rm", "/sdcard/ui_dump.xml", decode=True)try:+ # Strip the status message prefix (e.g., "UI hierchary dumped to: /sdcard/ui_dump.xml\n")+ xml_start = output.find("<?xml")+ if xml_start == -1:+ raise RuntimeError(f"No XML found in UI dump output: {output[:200]}")+ xml_content = output[xml_start:].strip()+# Parse XML- root = ET.fromstring(output.strip())+ root = ET.fromstring(xml_content)return {"tree": self._parse_accessibility_node(root)}except ET.ParseError as e:raise RuntimeError(f"Failed to parse UI hierarchy XML: {e}")diff --git a/libs/python/computer-server/computer_server/mcp_server.py b/libs/python/computer-server/computer_server/mcp_server.pyindex b40110a856..1a6c8676cd 100644--- a/libs/python/computer-server/computer_server/mcp_server.py+++ b/libs/python/computer-server/computer_server/mcp_server.py@@ -42,12 +42,27 @@ def _get_handlers():return _handlers+async def _detect_actual_resolution_async() -> Tuple[int, int]:+ """Detect the actual screen resolution using the automation handler."""+ try:+ _, automation_handler, _, _, _, _ = _get_handlers()+ result = await automation_handler.get_screen_size()+ return result["width"], result["height"]+ except Exception as e:+ logger.warning(f"Failed to detect resolution via handler: {e}")+ return 1920, 1080 # Default fallback++def _detect_actual_resolution() -> Tuple[int, int]:- """Detect the actual screen resolution using PIL ImageGrab."""+ """Detect the actual screen resolution (sync wrapper)."""+ import asynciotry:- from PIL import ImageGrab- img = ImageGrab.grab()- return img.width, img.height+ loop = asyncio.get_event_loop()+ if loop.is_running():+ # If called from async context, can't use run_until_complete+ # Return default and let async init handle it+ return 1920, 1080+ return loop.run_until_complete(_detect_actual_resolution_async())except Exception as e:logger.warning(f"Failed to detect resolution: {e}")return 1920, 1080 # Default fallback@@ -159,13 +174,9 @@ def create_mcp_server() -> FastMCP:# Return target resolution if scaling is configuredif _target_width is not None and _target_height is not None:return {"width": int(_target_width), "height": int(_target_height)}- # Fallback: detect resolution directly- try:- from PIL import ImageGrab- img = ImageGrab.grab()- return {"width": img.width, "height": img.height}- except Exception:- return {"width": 1920, "height": 1080}+ # Use handler to get screen size+ _, automation_handler, _, _, _, _ = _get_handlers()+ return await automation_handler.get_screen_size()@mcp.toolasync def computer_get_cursor_position() -> Dict[str, Any]:@@ -175,16 +186,13 @@ def create_mcp_server() -> FastMCP:Returns:Dictionary with 'x' and 'y' coordinates."""- # Get cursor position using pynput directly for reliability- try:- from pynput.mouse import Controller- mouse = Controller()- x, y = mouse.position- # Scale to target coordinates if scaling is configured- scaled_x, scaled_y = _scale_to_target(int(x), int(y))+ _, automation_handler, _, _, _, _ = _get_handlers()+ result = await automation_handler.get_cursor_position()+ # Scale to target coordinates if scaling is configured+ if "x" in result and "y" in result:+ scaled_x, scaled_y = _scale_to_target(int(result["x"]), int(result["y"]))return {"x": scaled_x, "y": scaled_y}- except Exception:- return {"x": 0, "y": 0}+ return result@mcp.toolasync def computer_click(