Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(computer-server): MCP handler consistency and uiautomator fix (#881)

Francesco Bonacci09e18ec2342f
diff --git a/libs/python/computer-server/computer_server/handlers/android.py b/libs/python/computer-server/computer_server/handlers/android.py
index 26350cea1d..90c582fc2a 100644
--- a/libs/python/computer-server/computer_server/handlers/android.py
+++ b/libs/python/computer-server/computer_server/handlers/android.py
@@ -51,8 +51,14 @@ class AndroidAccessibilityHandler(BaseAccessibilityHandler):
await adb_exec.run("shell", "rm", "/sdcard/ui_dump.xml", decode=True)
try:
+ # Strip the status message prefix (e.g., "UI hierchary dumped to: /sdcard/ui_dump.xml\n")
+ xml_start = output.find("<?xml")
+ if xml_start == -1:
+ raise RuntimeError(f"No XML found in UI dump output: {output[:200]}")
+ xml_content = output[xml_start:].strip()
+
# Parse XML
- root = ET.fromstring(output.strip())
+ root = ET.fromstring(xml_content)
return {"tree": self._parse_accessibility_node(root)}
except ET.ParseError as e:
raise RuntimeError(f"Failed to parse UI hierarchy XML: {e}")
diff --git a/libs/python/computer-server/computer_server/mcp_server.py b/libs/python/computer-server/computer_server/mcp_server.py
index b40110a856..1a6c8676cd 100644
--- a/libs/python/computer-server/computer_server/mcp_server.py
+++ b/libs/python/computer-server/computer_server/mcp_server.py
@@ -42,12 +42,27 @@ def _get_handlers():
return _handlers
+async def _detect_actual_resolution_async() -> Tuple[int, int]:
+ """Detect the actual screen resolution using the automation handler."""
+ try:
+ _, automation_handler, _, _, _, _ = _get_handlers()
+ result = await automation_handler.get_screen_size()
+ return result["width"], result["height"]
+ except Exception as e:
+ logger.warning(f"Failed to detect resolution via handler: {e}")
+ return 1920, 1080 # Default fallback
+
+
def _detect_actual_resolution() -> Tuple[int, int]:
- """Detect the actual screen resolution using PIL ImageGrab."""
+ """Detect the actual screen resolution (sync wrapper)."""
+ import asyncio
try:
- from PIL import ImageGrab
- img = ImageGrab.grab()
- return img.width, img.height
+ loop = asyncio.get_event_loop()
+ if loop.is_running():
+ # If called from async context, can't use run_until_complete
+ # Return default and let async init handle it
+ return 1920, 1080
+ return loop.run_until_complete(_detect_actual_resolution_async())
except Exception as e:
logger.warning(f"Failed to detect resolution: {e}")
return 1920, 1080 # Default fallback
@@ -159,13 +174,9 @@ def create_mcp_server() -> FastMCP:
# Return target resolution if scaling is configured
if _target_width is not None and _target_height is not None:
return {"width": int(_target_width), "height": int(_target_height)}
- # Fallback: detect resolution directly
- try:
- from PIL import ImageGrab
- img = ImageGrab.grab()
- return {"width": img.width, "height": img.height}
- except Exception:
- return {"width": 1920, "height": 1080}
+ # Use handler to get screen size
+ _, automation_handler, _, _, _, _ = _get_handlers()
+ return await automation_handler.get_screen_size()
@mcp.tool
async def computer_get_cursor_position() -> Dict[str, Any]:
@@ -175,16 +186,13 @@ def create_mcp_server() -> FastMCP:
Returns:
Dictionary with 'x' and 'y' coordinates.
"""
- # Get cursor position using pynput directly for reliability
- try:
- from pynput.mouse import Controller
- mouse = Controller()
- x, y = mouse.position
- # Scale to target coordinates if scaling is configured
- scaled_x, scaled_y = _scale_to_target(int(x), int(y))
+ _, automation_handler, _, _, _, _ = _get_handlers()
+ result = await automation_handler.get_cursor_position()
+ # Scale to target coordinates if scaling is configured
+ if "x" in result and "y" in result:
+ scaled_x, scaled_y = _scale_to_target(int(result["x"]), int(result["y"]))
return {"x": scaled_x, "y": scaled_y}
- except Exception:
- return {"x": 0, "y": 0}
+ return result
@mcp.tool
async def computer_click(