Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Merge branch 'main' into feat/pynput-server

Adam76c0ca430261
diff --cc libs/python/computer-server/computer_server/diorama/diorama.py
index e26f82959f,f5e34cacbe..710140bab0
--- a/libs/python/computer-server/computer_server/diorama/diorama.py
+++ b/libs/python/computer-server/computer_server/diorama/diorama.py
@@@ -466,9 -483,12 +483,10 @@@ class Diorama
return tx, ty
return x, y
-
import time
-
-import pyautogui
+from pynput.mouse import Controller as MouseController
+
async def main():
"""Main function demonstrating Diorama usage with multiple desktops and mouse tracking."""
desktop1 = Diorama.create_from_apps(["Discord", "Notes"])
diff --cc libs/python/computer-server/computer_server/handlers/linux.py
index 3304858752,bdafb1567c..eebb87e344
--- a/libs/python/computer-server/computer_server/handlers/linux.py
+++ b/libs/python/computer-server/computer_server/handlers/linux.py
@@@ -7,23 -7,34 +7,27 @@@ To use GUI automation in a headless env
1. Install Xvfb: sudo apt-get install xvfb
2. Run with virtual display: xvfb-run python -m computer_server
"""
- from typing import Dict, Any, List, Tuple, Optional
- import logging
- import subprocess
+
import asyncio
import base64
- import os
import json
+ import logging
+ import os
+ import subprocess
from io import BytesIO
- from PIL import ImageGrab, Image
+ from typing import Any, Dict, List, Optional, Tuple
+
++from PIL import Image, ImageGrab
+
# Configure logger
logger = logging.getLogger(__name__)
-# Try to import pyautogui, but don't fail if it's not available
-# This allows the server to run in headless environments
-try:
- import pyautogui
-
- pyautogui.FAILSAFE = False
-
- logger.info("pyautogui successfully imported, GUI automation available")
-except Exception as e:
- logger.warning(f"pyautogui import failed: {str(e)}. GUI operations will be simulated.")
+# pyautogui removed in favor of pynput
- from pynput.mouse import Button, Controller as MouseController
- from pynput.keyboard import Key, Controller as KeyboardController
+ from pynput.keyboard import Controller as KeyboardController
+ from pynput.keyboard import Key
+ from pynput.mouse import Button
+ from pynput.mouse import Controller as MouseController
from .base import BaseAccessibilityHandler, BaseAutomationHandler
@@@ -78,40 -91,43 +84,42 @@@ class LinuxAccessibilityHandler(BaseAcc
Returns (0, 0) if pyautogui is not available.
"""
try:
- pos = pyautogui.position()
- return pos.x, pos.y
+ # Use pynput mouse controller
+ from pynput.mouse import Controller as MouseController
+ m = MouseController()
+ x, y = m.position
+ return int(x), int(y)
except Exception as e:
- logger.warning(f"Failed to get cursor position with pyautogui: {e}")
-
- logger.info("Getting cursor position (simulated)")
- return 0, 0
-
+ logger.warning(f"Failed to get cursor position: {e}")
+ logger.info("Getting cursor position (simulated)")
+ return 0, 0
-
def get_screen_size(self) -> Tuple[int, int]:
"""Get the screen size.
-
+
Returns:
Tuple[int, int]: The width and height of the screen in pixels.
Returns (1920, 1080) if pyautogui is not available.
"""
try:
- size = pyautogui.size()
- return size.width, size.height
+ img = ImageGrab.grab()
+ return img.width, img.height
except Exception as e:
- logger.warning(f"Failed to get screen size with pyautogui: {e}")
-
- logger.info("Getting screen size (simulated)")
- return 1920, 1080
+ logger.warning(f"Failed to get screen size via ImageGrab: {e}")
+ logger.info("Getting screen size (simulated)")
+ return 1920, 1080
+
class LinuxAutomationHandler(BaseAutomationHandler):
- """Linux implementation of automation handler using pyautogui."""
-
+ """Linux implementation of automation handler using pynput."""
keyboard = KeyboardController()
mouse = MouseController()
-
+
# Mouse Actions
- async def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:
+ async def mouse_down(
+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+ ) -> Dict[str, Any]:
"""Press and hold a mouse button at the specified coordinates.
-
+
Args:
x: The x coordinate to move to before pressing. If None, uses current position.
y: The y coordinate to move to before pressing. If None, uses current position.
@@@ -281,12 -273,7 +274,11 @@@
Dict[str, Any]: A dictionary with success status and error message if failed.
"""
try:
- pyautogui.dragTo(x, y, duration=duration, button=button)
+ from pynput.mouse import Button
+ btn = getattr(Button, button if button in ["left","right","middle"] else "left")
- self.mouse.position = (start_x, start_y)
+ self.mouse.press(btn)
- self.mouse.position = (end_x, end_y)
++ self.mouse.position = (x, y)
+ self.mouse.release(btn)
return {"success": True}
except Exception as e:
return {"success": False, "error": str(e)}
diff --cc libs/python/computer-server/computer_server/handlers/macos.py
index 8f4008cd88,5692bea306..86f11d5a44
--- a/libs/python/computer-server/computer_server/handlers/macos.py
+++ b/libs/python/computer-server/computer_server/handlers/macos.py
@@@ -1,56 -1,88 +1,85 @@@
- from pynput.mouse import Button, Controller as MouseController
- from pynput.keyboard import Key, Controller as KeyboardController
- from PIL import ImageGrab, Image
- import time
-import pyautogui
-
-pyautogui.FAILSAFE = False
+ import asyncio
import base64
+ import copy
+ import json
+ import logging
+ import re
+ import time
+ from ctypes import POINTER, byref, c_void_p
from io import BytesIO
- from typing import Optional, Dict, Any, List, Tuple
- from ctypes import byref, c_void_p, POINTER
- from AppKit import NSWorkspace # type: ignore
+ from typing import Any, Dict, List, Optional, Tuple
+
import AppKit
- from Quartz.CoreGraphics import * # type: ignore
- from Quartz.CoreGraphics import CGPoint, CGSize # type: ignore
import Foundation
- from ApplicationServices import (
- AXUIElementCreateSystemWide, # type: ignore
- AXUIElementCreateApplication, # type: ignore
- AXUIElementCopyAttributeValue, # type: ignore
- AXUIElementCopyAttributeValues, # type: ignore
- kAXFocusedWindowAttribute, # type: ignore
- kAXWindowsAttribute, # type: ignore
- kAXMainWindowAttribute, # type: ignore
- kAXChildrenAttribute, # type: ignore
- kAXRoleAttribute, # type: ignore
- kAXTitleAttribute, # type: ignore
- kAXValueAttribute, # type: ignore
- kAXDescriptionAttribute, # type: ignore
- kAXEnabledAttribute, # type: ignore
- kAXPositionAttribute, # type: ignore
- kAXSizeAttribute, # type: ignore
- kAXErrorSuccess, # type: ignore
- AXValueGetType, # type: ignore
- kAXValueCGSizeType, # type: ignore
- kAXValueCGPointType, # type: ignore
- kAXValueCFRangeType, # type: ignore
- AXUIElementGetTypeID, # type: ignore
- AXValueGetValue, # type: ignore
- kAXVisibleChildrenAttribute, # type: ignore
- kAXRoleDescriptionAttribute, # type: ignore
- kAXFocusedApplicationAttribute, # type: ignore
- kAXFocusedUIElementAttribute, # type: ignore
- kAXSelectedTextAttribute, # type: ignore
- kAXSelectedTextRangeAttribute, # type: ignore
- )
import objc
- import re
- import json
- import copy
- import asyncio
+ from AppKit import NSWorkspace # type: ignore
+ from ApplicationServices import AXUIElementCopyAttributeValue # type: ignore
+ from ApplicationServices import AXUIElementCopyAttributeValues # type: ignore
+ from ApplicationServices import AXUIElementCreateApplication # type: ignore
+ from ApplicationServices import AXUIElementCreateSystemWide # type: ignore
+ from ApplicationServices import AXUIElementGetTypeID # type: ignore
+ from ApplicationServices import AXValueGetType # type: ignore
+ from ApplicationServices import AXValueGetValue # type: ignore
+ from ApplicationServices import kAXChildrenAttribute # type: ignore
+ from ApplicationServices import kAXDescriptionAttribute # type: ignore
+ from ApplicationServices import kAXEnabledAttribute # type: ignore
+ from ApplicationServices import kAXErrorSuccess # type: ignore
+ from ApplicationServices import kAXFocusedApplicationAttribute # type: ignore
+ from ApplicationServices import kAXFocusedUIElementAttribute # type: ignore
+ from ApplicationServices import kAXFocusedWindowAttribute # type: ignore
+ from ApplicationServices import kAXMainWindowAttribute # type: ignore
+ from ApplicationServices import kAXPositionAttribute # type: ignore
+ from ApplicationServices import kAXRoleAttribute # type: ignore
+ from ApplicationServices import kAXRoleDescriptionAttribute # type: ignore
+ from ApplicationServices import kAXSelectedTextAttribute # type: ignore
+ from ApplicationServices import kAXSelectedTextRangeAttribute # type: ignore
+ from ApplicationServices import kAXSizeAttribute # type: ignore
+ from ApplicationServices import kAXTitleAttribute # type: ignore
+ from ApplicationServices import kAXValueAttribute # type: ignore
+ from ApplicationServices import kAXValueCFRangeType # type: ignore
+ from ApplicationServices import kAXValueCGPointType # type: ignore
+ from ApplicationServices import kAXValueCGSizeType # type: ignore
+ from ApplicationServices import kAXVisibleChildrenAttribute # type: ignore
+ from ApplicationServices import kAXWindowsAttribute # type: ignore
+ from pynput.keyboard import Controller as KeyboardController
+ from pynput.keyboard import Key
+ from pynput.mouse import Button
+ from pynput.mouse import Controller as MouseController
+ from Quartz.CoreGraphics import * # type: ignore
+ from Quartz.CoreGraphics import CGPoint, CGSize # type: ignore
+
from .base import BaseAccessibilityHandler, BaseAutomationHandler
- import logging
logger = logging.getLogger(__name__)
+ # Trigger accessibility permissions prompt on macOS
+ try:
+ # Source - https://stackoverflow.com/a/17134
+ # Posted by Andreas
+ # Retrieved 2025-12-03, License - CC BY-SA 4.0
+ # Attempt to create and post a mouse event to trigger the permissions prompt
+ # This will cause macOS to show "Python would like to control this computer using accessibility features"
+ current_pos = CGEventGetLocation(CGEventCreate(None))
+ p = CGPoint()
+ p.x = current_pos.x
+ p.y = current_pos.y
+
+ me = CGEventCreateMouseEvent(None, kCGEventMouseMoved, p, 0)
+ if me:
+ CGEventPost(kCGHIDEventTap, me)
+ CFRelease(me)
+ except Exception as e:
+ logger.debug(f"Failed to trigger accessibility permissions prompt: {e}")
+
+ # Trigger screen recording prompt on macOS
+ try:
- import pyautogui
++ from PIL import ImageGrab
+
- pyautogui.screenshot()
++ ImageGrab.grab()
+ except Exception as e:
+ logger.debug(f"Failed to trigger screenshot permissions prompt: {e}")
+
+
# Constants for accessibility API
kAXErrorSuccess = 0
kAXRoleAttribute = "AXRole"
@@@ -663,27 -702,28 +699,30 @@@ def get_dock_items()
item_type = "document"
elif subrole == "AXSeparatorDockItem" or role == "AXSeparator":
item_type = "separator"
- elif "trash" in title.lower():
- item_type = "trash"
+ else:
+ title_str = title if isinstance(title, str) else str(title)
+ if "trash" in title_str.lower():
+ item_type = "trash"
- dock_items.append({
- "title": title,
- "description": description,
- "bounds": bounds,
- "index": i,
- "type": item_type,
- "role": role,
- "subrole": subrole
- })
+ dock_items.append(
+ {
+ "title": title,
+ "description": description,
+ "bounds": bounds,
+ "index": i,
+ "type": item_type,
+ "role": role,
+ "subrole": subrole,
+ }
+ )
return dock_items
+
class MacOSAccessibilityHandler(BaseAccessibilityHandler):
"""Handler for macOS accessibility features and UI element inspection."""
-
+
def get_desktop_state(self):
"""Get the current state of the desktop including windows, apps, menubar, and dock.
-
+
Returns:
Dictionary containing applications, windows, menubar_items, and dock_items
"""
@@@ -1115,10 -1176,10 +1175,10 @@@ class MacOSAutomationHandler(BaseAutoma
# Keyboard Actions
async def key_down(self, key: str) -> Dict[str, Any]:
"""Press and hold a keyboard key.
-
+
Args:
- key: Key name to press (using pyautogui key names)
+ key: Key name to press
-
+
Returns:
Dictionary containing success status and error message if failed
"""
@@@ -1130,13 -1189,13 +1190,13 @@@
return {"success": True}
except Exception as e:
return {"success": False, "error": str(e)}
-
+
async def key_up(self, key: str) -> Dict[str, Any]:
"""Release a keyboard key.
-
+
Args:
- key: Key name to release (using pyautogui key names)
+ key: Key name to release
-
+
Returns:
Dictionary containing success status and error message if failed
"""
@@@ -1166,10 -1224,10 +1226,10 @@@
async def press_key(self, key: str) -> Dict[str, Any]:
"""Press and release a keyboard key.
-
+
Args:
- key: Key name to press (using pyautogui key names)
+ key: Key name to press
-
+
Returns:
Dictionary containing success status and error message if failed
"""
@@@ -1185,10 -1240,10 +1245,10 @@@
async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
"""Press a combination of keys simultaneously.
-
+
Args:
- keys: List of key names to press together (using pyautogui key names)
+ keys: List of key names to press together
-
+
Returns:
Dictionary containing success status and error message if failed
"""
diff --cc libs/python/computer-server/computer_server/handlers/windows.py
index 523e6398f4,a01456d4d3..76fa5dfcc0
--- a/libs/python/computer-server/computer_server/handlers/windows.py
+++ b/libs/python/computer-server/computer_server/handlers/windows.py
@@@ -1,20 -1,20 +1,21 @@@
"""
Windows implementation of automation and accessibility handlers.
-This implementation uses pyautogui for GUI automation and Windows-specific APIs
+This implementation uses pynput for GUI automation and Windows-specific APIs
for accessibility and system operations.
"""
- from typing import Dict, Any, List, Tuple, Optional
- import logging
- import subprocess
+
import asyncio
import base64
+ import logging
import os
+ import subprocess
from io import BytesIO
- from pynput.mouse import Controller as MouseController, Button as MouseButton
-from typing import Any, Dict, List, Optional, Tuple
++from typing import Any, Dict, List, Optional, Tuple, Union
+
-from pynput.keyboard import Controller as KeyboardController
-from pynput.mouse import Controller as MouseController
++from PIL import Image, ImageGrab
+from pynput.keyboard import Controller as KeyboardController, Key as KBKey
- from typing import Union, List as _List
- from PIL import ImageGrab, Image
++from pynput.mouse import Button as MouseButton, Controller as MouseController
# Configure logger
logger = logging.getLogger(__name__)
@@@ -160,69 -175,19 +168,72 @@@ class WindowsAccessibilityHandler(BaseA
logger.error(f"Error finding element: {e}")
return {"success": False, "error": str(e)}
+
class WindowsAutomationHandler(BaseAutomationHandler):
- """Windows implementation of automation handler using pyautogui and Windows APIs."""
+ """Windows implementation of automation handler using pynput and Windows APIs."""
-
+
mouse = MouseController()
keyboard = KeyboardController()
+ def _map_button(self, button: str) -> MouseButton:
+ """Map a string button name to pynput MouseButton."""
+ b = (button or "left").lower()
+ if b == "left":
+ return MouseButton.left
+ if b == "right":
+ return MouseButton.right
+ if b == "middle":
+ return MouseButton.middle
+ # default to left
+ return MouseButton.left
+
+ def _key_from_string(self, key: str):
+ """Convert a key string (e.g., 'enter', 'ctrl', 'a') to pynput Key or char."""
+ if not key:
+ return None
+ lk = key.lower()
+ special = {
+ "enter": KBKey.enter,
+ "return": KBKey.enter,
+ "esc": KBKey.esc,
+ "escape": KBKey.esc,
+ "space": KBKey.space,
+ "tab": KBKey.tab,
+ "backspace": KBKey.backspace,
+ "delete": KBKey.delete,
+ "home": KBKey.home,
+ "end": KBKey.end,
+ "pageup": KBKey.page_up,
+ "pagedown": KBKey.page_down,
+ "up": KBKey.up,
+ "down": KBKey.down,
+ "left": KBKey.left,
+ "right": KBKey.right,
+ "shift": KBKey.shift,
+ "ctrl": KBKey.ctrl,
+ "control": KBKey.ctrl,
+ "alt": KBKey.alt,
+ "cmd": KBKey.cmd,
+ "win": KBKey.cmd,
+ "meta": KBKey.cmd,
+ "capslock": KBKey.caps_lock,
+ "f1": KBKey.f1, "f2": KBKey.f2, "f3": KBKey.f3, "f4": KBKey.f4,
+ "f5": KBKey.f5, "f6": KBKey.f6, "f7": KBKey.f7, "f8": KBKey.f8,
+ "f9": KBKey.f9, "f10": KBKey.f10, "f11": KBKey.f11, "f12": KBKey.f12,
+ }
+ if lk in special:
+ return special[lk]
+ # single character
+ if len(key) == 1:
+ return key
+ return None
+
# Mouse Actions
- async def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:
+ async def mouse_down(
+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+ ) -> Dict[str, Any]:
"""Press and hold a mouse button at the specified coordinates.
-
+
Args:
x (Optional[int]): The x-coordinate to move to before pressing. If None, uses current position.
y (Optional[int]): The y-coordinate to move to before pressing. If None, uses current position.
@@@ -363,16 -354,17 +382,16 @@@
try:
if not path:
return {"success": False, "error": "Path is empty"}
-
+
# Move to first position
- pyautogui.moveTo(*path[0])
+ self.mouse.position = path[0]
-
+
# Drag through all positions
for x, y in path[1:]:
- pyautogui.dragTo(x, y, duration=duration / len(path), button=button)
+ self.mouse.press(self._map_button(button))
+ self.mouse.position = (x, y)
+ self.mouse.release(self._map_button(button))
-
+
return {"success": True}
except Exception as e:
return {"success": False, "error": str(e)}
@@@ -457,24 -449,11 +477,24 @@@
Returns:
Dict[str, Any]: A dictionary with success status and optional error message.
"""
- if not pyautogui:
- return {"success": False, "error": "pyautogui not available"}
-
try:
- pyautogui.hotkey(*keys)
+ # press keys sequentially while holding modifiers
+ resolved = [self._key_from_string(k) for k in keys]
+ if any(k is None for k in resolved):
+ return {"success": False, "error": "Unknown key in hotkey sequence"}
- seq: _List[Union[str, KBKey]] = [k for k in resolved if k is not None] # type: ignore[assignment]
++ seq: List[Union[str, KBKey]] = [k for k in resolved if k is not None] # type: ignore[assignment]
+ if not seq:
+ return {"success": False, "error": "Empty hotkey sequence"}
+ # hold all except the last
+ for k in seq[:-1]:
+ self.keyboard.press(k)
+ # tap last
+ last = seq[-1]
+ self.keyboard.press(last)
+ self.keyboard.release(last)
+ # release modifiers
+ for k in reversed(seq[:-1]):
+ self.keyboard.release(k)
return {"success": True}
except Exception as e:
return {"success": False, "error": str(e)}
@@@ -534,11 -521,16 +554,11 @@@
Returns:
Dict[str, Any]: A dictionary containing the success status and either
base64-encoded image data or an error message.
- Structure: {"success": bool, "image_data": str} or
+ Structure: {"success": bool, "image_data": str} or
{"success": bool, "error": str}
"""
- if not pyautogui:
- return {"success": False, "error": "pyautogui not available"}
-
try:
- from PIL import Image
-
- screenshot = pyautogui.screenshot()
+ screenshot = ImageGrab.grab()
if not isinstance(screenshot, Image.Image):
return {"success": False, "error": "Failed to capture screenshot"}