Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Merge branch 'main' into feat/pynput-server
Adam76c0ca430261
diff --cc libs/python/computer-server/computer_server/diorama/diorama.pyindex e26f82959f,f5e34cacbe..710140bab0--- a/libs/python/computer-server/computer_server/diorama/diorama.py+++ b/libs/python/computer-server/computer_server/diorama/diorama.py@@@ -466,9 -483,12 +483,10 @@@ class Dioramareturn tx, tyreturn x, y-import time--import pyautogui+from pynput.mouse import Controller as MouseController+async def main():"""Main function demonstrating Diorama usage with multiple desktops and mouse tracking."""desktop1 = Diorama.create_from_apps(["Discord", "Notes"])diff --cc libs/python/computer-server/computer_server/handlers/linux.pyindex 3304858752,bdafb1567c..eebb87e344--- a/libs/python/computer-server/computer_server/handlers/linux.py+++ b/libs/python/computer-server/computer_server/handlers/linux.py@@@ -7,23 -7,34 +7,27 @@@ To use GUI automation in a headless env1. Install Xvfb: sudo apt-get install xvfb2. Run with virtual display: xvfb-run python -m computer_server"""- from typing import Dict, Any, List, Tuple, Optional- import logging- import subprocess+import asyncioimport base64- import osimport json+ import logging+ import os+ import subprocessfrom io import BytesIO- from PIL import ImageGrab, Image+ from typing import Any, Dict, List, Optional, Tuple+++from PIL import Image, ImageGrab+# Configure loggerlogger = logging.getLogger(__name__)-# Try to import pyautogui, but don't fail if it's not available-# This allows the server to run in headless environments-try:- import pyautogui-- pyautogui.FAILSAFE = False-- logger.info("pyautogui successfully imported, GUI automation available")-except Exception as e:- logger.warning(f"pyautogui import failed: {str(e)}. GUI operations will be simulated.")+# pyautogui removed in favor of pynput- from pynput.mouse import Button, Controller as MouseController- from pynput.keyboard import Key, Controller as KeyboardController+ from pynput.keyboard import Controller as KeyboardController+ from pynput.keyboard import Key+ from pynput.mouse import Button+ from pynput.mouse import Controller as MouseControllerfrom .base import BaseAccessibilityHandler, BaseAutomationHandler@@@ -78,40 -91,43 +84,42 @@@ class LinuxAccessibilityHandler(BaseAccReturns (0, 0) if pyautogui is not available."""try:- pos = pyautogui.position()- return pos.x, pos.y+ # Use pynput mouse controller+ from pynput.mouse import Controller as MouseController+ m = MouseController()+ x, y = m.position+ return int(x), int(y)except Exception as e:- logger.warning(f"Failed to get cursor position with pyautogui: {e}")-- logger.info("Getting cursor position (simulated)")- return 0, 0-+ logger.warning(f"Failed to get cursor position: {e}")+ logger.info("Getting cursor position (simulated)")+ return 0, 0-def get_screen_size(self) -> Tuple[int, int]:"""Get the screen size.-+Returns:Tuple[int, int]: The width and height of the screen in pixels.Returns (1920, 1080) if pyautogui is not available."""try:- size = pyautogui.size()- return size.width, size.height+ img = ImageGrab.grab()+ return img.width, img.heightexcept Exception as e:- logger.warning(f"Failed to get screen size with pyautogui: {e}")-- logger.info("Getting screen size (simulated)")- return 1920, 1080+ logger.warning(f"Failed to get screen size via ImageGrab: {e}")+ logger.info("Getting screen size (simulated)")+ return 1920, 1080+class LinuxAutomationHandler(BaseAutomationHandler):- """Linux implementation of automation handler using pyautogui."""-+ """Linux implementation of automation handler using pynput."""keyboard = KeyboardController()mouse = MouseController()-+# Mouse Actions- async def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:+ async def mouse_down(+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"+ ) -> Dict[str, Any]:"""Press and hold a mouse button at the specified coordinates.-+Args:x: The x coordinate to move to before pressing. If None, uses current position.y: The y coordinate to move to before pressing. If None, uses current position.@@@ -281,12 -273,7 +274,11 @@@Dict[str, Any]: A dictionary with success status and error message if failed."""try:- pyautogui.dragTo(x, y, duration=duration, button=button)+ from pynput.mouse import Button+ btn = getattr(Button, button if button in ["left","right","middle"] else "left")- self.mouse.position = (start_x, start_y)+ self.mouse.press(btn)- self.mouse.position = (end_x, end_y)++ self.mouse.position = (x, y)+ self.mouse.release(btn)return {"success": True}except Exception as e:return {"success": False, "error": str(e)}diff --cc libs/python/computer-server/computer_server/handlers/macos.pyindex 8f4008cd88,5692bea306..86f11d5a44--- a/libs/python/computer-server/computer_server/handlers/macos.py+++ b/libs/python/computer-server/computer_server/handlers/macos.py@@@ -1,56 -1,88 +1,85 @@@- from pynput.mouse import Button, Controller as MouseController- from pynput.keyboard import Key, Controller as KeyboardController- from PIL import ImageGrab, Image- import time-import pyautogui--pyautogui.FAILSAFE = False+ import asyncioimport base64+ import copy+ import json+ import logging+ import re+ import time+ from ctypes import POINTER, byref, c_void_pfrom io import BytesIO- from typing import Optional, Dict, Any, List, Tuple- from ctypes import byref, c_void_p, POINTER- from AppKit import NSWorkspace # type: ignore+ from typing import Any, Dict, List, Optional, Tuple+import AppKit- from Quartz.CoreGraphics import * # type: ignore- from Quartz.CoreGraphics import CGPoint, CGSize # type: ignoreimport Foundation- from ApplicationServices import (- AXUIElementCreateSystemWide, # type: ignore- AXUIElementCreateApplication, # type: ignore- AXUIElementCopyAttributeValue, # type: ignore- AXUIElementCopyAttributeValues, # type: ignore- kAXFocusedWindowAttribute, # type: ignore- kAXWindowsAttribute, # type: ignore- kAXMainWindowAttribute, # type: ignore- kAXChildrenAttribute, # type: ignore- kAXRoleAttribute, # type: ignore- kAXTitleAttribute, # type: ignore- kAXValueAttribute, # type: ignore- kAXDescriptionAttribute, # type: ignore- kAXEnabledAttribute, # type: ignore- kAXPositionAttribute, # type: ignore- kAXSizeAttribute, # type: ignore- kAXErrorSuccess, # type: ignore- AXValueGetType, # type: ignore- kAXValueCGSizeType, # type: ignore- kAXValueCGPointType, # type: ignore- kAXValueCFRangeType, # type: ignore- AXUIElementGetTypeID, # type: ignore- AXValueGetValue, # type: ignore- kAXVisibleChildrenAttribute, # type: ignore- kAXRoleDescriptionAttribute, # type: ignore- kAXFocusedApplicationAttribute, # type: ignore- kAXFocusedUIElementAttribute, # type: ignore- kAXSelectedTextAttribute, # type: ignore- kAXSelectedTextRangeAttribute, # type: ignore- )import objc- import re- import json- import copy- import asyncio+ from AppKit import NSWorkspace # type: ignore+ from ApplicationServices import AXUIElementCopyAttributeValue # type: ignore+ from ApplicationServices import AXUIElementCopyAttributeValues # type: ignore+ from ApplicationServices import AXUIElementCreateApplication # type: ignore+ from ApplicationServices import AXUIElementCreateSystemWide # type: ignore+ from ApplicationServices import AXUIElementGetTypeID # type: ignore+ from ApplicationServices import AXValueGetType # type: ignore+ from ApplicationServices import AXValueGetValue # type: ignore+ from ApplicationServices import kAXChildrenAttribute # type: ignore+ from ApplicationServices import kAXDescriptionAttribute # type: ignore+ from ApplicationServices import kAXEnabledAttribute # type: ignore+ from ApplicationServices import kAXErrorSuccess # type: ignore+ from ApplicationServices import kAXFocusedApplicationAttribute # type: ignore+ from ApplicationServices import kAXFocusedUIElementAttribute # type: ignore+ from ApplicationServices import kAXFocusedWindowAttribute # type: ignore+ from ApplicationServices import kAXMainWindowAttribute # type: ignore+ from ApplicationServices import kAXPositionAttribute # type: ignore+ from ApplicationServices import kAXRoleAttribute # type: ignore+ from ApplicationServices import kAXRoleDescriptionAttribute # type: ignore+ from ApplicationServices import kAXSelectedTextAttribute # type: ignore+ from ApplicationServices import kAXSelectedTextRangeAttribute # type: ignore+ from ApplicationServices import kAXSizeAttribute # type: ignore+ from ApplicationServices import kAXTitleAttribute # type: ignore+ from ApplicationServices import kAXValueAttribute # type: ignore+ from ApplicationServices import kAXValueCFRangeType # type: ignore+ from ApplicationServices import kAXValueCGPointType # type: ignore+ from ApplicationServices import kAXValueCGSizeType # type: ignore+ from ApplicationServices import kAXVisibleChildrenAttribute # type: ignore+ from ApplicationServices import kAXWindowsAttribute # type: ignore+ from pynput.keyboard import Controller as KeyboardController+ from pynput.keyboard import Key+ from pynput.mouse import Button+ from pynput.mouse import Controller as MouseController+ from Quartz.CoreGraphics import * # type: ignore+ from Quartz.CoreGraphics import CGPoint, CGSize # type: ignore+from .base import BaseAccessibilityHandler, BaseAutomationHandler- import logginglogger = logging.getLogger(__name__)+ # Trigger accessibility permissions prompt on macOS+ try:+ # Source - https://stackoverflow.com/a/17134+ # Posted by Andreas+ # Retrieved 2025-12-03, License - CC BY-SA 4.0+ # Attempt to create and post a mouse event to trigger the permissions prompt+ # This will cause macOS to show "Python would like to control this computer using accessibility features"+ current_pos = CGEventGetLocation(CGEventCreate(None))+ p = CGPoint()+ p.x = current_pos.x+ p.y = current_pos.y++ me = CGEventCreateMouseEvent(None, kCGEventMouseMoved, p, 0)+ if me:+ CGEventPost(kCGHIDEventTap, me)+ CFRelease(me)+ except Exception as e:+ logger.debug(f"Failed to trigger accessibility permissions prompt: {e}")++ # Trigger screen recording prompt on macOS+ try:- import pyautogui++ from PIL import ImageGrab+- pyautogui.screenshot()++ ImageGrab.grab()+ except Exception as e:+ logger.debug(f"Failed to trigger screenshot permissions prompt: {e}")++# Constants for accessibility APIkAXErrorSuccess = 0kAXRoleAttribute = "AXRole"@@@ -663,27 -702,28 +699,30 @@@ def get_dock_items()item_type = "document"elif subrole == "AXSeparatorDockItem" or role == "AXSeparator":item_type = "separator"- elif "trash" in title.lower():- item_type = "trash"+ else:+ title_str = title if isinstance(title, str) else str(title)+ if "trash" in title_str.lower():+ item_type = "trash"- dock_items.append({- "title": title,- "description": description,- "bounds": bounds,- "index": i,- "type": item_type,- "role": role,- "subrole": subrole- })+ dock_items.append(+ {+ "title": title,+ "description": description,+ "bounds": bounds,+ "index": i,+ "type": item_type,+ "role": role,+ "subrole": subrole,+ }+ )return dock_items+class MacOSAccessibilityHandler(BaseAccessibilityHandler):"""Handler for macOS accessibility features and UI element inspection."""-+def get_desktop_state(self):"""Get the current state of the desktop including windows, apps, menubar, and dock.-+Returns:Dictionary containing applications, windows, menubar_items, and dock_items"""@@@ -1115,10 -1176,10 +1175,10 @@@ class MacOSAutomationHandler(BaseAutoma# Keyboard Actionsasync def key_down(self, key: str) -> Dict[str, Any]:"""Press and hold a keyboard key.-+Args:- key: Key name to press (using pyautogui key names)+ key: Key name to press-+Returns:Dictionary containing success status and error message if failed"""@@@ -1130,13 -1189,13 +1190,13 @@@return {"success": True}except Exception as e:return {"success": False, "error": str(e)}-+async def key_up(self, key: str) -> Dict[str, Any]:"""Release a keyboard key.-+Args:- key: Key name to release (using pyautogui key names)+ key: Key name to release-+Returns:Dictionary containing success status and error message if failed"""@@@ -1166,10 -1224,10 +1226,10 @@@async def press_key(self, key: str) -> Dict[str, Any]:"""Press and release a keyboard key.-+Args:- key: Key name to press (using pyautogui key names)+ key: Key name to press-+Returns:Dictionary containing success status and error message if failed"""@@@ -1185,10 -1240,10 +1245,10 @@@async def hotkey(self, keys: List[str]) -> Dict[str, Any]:"""Press a combination of keys simultaneously.-+Args:- keys: List of key names to press together (using pyautogui key names)+ keys: List of key names to press together-+Returns:Dictionary containing success status and error message if failed"""diff --cc libs/python/computer-server/computer_server/handlers/windows.pyindex 523e6398f4,a01456d4d3..76fa5dfcc0--- a/libs/python/computer-server/computer_server/handlers/windows.py+++ b/libs/python/computer-server/computer_server/handlers/windows.py@@@ -1,20 -1,20 +1,21 @@@"""Windows implementation of automation and accessibility handlers.-This implementation uses pyautogui for GUI automation and Windows-specific APIs+This implementation uses pynput for GUI automation and Windows-specific APIsfor accessibility and system operations."""- from typing import Dict, Any, List, Tuple, Optional- import logging- import subprocess+import asyncioimport base64+ import loggingimport os+ import subprocessfrom io import BytesIO- from pynput.mouse import Controller as MouseController, Button as MouseButton-from typing import Any, Dict, List, Optional, Tuple++from typing import Any, Dict, List, Optional, Tuple, Union+-from pynput.keyboard import Controller as KeyboardController-from pynput.mouse import Controller as MouseController++from PIL import Image, ImageGrab+from pynput.keyboard import Controller as KeyboardController, Key as KBKey- from typing import Union, List as _List- from PIL import ImageGrab, Image++from pynput.mouse import Button as MouseButton, Controller as MouseController# Configure loggerlogger = logging.getLogger(__name__)@@@ -160,69 -175,19 +168,72 @@@ class WindowsAccessibilityHandler(BaseAlogger.error(f"Error finding element: {e}")return {"success": False, "error": str(e)}+class WindowsAutomationHandler(BaseAutomationHandler):- """Windows implementation of automation handler using pyautogui and Windows APIs."""+ """Windows implementation of automation handler using pynput and Windows APIs."""-+mouse = MouseController()keyboard = KeyboardController()+ def _map_button(self, button: str) -> MouseButton:+ """Map a string button name to pynput MouseButton."""+ b = (button or "left").lower()+ if b == "left":+ return MouseButton.left+ if b == "right":+ return MouseButton.right+ if b == "middle":+ return MouseButton.middle+ # default to left+ return MouseButton.left++ def _key_from_string(self, key: str):+ """Convert a key string (e.g., 'enter', 'ctrl', 'a') to pynput Key or char."""+ if not key:+ return None+ lk = key.lower()+ special = {+ "enter": KBKey.enter,+ "return": KBKey.enter,+ "esc": KBKey.esc,+ "escape": KBKey.esc,+ "space": KBKey.space,+ "tab": KBKey.tab,+ "backspace": KBKey.backspace,+ "delete": KBKey.delete,+ "home": KBKey.home,+ "end": KBKey.end,+ "pageup": KBKey.page_up,+ "pagedown": KBKey.page_down,+ "up": KBKey.up,+ "down": KBKey.down,+ "left": KBKey.left,+ "right": KBKey.right,+ "shift": KBKey.shift,+ "ctrl": KBKey.ctrl,+ "control": KBKey.ctrl,+ "alt": KBKey.alt,+ "cmd": KBKey.cmd,+ "win": KBKey.cmd,+ "meta": KBKey.cmd,+ "capslock": KBKey.caps_lock,+ "f1": KBKey.f1, "f2": KBKey.f2, "f3": KBKey.f3, "f4": KBKey.f4,+ "f5": KBKey.f5, "f6": KBKey.f6, "f7": KBKey.f7, "f8": KBKey.f8,+ "f9": KBKey.f9, "f10": KBKey.f10, "f11": KBKey.f11, "f12": KBKey.f12,+ }+ if lk in special:+ return special[lk]+ # single character+ if len(key) == 1:+ return key+ return None+# Mouse Actions- async def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:+ async def mouse_down(+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"+ ) -> Dict[str, Any]:"""Press and hold a mouse button at the specified coordinates.-+Args:x (Optional[int]): The x-coordinate to move to before pressing. If None, uses current position.y (Optional[int]): The y-coordinate to move to before pressing. If None, uses current position.@@@ -363,16 -354,17 +382,16 @@@try:if not path:return {"success": False, "error": "Path is empty"}-+# Move to first position- pyautogui.moveTo(*path[0])+ self.mouse.position = path[0]-+# Drag through all positionsfor x, y in path[1:]:- pyautogui.dragTo(x, y, duration=duration / len(path), button=button)+ self.mouse.press(self._map_button(button))+ self.mouse.position = (x, y)+ self.mouse.release(self._map_button(button))-+return {"success": True}except Exception as e:return {"success": False, "error": str(e)}@@@ -457,24 -449,11 +477,24 @@@Returns:Dict[str, Any]: A dictionary with success status and optional error message."""- if not pyautogui:- return {"success": False, "error": "pyautogui not available"}-try:- pyautogui.hotkey(*keys)+ # press keys sequentially while holding modifiers+ resolved = [self._key_from_string(k) for k in keys]+ if any(k is None for k in resolved):+ return {"success": False, "error": "Unknown key in hotkey sequence"}- seq: _List[Union[str, KBKey]] = [k for k in resolved if k is not None] # type: ignore[assignment]++ seq: List[Union[str, KBKey]] = [k for k in resolved if k is not None] # type: ignore[assignment]+ if not seq:+ return {"success": False, "error": "Empty hotkey sequence"}+ # hold all except the last+ for k in seq[:-1]:+ self.keyboard.press(k)+ # tap last+ last = seq[-1]+ self.keyboard.press(last)+ self.keyboard.release(last)+ # release modifiers+ for k in reversed(seq[:-1]):+ self.keyboard.release(k)return {"success": True}except Exception as e:return {"success": False, "error": str(e)}@@@ -534,11 -521,16 +554,11 @@@Returns:Dict[str, Any]: A dictionary containing the success status and eitherbase64-encoded image data or an error message.- Structure: {"success": bool, "image_data": str} or+ Structure: {"success": bool, "image_data": str} or{"success": bool, "error": str}"""- if not pyautogui:- return {"success": False, "error": "pyautogui not available"}-try:- from PIL import Image-- screenshot = pyautogui.screenshot()+ screenshot = ImageGrab.grab()if not isinstance(screenshot, Image.Image):return {"success": False, "error": "Failed to capture screenshot"}