Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Added reference documentation for: libs/python/computer-server/computer_server/handlers/windows.py
Andrei Onelbe339c2f35fe
diff --git a/libs/python/computer-server/computer_server/handlers/windows.py b/libs/python/computer-server/computer_server/handlers/windows.pyindex 2d91ce5336..eeb0d6bf05 100644--- a/libs/python/computer-server/computer_server/handlers/windows.py+++ b/libs/python/computer-server/computer_server/handlers/windows.py@@ -41,7 +41,14 @@ class WindowsAccessibilityHandler(BaseAccessibilityHandler):"""Windows implementation of accessibility handler."""async def get_accessibility_tree(self) -> Dict[str, Any]:- """Get the accessibility tree of the current window."""+ """Get the accessibility tree of the current window.++ Returns:+ Dict[str, Any]: A dictionary containing the success status and either+ the accessibility tree or an error message.+ Structure: {"success": bool, "tree": dict} or+ {"success": bool, "error": str}+ """if not WINDOWS_API_AVAILABLE:return {"success": False, "error": "Windows API not available"}@@ -65,6 +72,15 @@ class WindowsAccessibilityHandler(BaseAccessibilityHandler):# Enumerate child windowsdef enum_child_proc(hwnd_child, children_list):+ """Callback function to enumerate child windows and collect their information.++ Args:+ hwnd_child: Handle to the child window being enumerated.+ children_list: List to append child window information to.++ Returns:+ bool: True to continue enumeration, False to stop.+ """try:child_text = win32gui.GetWindowText(hwnd_child)child_rect = win32gui.GetWindowRect(hwnd_child)@@ -93,7 +109,19 @@ class WindowsAccessibilityHandler(BaseAccessibilityHandler):async def find_element(self, role: Optional[str] = None,title: Optional[str] = None,value: Optional[str] = None) -> Dict[str, Any]:- """Find an element in the accessibility tree by criteria."""+ """Find an element in the accessibility tree by criteria.++ Args:+ role (Optional[str]): The role or class name of the element to find.+ title (Optional[str]): The title or text of the element to find.+ value (Optional[str]): The value of the element (not used in Windows implementation).++ Returns:+ Dict[str, Any]: A dictionary containing the success status and either+ the found element or an error message.+ Structure: {"success": bool, "element": dict} or+ {"success": bool, "error": str}+ """if not WINDOWS_API_AVAILABLE:return {"success": False, "error": "Windows API not available"}@@ -140,6 +168,16 @@ class WindowsAutomationHandler(BaseAutomationHandler):# Mouse Actionsasync def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:+ """Press and hold a mouse button at the specified coordinates.++ Args:+ x (Optional[int]): The x-coordinate to move to before pressing. If None, uses current position.+ y (Optional[int]): The y-coordinate to move to before pressing. If None, uses current position.+ button (str): The mouse button to press ("left", "right", or "middle").++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -152,6 +190,16 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def mouse_up(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:+ """Release a mouse button at the specified coordinates.++ Args:+ x (Optional[int]): The x-coordinate to move to before releasing. If None, uses current position.+ y (Optional[int]): The y-coordinate to move to before releasing. If None, uses current position.+ button (str): The mouse button to release ("left", "right", or "middle").++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -164,6 +212,15 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:+ """Move the mouse cursor to the specified coordinates.++ Args:+ x (int): The x-coordinate to move to.+ y (int): The y-coordinate to move to.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -174,6 +231,15 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ """Perform a left mouse click at the specified coordinates.++ Args:+ x (Optional[int]): The x-coordinate to click at. If None, clicks at current position.+ y (Optional[int]): The y-coordinate to click at. If None, clicks at current position.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -186,6 +252,15 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ """Perform a right mouse click at the specified coordinates.++ Args:+ x (Optional[int]): The x-coordinate to click at. If None, clicks at current position.+ y (Optional[int]): The y-coordinate to click at. If None, clicks at current position.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -198,6 +273,15 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def double_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ """Perform a double left mouse click at the specified coordinates.++ Args:+ x (Optional[int]): The x-coordinate to double-click at. If None, clicks at current position.+ y (Optional[int]): The y-coordinate to double-click at. If None, clicks at current position.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -210,6 +294,17 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def drag_to(self, x: int, y: int, button: str = "left", duration: float = 0.5) -> Dict[str, Any]:+ """Drag from the current position to the specified coordinates.++ Args:+ x (int): The x-coordinate to drag to.+ y (int): The y-coordinate to drag to.+ button (str): The mouse button to use for dragging ("left", "right", or "middle").+ duration (float): The time in seconds to take for the drag operation.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -220,6 +315,16 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def drag(self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5) -> Dict[str, Any]:+ """Drag the mouse through a series of coordinates.++ Args:+ path (List[Tuple[int, int]]): A list of (x, y) coordinate tuples to drag through.+ button (str): The mouse button to use for dragging ("left", "right", or "middle").+ duration (float): The total time in seconds for the entire drag operation.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -240,6 +345,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):# Keyboard Actionsasync def key_down(self, key: str) -> Dict[str, Any]:+ """Press and hold a keyboard key.++ Args:+ key (str): The key to press down (e.g., 'ctrl', 'shift', 'a').++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -250,6 +363,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def key_up(self, key: str) -> Dict[str, Any]:+ """Release a keyboard key.++ Args:+ key (str): The key to release (e.g., 'ctrl', 'shift', 'a').++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -260,6 +381,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def type_text(self, text: str) -> Dict[str, Any]:+ """Type the specified text.++ Args:+ text (str): The text to type.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -270,6 +399,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def press_key(self, key: str) -> Dict[str, Any]:+ """Press and release a keyboard key.++ Args:+ key (str): The key to press (e.g., 'enter', 'space', 'tab').++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -280,6 +417,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def hotkey(self, keys: str) -> Dict[str, Any]:+ """Press a combination of keys simultaneously.++ Args:+ keys (str): The keys to press together (e.g., 'ctrl+c', 'alt+tab').++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -291,6 +436,15 @@ class WindowsAutomationHandler(BaseAutomationHandler):# Scrolling Actionsasync def scroll(self, x: int, y: int) -> Dict[str, Any]:+ """Scroll vertically at the current cursor position.++ Args:+ x (int): Horizontal scroll amount (not used in pyautogui implementation).+ y (int): Vertical scroll amount. Positive values scroll up, negative values scroll down.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -302,6 +456,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:+ """Scroll down by the specified number of clicks.++ Args:+ clicks (int): The number of scroll clicks to perform downward.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -312,6 +474,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:+ """Scroll up by the specified number of clicks.++ Args:+ clicks (int): The number of scroll clicks to perform upward.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -323,6 +493,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):# Screen Actionsasync def screenshot(self) -> Dict[str, Any]:+ """Capture a screenshot of the entire screen.++ Returns:+ Dict[str, Any]: A dictionary containing the success status and either+ base64-encoded image data or an error message.+ Structure: {"success": bool, "image_data": str} or+ {"success": bool, "error": str}+ """if not pyautogui:return {"success": False, "error": "pyautogui not available"}@@ -341,6 +519,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": f"Screenshot error: {str(e)}"}async def get_screen_size(self) -> Dict[str, Any]:+ """Get the size of the screen in pixels.++ Returns:+ Dict[str, Any]: A dictionary containing the success status and either+ screen size information or an error message.+ Structure: {"success": bool, "size": {"width": int, "height": int}} or+ {"success": bool, "error": str}+ """try:if pyautogui:size = pyautogui.size()@@ -356,6 +542,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def get_cursor_position(self) -> Dict[str, Any]:+ """Get the current position of the mouse cursor.++ Returns:+ Dict[str, Any]: A dictionary containing the success status and either+ cursor position or an error message.+ Structure: {"success": bool, "position": {"x": int, "y": int}} or+ {"success": bool, "error": str}+ """try:if pyautogui:pos = pyautogui.position()@@ -371,6 +565,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):# Clipboard Actionsasync def copy_to_clipboard(self) -> Dict[str, Any]:+ """Get the current content of the clipboard.++ Returns:+ Dict[str, Any]: A dictionary containing the success status and either+ clipboard content or an error message.+ Structure: {"success": bool, "content": str} or+ {"success": bool, "error": str}+ """try:import pyperclipcontent = pyperclip.paste()@@ -379,6 +581,14 @@ class WindowsAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def set_clipboard(self, text: str) -> Dict[str, Any]:+ """Set the clipboard content to the specified text.++ Args:+ text (str): The text to copy to the clipboard.++ Returns:+ Dict[str, Any]: A dictionary with success status and optional error message.+ """try:import pyperclippyperclip.copy(text)@@ -388,6 +598,17 @@ class WindowsAutomationHandler(BaseAutomationHandler):# Command Executionasync def run_command(self, command: str) -> Dict[str, Any]:+ """Execute a shell command asynchronously.++ Args:+ command (str): The shell command to execute.++ Returns:+ Dict[str, Any]: A dictionary containing the success status and either+ command output or an error message.+ Structure: {"success": bool, "stdout": str, "stderr": str, "return_code": int} or+ {"success": bool, "error": str}+ """try:# Create subprocessprocess = await asyncio.create_subprocess_shell(