Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Added reference documentation for: libs/python/computer-server/computer_server/handlers/linux.py
Andrei Onel890fcfdeb313
diff --git a/libs/python/computer-server/computer_server/handlers/linux.py b/libs/python/computer-server/computer_server/handlers/linux.pyindex 34a63de5b2..82fc51c9be 100644--- a/libs/python/computer-server/computer_server/handlers/linux.py+++ b/libs/python/computer-server/computer_server/handlers/linux.py@@ -38,7 +38,12 @@ class LinuxAccessibilityHandler(BaseAccessibilityHandler):"""Linux implementation of accessibility handler."""async def get_accessibility_tree(self) -> Dict[str, Any]:- """Get the accessibility tree of the current window."""+ """Get the accessibility tree of the current window.++ Returns:+ Dict[str, Any]: A dictionary containing success status and a simulated tree structure+ since Linux doesn't have equivalent accessibility API like macOS.+ """# Linux doesn't have equivalent accessibility API like macOS# Return a minimal dummy treelogger.info("Getting accessibility tree (simulated, no accessibility API available on Linux)")@@ -56,7 +61,16 @@ class LinuxAccessibilityHandler(BaseAccessibilityHandler):async def find_element(self, role: Optional[str] = None,title: Optional[str] = None,value: Optional[str] = None) -> Dict[str, Any]:- """Find an element in the accessibility tree by criteria."""+ """Find an element in the accessibility tree by criteria.++ Args:+ role: The role of the element to find.+ title: The title of the element to find.+ value: The value of the element to find.++ Returns:+ Dict[str, Any]: A dictionary indicating that element search is not supported on Linux.+ """logger.info(f"Finding element with role={role}, title={title}, value={value} (not supported on Linux)")return {"success": False,@@ -64,7 +78,12 @@ class LinuxAccessibilityHandler(BaseAccessibilityHandler):}def get_cursor_position(self) -> Tuple[int, int]:- """Get the current cursor position."""+ """Get the current cursor position.++ Returns:+ Tuple[int, int]: The x and y coordinates of the cursor position.+ Returns (0, 0) if pyautogui is not available.+ """try:pos = pyautogui.position()return pos.x, pos.y@@ -75,7 +94,12 @@ class LinuxAccessibilityHandler(BaseAccessibilityHandler):return 0, 0def get_screen_size(self) -> Tuple[int, int]:- """Get the screen size."""+ """Get the screen size.++ Returns:+ Tuple[int, int]: The width and height of the screen in pixels.+ Returns (1920, 1080) if pyautogui is not available.+ """try:size = pyautogui.size()return size.width, size.height@@ -91,6 +115,16 @@ class LinuxAutomationHandler(BaseAutomationHandler):# Mouse Actionsasync def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:+ """Press and hold a mouse button at the specified coordinates.++ Args:+ x: The x coordinate to move to before pressing. If None, uses current position.+ y: The y coordinate to move to before pressing. If None, uses current position.+ button: The mouse button to press ("left", "right", or "middle").++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:if x is not None and y is not None:pyautogui.moveTo(x, y)@@ -100,6 +134,16 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def mouse_up(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:+ """Release a mouse button at the specified coordinates.++ Args:+ x: The x coordinate to move to before releasing. If None, uses current position.+ y: The y coordinate to move to before releasing. If None, uses current position.+ button: The mouse button to release ("left", "right", or "middle").++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:if x is not None and y is not None:pyautogui.moveTo(x, y)@@ -109,6 +153,15 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:+ """Move the cursor to the specified coordinates.++ Args:+ x: The x coordinate to move to.+ y: The y coordinate to move to.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.moveTo(x, y)return {"success": True}@@ -116,6 +169,15 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ """Perform a left mouse click at the specified coordinates.++ Args:+ x: The x coordinate to click at. If None, clicks at current position.+ y: The y coordinate to click at. If None, clicks at current position.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:if x is not None and y is not None:pyautogui.moveTo(x, y)@@ -125,6 +187,15 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ """Perform a right mouse click at the specified coordinates.++ Args:+ x: The x coordinate to click at. If None, clicks at current position.+ y: The y coordinate to click at. If None, clicks at current position.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:if x is not None and y is not None:pyautogui.moveTo(x, y)@@ -134,6 +205,15 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def double_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ """Perform a double click at the specified coordinates.++ Args:+ x: The x coordinate to double click at. If None, clicks at current position.+ y: The y coordinate to double click at. If None, clicks at current position.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:if x is not None and y is not None:pyautogui.moveTo(x, y)@@ -143,6 +223,16 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def click(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left") -> Dict[str, Any]:+ """Perform a mouse click with the specified button at the given coordinates.++ Args:+ x: The x coordinate to click at. If None, clicks at current position.+ y: The y coordinate to click at. If None, clicks at current position.+ button: The mouse button to click ("left", "right", or "middle").++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:if x is not None and y is not None:pyautogui.moveTo(x, y)@@ -152,6 +242,17 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def drag_to(self, x: int, y: int, button: str = "left", duration: float = 0.5) -> Dict[str, Any]:+ """Drag from the current position to the specified coordinates.++ Args:+ x: The x coordinate to drag to.+ y: The y coordinate to drag to.+ button: The mouse button to use for dragging.+ duration: The time in seconds to take for the drag operation.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.dragTo(x, y, duration=duration, button=button)return {"success": True}@@ -159,6 +260,18 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def drag(self, start_x: int, start_y: int, end_x: int, end_y: int, button: str = "left") -> Dict[str, Any]:+ """Drag from start coordinates to end coordinates.++ Args:+ start_x: The starting x coordinate.+ start_y: The starting y coordinate.+ end_x: The ending x coordinate.+ end_y: The ending y coordinate.+ button: The mouse button to use for dragging.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.moveTo(start_x, start_y)pyautogui.dragTo(end_x, end_y, duration=0.5, button=button)@@ -167,6 +280,16 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def drag_path(self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5) -> Dict[str, Any]:+ """Drag along a path defined by a list of coordinates.++ Args:+ path: A list of (x, y) coordinate tuples defining the drag path.+ button: The mouse button to use for dragging.+ duration: The time in seconds to take for each segment of the drag.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:if not path:return {"success": False, "error": "Path is empty"}@@ -179,6 +302,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):# Keyboard Actionsasync def key_down(self, key: str) -> Dict[str, Any]:+ """Press and hold a key.++ Args:+ key: The key to press down.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.keyDown(key)return {"success": True}@@ -186,6 +317,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def key_up(self, key: str) -> Dict[str, Any]:+ """Release a key.++ Args:+ key: The key to release.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.keyUp(key)return {"success": True}@@ -193,6 +332,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def type_text(self, text: str) -> Dict[str, Any]:+ """Type the specified text using the keyboard.++ Args:+ text: The text to type.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:# use pynput for Unicode supportself.keyboard.type(text)@@ -201,6 +348,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def press_key(self, key: str) -> Dict[str, Any]:+ """Press and release a key.++ Args:+ key: The key to press.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.press(key)return {"success": True}@@ -208,6 +363,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def hotkey(self, keys: List[str]) -> Dict[str, Any]:+ """Press a combination of keys simultaneously.++ Args:+ keys: A list of keys to press together as a hotkey combination.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.hotkey(*keys)return {"success": True}@@ -216,6 +379,15 @@ class LinuxAutomationHandler(BaseAutomationHandler):# Scrolling Actionsasync def scroll(self, x: int, y: int) -> Dict[str, Any]:+ """Scroll the mouse wheel.++ Args:+ x: The horizontal scroll amount.+ y: The vertical scroll amount.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.scroll(x, y)return {"success": True}@@ -223,6 +395,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:+ """Scroll down by the specified number of clicks.++ Args:+ clicks: The number of scroll clicks to perform downward.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.scroll(-clicks)return {"success": True}@@ -230,6 +410,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:+ """Scroll up by the specified number of clicks.++ Args:+ clicks: The number of scroll clicks to perform upward.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:pyautogui.scroll(clicks)return {"success": True}@@ -238,6 +426,12 @@ class LinuxAutomationHandler(BaseAutomationHandler):# Screen Actionsasync def screenshot(self) -> Dict[str, Any]:+ """Take a screenshot of the current screen.++ Returns:+ Dict[str, Any]: A dictionary containing success status and base64-encoded image data,+ or error message if failed.+ """try:from PIL import Imagescreenshot = pyautogui.screenshot()@@ -252,6 +446,12 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": f"Screenshot error: {str(e)}"}async def get_screen_size(self) -> Dict[str, Any]:+ """Get the size of the screen.++ Returns:+ Dict[str, Any]: A dictionary containing success status and screen dimensions,+ or error message if failed.+ """try:size = pyautogui.size()return {"success": True, "size": {"width": size.width, "height": size.height}}@@ -259,6 +459,12 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def get_cursor_position(self) -> Dict[str, Any]:+ """Get the current position of the cursor.++ Returns:+ Dict[str, Any]: A dictionary containing success status and cursor coordinates,+ or error message if failed.+ """try:pos = pyautogui.position()return {"success": True, "position": {"x": pos.x, "y": pos.y}}@@ -267,6 +473,12 @@ class LinuxAutomationHandler(BaseAutomationHandler):# Clipboard Actionsasync def copy_to_clipboard(self) -> Dict[str, Any]:+ """Get the current content of the clipboard.++ Returns:+ Dict[str, Any]: A dictionary containing success status and clipboard content,+ or error message if failed.+ """try:import pyperclipcontent = pyperclip.paste()@@ -275,6 +487,14 @@ class LinuxAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def set_clipboard(self, text: str) -> Dict[str, Any]:+ """Set the clipboard content to the specified text.++ Args:+ text: The text to copy to the clipboard.++ Returns:+ Dict[str, Any]: A dictionary with success status and error message if failed.+ """try:import pyperclippyperclip.copy(text)@@ -284,6 +504,15 @@ class LinuxAutomationHandler(BaseAutomationHandler):# Command Executionasync def run_command(self, command: str) -> Dict[str, Any]:+ """Execute a shell command asynchronously.++ Args:+ command: The shell command to execute.++ Returns:+ Dict[str, Any]: A dictionary containing success status, stdout, stderr,+ and return code, or error message if failed.+ """try:# Create subprocessprocess = await asyncio.create_subprocess_shell(