Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
add get_keyboard_focus to computer_server
Dillon DuPontfffe51be1b1b
diff --git a/libs/computer-server/computer_server/handlers/base.py b/libs/computer-server/computer_server/handlers/base.pyindex 08d57ad5b8..95697ce685 100644--- a/libs/computer-server/computer_server/handlers/base.py+++ b/libs/computer-server/computer_server/handlers/base.py@@ -16,6 +16,11 @@ class BaseAccessibilityHandler(ABC):"""Find an element in the accessibility tree by criteria."""pass+ @abstractmethod+ async def get_keyboard_focus(self) -> Dict[str, Any]:+ """Get the currently focused UI element."""+ pass+class BaseAutomationHandler(ABC):"""Abstract base class for OS-specific automation handlers.diff --git a/libs/computer-server/computer_server/handlers/macos.py b/libs/computer-server/computer_server/handlers/macos.pyindex abdedc410e..10df3636f0 100644--- a/libs/computer-server/computer_server/handlers/macos.py+++ b/libs/computer-server/computer_server/handlers/macos.py@@ -33,6 +33,8 @@ from ApplicationServices import (AXValueGetValue, # type: ignorekAXVisibleChildrenAttribute, # type: ignorekAXRoleDescriptionAttribute, # type: ignore+ kAXFocusedApplicationAttribute, # type: ignore+ kAXFocusedUIElementAttribute, # type: ignore)import objcimport re@@ -514,6 +516,68 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler):except Exception as e:return {"success": False, "error": str(e)}++ async def get_keyboard_focus(self) -> Dict[str, Any]:+ """Get the coordinates of the currently focused UI element.++ Returns:+ A dictionary with success status and coordinates if found.+ """+ try:+ # Create system-wide accessibility object+ system = AXUIElementCreateSystemWide()++ # Get focused application+ err, focused_app = AXUIElementCopyAttributeValue(system, kAXFocusedApplicationAttribute, None)+ if err != kAXErrorSuccess or not focused_app:+ return {"success": False, "error": "Could not get focused application"}++ # Get focused UI element+ err, focused_element = AXUIElementCopyAttributeValue(focused_app, kAXFocusedUIElementAttribute, None)+ if err != kAXErrorSuccess or not focused_element:+ return {"success": False, "error": "Could not get focused UI element"}++ # Get position of focused element+ position = self.get_ax_attribute(focused_element, kAXPositionAttribute)+ if not position:+ return {"success": False, "error": "Could not get position of focused element"}++ # Get size of focused element+ size = self.get_ax_attribute(focused_element, kAXSizeAttribute)+ if not size:+ return {"success": False, "error": "Could not get size of focused element"}++ # Convert position to point+ position_point = element_value(position, kAXValueCGPointType)+ if not position_point:+ return {"success": False, "error": "Could not convert position to point"}++ # Convert size to CGSize+ size_value = element_value(size, kAXValueCGSizeType)+ if not size_value:+ return {"success": False, "error": "Could not convert size to CGSize"}++ # Calculate center point of the element+ center_x = position_point.x + (size_value.width / 2)+ center_y = position_point.y + (size_value.height / 2)++ # Get additional information about the focused element+ role = self.get_ax_attribute(focused_element, kAXRoleAttribute)+ title = self.get_ax_attribute(focused_element, kAXTitleAttribute)+ value = self.get_ax_attribute(focused_element, kAXValueAttribute)++ return {+ "success": True,+ "position": {"x": position_point.x, "y": position_point.y},+ "size": {"width": size_value.width, "height": size_value.height},+ "center": {"x": center_x, "y": center_y},+ "role": role,+ "title": title,+ "value": value+ }++ except Exception as e:+ return {"success": False, "error": str(e)}class MacOSAutomationHandler(BaseAutomationHandler):diff --git a/libs/computer-server/computer_server/main.py b/libs/computer-server/computer_server/main.pyindex d7f66f89a9..ab893ed18d 100644--- a/libs/computer-server/computer_server/main.py+++ b/libs/computer-server/computer_server/main.py@@ -54,6 +54,7 @@ async def websocket_endpoint(websocket: WebSocket):# Accessibility commands"get_accessibility_tree": manager.accessibility_handler.get_accessibility_tree,"find_element": manager.accessibility_handler.find_element,+ "get_keyboard_focus": manager.accessibility_handler.get_keyboard_focus,# Automation commands"screenshot": manager.automation_handler.screenshot,"left_click": manager.automation_handler.left_click,diff --git a/libs/computer/computer/interface/base.py b/libs/computer/computer/interface/base.pyindex 8fcbd21ca6..0b9f8b7478 100644--- a/libs/computer/computer/interface/base.py+++ b/libs/computer/computer/interface/base.py@@ -173,6 +173,11 @@ class BaseComputerInterface(ABC):async def get_accessibility_tree(self) -> Dict:"""Get the accessibility tree of the current screen."""pass++ @abstractmethod+ async def get_keyboard_focus(self) -> Dict:+ """Get the currently focused UI element."""+ pass@abstractmethodasync def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:diff --git a/libs/computer/computer/interface/macos.py b/libs/computer/computer/interface/macos.pyindex dcac634208..a648352ac4 100644--- a/libs/computer/computer/interface/macos.py+++ b/libs/computer/computer/interface/macos.py@@ -532,6 +532,13 @@ class MacOSComputerInterface(BaseComputerInterface):if not result.get("success", False):raise RuntimeError(result.get("error", "Failed to get accessibility tree"))return result++ async def get_keyboard_focus(self) -> Dict[str, Any]:+ """Get the currently focused UI element."""+ result = await self._send_command("get_keyboard_focus")+ if not result.get("success", False):+ raise RuntimeError(result.get("error", "Failed to get keyboard focus"))+ return resultasync def get_active_window_bounds(self) -> Dict[str, int]:"""Get the bounds of the currently active window."""