Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added AXFocused attribute
Dillon DuPontbc8f0d9f54af
diff --git a/libs/computer-server/computer_server/handlers/macos.py b/libs/computer-server/computer_server/handlers/macos.pyindex a4ad6f5fed..8664179b73 100644--- a/libs/computer-server/computer_server/handlers/macos.py+++ b/libs/computer-server/computer_server/handlers/macos.py@@ -134,6 +134,9 @@ class UIElement:self.role_description = ""self.value = Noneself.max_depth = max_depth+ self.selected_text = None+ self.selected_range = None+ self.focused = False# Set roleself.role = element_attribute(element, kAXRoleAttribute)@@ -152,6 +155,21 @@ class UIElement:self.enabled = element_attribute(element, kAXEnabledAttribute)if self.enabled is None:self.enabled = False++ # Check if element is focused - using a simpler approach+ try:+ # Get the focused state directly from the element if available+ # focused_attr = element_attribute(element, "AXFocused")+ # if focused_attr is not None and focused_attr is True:+ # self.focused = True++ # Get the focused element+ focused_element = element_attribute(element, kAXFocusedUIElementAttribute)+ if focused_element and Foundation.CFEqual(focused_element, element):+ self.focused = True+ except:+ # If there's an error, assume it's not focused+ pass# Set position and sizeposition = element_attribute(element, kAXPositionAttribute)@@ -194,6 +212,14 @@ class UIElement:# Check if it's an accessibility element by checking its type IDelif Foundation.CFGetTypeID(attribute_value) == AXUIElementGetTypeID(): # type: ignoreself.value = UIElement(attribute_value, offset_x, offset_y)++ # Get selected text if available+ self.selected_text = element_attribute(element, kAXSelectedTextAttribute)++ # Get selected text range if available+ selected_range_value = element_attribute(element, kAXSelectedTextRangeAttribute)+ if selected_range_value:+ self.selected_range = element_value(selected_range_value, kAXValueCFRangeType)# Set childrenif self.max_depth is None or self.max_depth > 0:@@ -321,8 +347,8 @@ class UIElement:size = f"{self.size.width:.0f};{self.size.height:.0f}"else:size = ""-- return {++ result = {"id": self.identifier,"name": self.name,"role": self.role,@@ -333,10 +359,23 @@ class UIElement:"position": position,"size": size,"enabled": self.enabled,+ "focused": self.focused,"bbox": self.bbox,"visible_bbox": self.visible_bbox,"children": children_to_dict(self.children),}++ # Add selected text information if available+ if self.selected_text:+ result["selected_text"] = self.selected_text++ if self.selected_range:+ result["selected_range"] = {+ "location": self.selected_range.location,+ "length": self.selected_range.length+ }++ return resultclass MacOSAccessibilityHandler(BaseAccessibilityHandler):