Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
include selected text range
Dillon DuPont10ba905dc7bd
diff --git a/libs/computer-server/computer_server/handlers/macos.py b/libs/computer-server/computer_server/handlers/macos.pyindex 10df3636f0..b76b28c7d2 100644--- a/libs/computer-server/computer_server/handlers/macos.py+++ b/libs/computer-server/computer_server/handlers/macos.py@@ -35,6 +35,8 @@ from ApplicationServices import (kAXRoleDescriptionAttribute, # type: ignorekAXFocusedApplicationAttribute, # type: ignorekAXFocusedUIElementAttribute, # type: ignore+ kAXSelectedTextAttribute, # type: ignore+ kAXSelectedTextRangeAttribute, # type: ignore)import objcimport re@@ -527,13 +529,8 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler):# Create system-wide accessibility objectsystem = AXUIElementCreateSystemWide()- # Get focused application- err, focused_app = AXUIElementCopyAttributeValue(system, kAXFocusedApplicationAttribute, None)- if err != kAXErrorSuccess or not focused_app:- return {"success": False, "error": "Could not get focused application"}-- # Get focused UI element- err, focused_element = AXUIElementCopyAttributeValue(focused_app, kAXFocusedUIElementAttribute, None)+ # Get focused element directly from system-wide element+ err, focused_element = AXUIElementCopyAttributeValue(system, kAXFocusedUIElementAttribute, None)if err != kAXErrorSuccess or not focused_element:return {"success": False, "error": "Could not get focused UI element"}@@ -566,7 +563,16 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler):title = self.get_ax_attribute(focused_element, kAXTitleAttribute)value = self.get_ax_attribute(focused_element, kAXValueAttribute)- return {+ # Check if there's selected text (as mentioned in the blog post)+ selected_text = self.get_ax_attribute(focused_element, kAXSelectedTextAttribute)++ # Get selected text range if available+ selected_range = None+ selected_range_value = self.get_ax_attribute(focused_element, kAXSelectedTextRangeAttribute)+ if selected_range_value:+ selected_range = element_value(selected_range_value, kAXValueCFRangeType)++ result = {"success": True,"position": {"x": position_point.x, "y": position_point.y},"size": {"width": size_value.width, "height": size_value.height},@@ -576,6 +582,18 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler):"value": value}+ # Add selected text information if available+ if selected_text:+ result["selected_text"] = selected_text++ if selected_range:+ result["selected_range"] = {+ "location": selected_range.location,+ "length": selected_range.length+ }++ return result+except Exception as e:return {"success": False, "error": str(e)}