Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): match "..." to the ellipsis in macOS menu paths (#4578)
Ed-Keya207d402d0c6
diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.jsonindex 8a8486b1e6..5402407c76 100644--- a/libs/cua-driver/contract/manifest.json+++ b/libs/cua-driver/contract/manifest.json@@ -2214,7 +2214,7 @@},"input_schema": {"additionalProperties": false,- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).","properties": {"path": {"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",diff --git a/libs/cua-driver/python/src/cua_driver/_native_contract.py b/libs/cua-driver/python/src/cua_driver/_native_contract.pyindex 674ca47c45..ccdc889daa 100644--- a/libs/cua-driver/python/src/cua_driver/_native_contract.py+++ b/libs/cua-driver/python/src/cua_driver/_native_contract.py@@ -3853,7 +3853,9 @@ class InvokeMenuInput:"""Exact, immediate-child application menu path to resolve and invoke throughthe operating system's accessibility API. Path labels are matched after- trimming surrounding whitespace and otherwise remain case-sensitive.+ trimming surrounding whitespace and otherwise remain case-sensitive. On+ macOS, three periods in a label also match the ellipsis character that+ native menu titles use (`Save As...` finds `Save As…`)."""def __init__(self, *, pid:int, window_id:int, path:typing.List[str], session:typing.Optional[str]):self.pid = piddiff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rsindex 66a5c9f779..c612544b37 100644--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs@@ -589,7 +589,9 @@ impl ToolInput for SetWindowFrameInput {/// Exact, immediate-child application menu path to resolve and invoke through/// the operating system's accessibility API. Path labels are matched after-/// trimming surrounding whitespace and otherwise remain case-sensitive.+/// trimming surrounding whitespace and otherwise remain case-sensitive. On+/// macOS, three periods in a label also match the ellipsis character that+/// native menu titles use (`Save As...` finds `Save As…`).#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]#[serde(deny_unknown_fields)]pub struct InvokeMenuInput {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rsindex ebc09104c6..e253aae092 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs@@ -86,6 +86,13 @@ unsafe fn semantic_children(parent: AXUIElementRef) -> Vec<AXUIElementRef> {out}+/// A menu title as paths compare it: trimmed, with three periods read as+/// the ellipsis character macOS menu titles use ("Save As..." finds+/// "Save As…").+fn menu_title_key(title: &str) -> String {+ title.trim().replace("...", "\u{2026}")+}+unsafe fn resolve_exact_prefix(menu_bar: AXUIElementRef,prefix: &[String],@@ -99,10 +106,11 @@ unsafe fn resolve_exact_prefix(CFRelease(current as CFTypeRef);}+ let wanted = menu_title_key(segment);let mut matches = Vec::new();for child in children {let title = copy_string_attr(child, "AXTitle").unwrap_or_default();- if title.trim() == segment {+ if menu_title_key(&title) == wanted {matches.push(child);} else {CFRelease(child as CFTypeRef);@@ -577,6 +585,20 @@ mod tests {}}+ #[test]+ fn three_periods_match_the_ellipsis_in_menu_titles() {+ assert_eq!(+ menu_title_key(" Save As... "),+ menu_title_key("Save As\u{2026}")+ );+ assert_eq!(menu_title_key("Save As..."), menu_title_key("Save As..."));+ assert_ne!(menu_title_key("Save As"), menu_title_key("Save As\u{2026}"));+ assert_ne!(+ menu_title_key("save as..."),+ menu_title_key("Save As\u{2026}")+ );+ }+#[test]fn menu_focus_requires_the_exact_frontmost_app_and_window() {assert!(exact_window_is_ready(Some(7), 7, Some(42), 42));diff --git a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts b/libs/cua-driver/typescript/src/native/cua_driver_contract.tsindex e4067cd8ef..2bb7517083 100644--- a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts+++ b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts@@ -2624,7 +2624,9 @@ const FfiConverterTypeHotkeyInput = (() => {/*** Exact, immediate-child application menu path to resolve and invoke through* the operating system's accessibility API. Path labels are matched after- * trimming surrounding whitespace and otherwise remain case-sensitive.+ * trimming surrounding whitespace and otherwise remain case-sensitive. On+ * macOS, three periods in a label also match the ellipsis character that+ * native menu titles use (`Save As...` finds `Save As…`).*/export type InvokeMenuInput = {/**diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.jsonindex de6251f684..ac9c89b772 100644--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json@@ -436,7 +436,7 @@"idempotent": false,"input_schema": {"additionalProperties": false,- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).","properties": {"path": {"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.jsonindex 2e92c69940..7428a9171e 100644--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json@@ -441,7 +441,7 @@"idempotent": false,"input_schema": {"additionalProperties": false,- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).","properties": {"path": {"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.jsonindex 8b0f09d484..4f17f5c5c2 100644--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json@@ -457,7 +457,7 @@"idempotent": false,"input_schema": {"additionalProperties": false,- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).","properties": {"path": {"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",