Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): match "..." to the ellipsis in macOS menu paths (#4578)

Ed-Keya207d402d0c6
diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.json
index 8a8486b1e6..5402407c76 100644
--- a/libs/cua-driver/contract/manifest.json
+++ b/libs/cua-driver/contract/manifest.json
@@ -2214,7 +2214,7 @@
},
"input_schema": {
"additionalProperties": false,
- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",
+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).",
"properties": {
"path": {
"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",
diff --git a/libs/cua-driver/python/src/cua_driver/_native_contract.py b/libs/cua-driver/python/src/cua_driver/_native_contract.py
index 674ca47c45..ccdc889daa 100644
--- a/libs/cua-driver/python/src/cua_driver/_native_contract.py
+++ b/libs/cua-driver/python/src/cua_driver/_native_contract.py
@@ -3853,7 +3853,9 @@ class InvokeMenuInput:
"""
Exact, immediate-child application menu path to resolve and invoke through
the operating system's accessibility API. Path labels are matched after
- trimming surrounding whitespace and otherwise remain case-sensitive.
+ trimming surrounding whitespace and otherwise remain case-sensitive. On
+ macOS, three periods in a label also match the ellipsis character that
+ native menu titles use (`Save As...` finds `Save As…`).
"""
def __init__(self, *, pid:int, window_id:int, path:typing.List[str], session:typing.Optional[str]):
self.pid = pid
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
index 66a5c9f779..c612544b37 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
@@ -589,7 +589,9 @@ impl ToolInput for SetWindowFrameInput {
/// Exact, immediate-child application menu path to resolve and invoke through
/// the operating system's accessibility API. Path labels are matched after
-/// trimming surrounding whitespace and otherwise remain case-sensitive.
+/// trimming surrounding whitespace and otherwise remain case-sensitive. On
+/// macOS, three periods in a label also match the ellipsis character that
+/// native menu titles use (`Save As...` finds `Save As…`).
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct InvokeMenuInput {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
index ebc09104c6..e253aae092 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
@@ -86,6 +86,13 @@ unsafe fn semantic_children(parent: AXUIElementRef) -> Vec<AXUIElementRef> {
out
}
+/// A menu title as paths compare it: trimmed, with three periods read as
+/// the ellipsis character macOS menu titles use ("Save As..." finds
+/// "Save As…").
+fn menu_title_key(title: &str) -> String {
+ title.trim().replace("...", "\u{2026}")
+}
+
unsafe fn resolve_exact_prefix(
menu_bar: AXUIElementRef,
prefix: &[String],
@@ -99,10 +106,11 @@ unsafe fn resolve_exact_prefix(
CFRelease(current as CFTypeRef);
}
+ let wanted = menu_title_key(segment);
let mut matches = Vec::new();
for child in children {
let title = copy_string_attr(child, "AXTitle").unwrap_or_default();
- if title.trim() == segment {
+ if menu_title_key(&title) == wanted {
matches.push(child);
} else {
CFRelease(child as CFTypeRef);
@@ -577,6 +585,20 @@ mod tests {
}
}
+ #[test]
+ fn three_periods_match_the_ellipsis_in_menu_titles() {
+ assert_eq!(
+ menu_title_key(" Save As... "),
+ menu_title_key("Save As\u{2026}")
+ );
+ assert_eq!(menu_title_key("Save As..."), menu_title_key("Save As..."));
+ assert_ne!(menu_title_key("Save As"), menu_title_key("Save As\u{2026}"));
+ assert_ne!(
+ menu_title_key("save as..."),
+ menu_title_key("Save As\u{2026}")
+ );
+ }
+
#[test]
fn menu_focus_requires_the_exact_frontmost_app_and_window() {
assert!(exact_window_is_ready(Some(7), 7, Some(42), 42));
diff --git a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts
index e4067cd8ef..2bb7517083 100644
--- a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts
+++ b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts
@@ -2624,7 +2624,9 @@ const FfiConverterTypeHotkeyInput = (() => {
/**
* Exact, immediate-child application menu path to resolve and invoke through
* the operating system's accessibility API. Path labels are matched after
- * trimming surrounding whitespace and otherwise remain case-sensitive.
+ * trimming surrounding whitespace and otherwise remain case-sensitive. On
+ * macOS, three periods in a label also match the ellipsis character that
+ * native menu titles use (`Save As...` finds `Save As…`).
*/
export type InvokeMenuInput = {
/**
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
index de6251f684..ac9c89b772 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
@@ -436,7 +436,7 @@
"idempotent": false,
"input_schema": {
"additionalProperties": false,
- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",
+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).",
"properties": {
"path": {
"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index 2e92c69940..7428a9171e 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -441,7 +441,7 @@
"idempotent": false,
"input_schema": {
"additionalProperties": false,
- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",
+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).",
"properties": {
"path": {
"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
index 8b0f09d484..4f17f5c5c2 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
@@ -457,7 +457,7 @@
"idempotent": false,
"input_schema": {
"additionalProperties": false,
- "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive.",
+ "description": "Exact, immediate-child application menu path to resolve and invoke through\nthe operating system's accessibility API. Path labels are matched after\ntrimming surrounding whitespace and otherwise remain case-sensitive. On\nmacOS, three periods in a label also match the ellipsis character that\nnative menu titles use (`Save As...` finds `Save As…`).",
"properties": {
"path": {
"description": "Menu labels from the top-level menu to the item, e.g. `[\"File\", \"Save As...\"]`\n(1 to 16 labels).",