Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

perf(cua-driver): leaner window reads: collapse closed menus, drop noise rows and metadata (#4892)

Francesco Bonaccic1c2b5fee428
diff --git a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
index 6ed6177aee..2efe588936 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
@@ -463,6 +463,10 @@ and the refusal says whether any menu window it opened is still on screen. If
the refusal says the menu may still be open, press `escape` on the window
before other input.
+A `get_window_state` read lists only the menu bar's titles and says
+`menus collapsed`; pass `query` to find an item in any menu, or run it with
+`invoke_menu`. A missed path segment lists the items at that level.
+
```bash
cua-driver invoke_menu \
'{"pid":844,"window_id":10725,"path":["Window","Move & Resize","Left"]}'
@@ -545,4 +549,4 @@ starting point for new browser workflows.
If the user instead asks to navigate _within_ an already-open Finder
window, use the menu-bar flow from "Navigating native menu bars"
-above (click Go → pick a menu item → re-snapshot → click it).
+above (`invoke_menu` with a path such as `["Go", "Downloads"]`).
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
index 4e89f7e959..8505c3d1c9 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
@@ -279,6 +279,7 @@ pub fn apply(
.and_then(Value::as_str)
.map(str::to_owned)
else {
+ drop_walk_diagnostics(opts, structured);
return;
};
let snapshot_id = structured
@@ -412,6 +413,26 @@ pub fn apply(
}
rewrite_tree_block(content, &md, &header_extra, &body);
+ drop_walk_diagnostics(opts, structured);
+}
+
+/// Walk diagnostics cost every read context and no caller acts on them:
+/// `truncated` and the truncation hint (built from them above) already say
+/// when a walk stopped early. `verbose:true` keeps them.
+fn drop_walk_diagnostics(opts: &ViewOptions, structured: &mut Value) {
+ if opts.verbose {
+ return;
+ }
+ if let Some(obj) = structured.as_object_mut() {
+ for key in [
+ "nodes_pending",
+ "nodes_visited",
+ "walk_elapsed_ms",
+ "screenshot_refreshed_snapshot",
+ ] {
+ obj.remove(key);
+ }
+ }
}
/// `since` value that names the window's most recent read of the same view.
@@ -514,6 +535,7 @@ fn trim_metadata(opts: &ViewOptions, structured: &mut Value) {
// Linux restates the pixel-coordinate convention on every read; the
// tool description already carries it.
obj.remove("frame_note");
+
if !degraded {
obj.remove("background_input");
}
@@ -1481,6 +1503,25 @@ mod tests {
.contains("larger max_elements"));
}
+ #[test]
+ fn walk_diagnostics_are_dropped_unless_verbose() {
+ let (mut content, mut s) = payload("s000000d2", OLD);
+ s["nodes_visited"] = json!(261);
+ s["walk_elapsed_ms"] = json!(149);
+ apply(&opts(json!({})), &ctx(7016), &mut content, &mut s);
+ assert!(s.get("nodes_visited").is_none());
+ assert!(s.get("walk_elapsed_ms").is_none());
+ let (mut content, mut s) = payload("s000000d3", OLD);
+ s["nodes_visited"] = json!(261);
+ apply(
+ &opts(json!({"verbose": true})),
+ &ctx(7017),
+ &mut content,
+ &mut s,
+ );
+ assert_eq!(s["nodes_visited"], 261);
+ }
+
#[test]
fn degraded_snapshots_keep_background_input() {
let (mut content, mut s) = payload("s000000e1", OLD);
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
index 540b105ae9..cedbbe404d 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
@@ -307,6 +307,7 @@ pub fn walk_tree_budgeted(
}
let walk = budget.outcome();
+ prune_echo_text_rows(&mut lines);
let raw_markdown = render_lines(&lines);
let mut tree_markdown = if let Some(q) = query {
filter_tree(&raw_markdown, q)
@@ -617,9 +618,23 @@ unsafe fn walk_element(
// indexed rows are addressable in click(element_index=N)).
let next_parent = node.element_index.or(parent_index);
- let line = format_node_line(&node);
+ // A closed menu of the menu bar stays collapsed when the caller asked for
+ // it: its items are reachable through invoke_menu or a query, and walking
+ // them cost most of a read (94% of a Chrome window read in a VM check)
+ // and ate the node budget before the window content.
+ let collapse_here = role == "AXMenuBarItem"
+ && MENUS_COLLAPSED.with(std::cell::Cell::get)
+ && copy_bool_attr(element, "AXSelected") != Some(true);
+ let line = if role == "AXMenuBar" && MENUS_COLLAPSED.with(std::cell::Cell::get) {
+ format!("{} {COLLAPSED_MENU_NOTE}", format_node_line(&node))
+ } else {
+ format_node_line(&node)
+ };
lines.push((depth, line));
nodes.push(node);
+ if collapse_here {
+ return;
+ }
let children = copy_children(element);
for child in children {
@@ -681,28 +696,37 @@ fn format_node_line(node: &AXNode) -> String {
parts.push_str(&format!(" = \"{}\"", v));
}
// AXDescription → (description) — critical for Calculator digit buttons
- // where AXTitle="" but AXDescription="2".
- if let Some(d) = &node.description {
+ // where AXTitle="" but AXDescription="2". A description that only repeats
+ // the title (Chrome's `"Back" (Back)`) is not shown twice.
+ if let Some(d) = node
+ .description
+ .as_ref()
+ .filter(|d| node.title.as_ref() != Some(*d))
+ {
parts.push_str(&format!(" ({})", d));
}
// Bracketed metadata block (identifier, help, actions).
if node.element_index.is_some() {
let mut attrs: Vec<String> = Vec::new();
- if let Some(id) = &node.identifier {
+ if let Some(id) = node
+ .identifier
+ .as_deref()
+ .filter(|id| !id.trim().is_empty())
+ {
attrs.push(format!("id={}", id));
}
if let Some(h) = &node.help {
attrs.push(format!("help=\"{}\"", h));
}
- if !node.actions.is_empty() {
- let action_str = node
- .actions
- .iter()
- .map(|a| a.strip_prefix("AX").unwrap_or(a).to_lowercase())
- .collect::<Vec<_>>()
- .join(",");
- attrs.push(format!("actions=[{}]", action_str));
+ let shown: Vec<String> = node
+ .actions
+ .iter()
+ .filter(|a| !UNLISTED_ACTIONS.contains(&a.as_str()))
+ .map(|a| a.strip_prefix("AX").unwrap_or(a).to_lowercase())
+ .collect();
+ if !shown.is_empty() {
+ attrs.push(format!("actions=[{}]", shown.join(",")));
}
if !attrs.is_empty() {
parts.push_str(" [");
@@ -714,6 +738,77 @@ fn format_node_line(node: &AXNode) -> String {
parts
}
+thread_local! {
+ static MENUS_COLLAPSED: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
+}
+
+/// Shown on the menu bar row of a read whose menus were not walked.
+pub(crate) const COLLAPSED_MENU_NOTE: &str =
+ "(menus collapsed: run an item with invoke_menu path, or pass query to search menu items)";
+
+/// Run `walk` with the menu bar's closed menus collapsed (or not) on this
+/// thread. Only `get_window_state` collapses; every other walker keeps the
+/// full menu tree.
+pub fn with_menus_collapsed<T>(collapse: bool, walk: impl FnOnce() -> T) -> T {
+ let previous = MENUS_COLLAPSED.with(|flag| flag.replace(collapse));
+ let result = walk();
+ MENUS_COLLAPSED.with(|flag| flag.set(previous));
+ result
+}
+
+/// Actions every web-content element advertises. Listing them on each row
+/// cost about a third of a web tree's text and told the caller nothing; the
+/// structured `elements` still carry the full list.
+const UNLISTED_ACTIONS: &[&str] = &["AXShowMenu", "AXScrollToVisible"];
+
+/// Drop static-text leaf rows that only repeat text already shown on their
+/// parent row (a web button "Save case" with a child text "Save case"). The
+/// rows keep their element indices; only the rendering omits them, and a
+/// `query` for the text still matches the parent.
+fn prune_echo_text_rows(lines: &mut Vec<(usize, String)>) {
+ let mut keep = vec![true; lines.len()];
+ for i in 0..lines.len() {
+ let (depth, line) = &lines[i];
+ let Some(text) = echo_text_of(line) else {
+ continue;
+ };
+ let is_leaf = lines.get(i + 1).is_none_or(|(next, _)| next <= depth);
+ if !is_leaf || *depth == 0 {
+ continue;
+ }
+ let parent = lines[..i].iter().rev().find(|(d, _)| *d + 1 == *depth);
+ if parent.is_some_and(|(_, parent_line)| quoted_text_contains(parent_line, text)) {
+ keep[i] = false;
+ }
+ }
+ let mut index = 0;
+ lines.retain(|_| {
+ let kept = keep[index];
+ index += 1;
+ kept
+ });
+}
+
+/// The text of a bare static-text row: `- [N] AXStaticText = "text"` with
+/// nothing else on the row.
+fn echo_text_of(line: &str) -> Option<&str> {
+ let rest = line.strip_prefix("- ")?;
+ let rest = match rest.strip_prefix('[') {
+ Some(after) => after.split_once("] ")?.1,
+ None => rest,
+ };
+ let text = rest.strip_prefix("AXStaticText = \"")?.strip_suffix('"')?;
+ (!text.is_empty() && !text.contains('"')).then_some(text)
+}
+
+/// Whether one of the double-quoted strings on `line` contains `text`.
+fn quoted_text_contains(line: &str, text: &str) -> bool {
+ line.split('"')
+ .skip(1)
+ .step_by(2)
+ .any(|quoted| quoted.contains(text))
+}
+
fn render_lines(lines: &[(usize, String)]) -> String {
let mut out = String::new();
for (depth, line) in lines {
@@ -836,3 +931,65 @@ mod tests {
assert_eq!(actionable.enabled, Some(true));
}
}
+
+#[cfg(test)]
+mod lean_row_tests {
+ use super::{echo_text_of, prune_echo_text_rows};
+
+ fn rows(lines: &[(usize, &str)]) -> Vec<(usize, String)> {
+ lines.iter().map(|(d, l)| (*d, l.to_string())).collect()
+ }
+
+ #[test]
+ fn echo_text_rows_under_a_labelled_parent_are_dropped() {
+ let mut lines = rows(&[
+ (0, "- [0] AXWindow \"Mail\" [actions=[raise]]"),
+ (
+ 1,
+ "- [8] AXButton \"Atrium plant service CM-199\" [actions=[press]]",
+ ),
+ (2, "- [9] AXStaticText = \"Atrium plant service\""),
+ (2, "- [10] AXStaticText = \"CM-199\""),
+ (1, "- [11] AXHeading \"Messages\""),
+ (2, "- [12] AXStaticText = \"Inbox\""),
+ (1, "- [13] AXStaticText = \"Mail\""),
+ ]);
+ prune_echo_text_rows(&mut lines);
+ let kept: Vec<&str> = lines.iter().map(|(_, l)| l.as_str()).collect();
+ assert_eq!(
+ kept,
+ [
+ "- [0] AXWindow \"Mail\" [actions=[raise]]",
+ "- [8] AXButton \"Atrium plant service CM-199\" [actions=[press]]",
+ "- [11] AXHeading \"Messages\"",
+ "- [12] AXStaticText = \"Inbox\"",
+ ]
+ );
+ }
+
+ #[test]
+ fn menu_collapsing_is_scoped_to_the_wrapped_walk() {
+ use super::{with_menus_collapsed, MENUS_COLLAPSED};
+ assert!(!MENUS_COLLAPSED.with(std::cell::Cell::get));
+ let inside = with_menus_collapsed(true, || {
+ let nested = with_menus_collapsed(false, || MENUS_COLLAPSED.with(std::cell::Cell::get));
+ (MENUS_COLLAPSED.with(std::cell::Cell::get), nested)
+ });
+ assert_eq!(inside, (true, false));
+ assert!(!MENUS_COLLAPSED.with(std::cell::Cell::get));
+ }
+
+ #[test]
+ fn rows_with_children_or_other_fields_are_kept() {
+ let mut lines = rows(&[
+ (0, "- [1] AXLink \"Docs\""),
+ (1, "- [2] AXStaticText = \"Docs\" [actions=[press]]"),
+ (1, "- [3] AXStaticText = \"Docs\""),
+ (2, "- [4] AXStaticText = \"child\""),
+ ]);
+ prune_echo_text_rows(&mut lines);
+ assert_eq!(lines.len(), 4);
+ assert_eq!(echo_text_of("- AXStaticText = \"x\""), Some("x"));
+ assert_eq!(echo_text_of("- [2] AXStaticText = \"\""), None);
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
index 1ce49017f3..37f817a5e5 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
@@ -253,6 +253,11 @@ impl Tool for GetWindowStateTool {
.unwrap_or(crate::ax::tree::DEFAULT_MAX_DEPTH);
let timeout_ms = cua_driver_core::tool_schema::resolve_timeout_ms(args.get("timeout_ms"));
+ // A model-facing read lists the menu bar's top-level titles only; a
+ // query or full_output walks every menu as before.
+ let collapse_menus = query.as_deref().is_none_or(|q| q.trim().is_empty())
+ && !view.full_output
+ && !observation_only;
let (tree_result, prepared_snapshot) = if want_tree {
let q = query.clone();
// `timeout_ms` bounds the walk itself: it returns the partial tree
@@ -260,13 +265,15 @@ impl Tool for GetWindowStateTool {
// for an AX call that ignores the per-element messaging timeout
// (dropping a spawn_blocking JoinHandle cannot cancel it).
let walk_future = tokio::task::spawn_blocking(move || {
- let tree = crate::ax::tree::walk_tree_budgeted(
- pid,
- Some(window_id),
- q.as_deref(),
- max_depth,
- cua_driver_core::walk_budget::WalkBudget::new(timeout_ms, max_elements),
- );
+ let tree = crate::ax::tree::with_menus_collapsed(collapse_menus, || {
+ crate::ax::tree::walk_tree_budgeted(
+ pid,
+ Some(window_id),
+ q.as_deref(),
+ max_depth,
+ cua_driver_core::walk_budget::WalkBudget::new(timeout_ms, max_elements),
+ )
+ });
let payload = crate::ax::snapshot::AxSnapshot::from_nodes(&tree.nodes);
(tree, payload)
});