Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
perf(cua-driver): leaner window reads: collapse closed menus, drop noise rows and metadata (#4892)
Francesco Bonaccic1c2b5fee428
diff --git a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md b/libs/cua-driver/rust/Skills/cua-driver/MACOS.mdindex 6ed6177aee..2efe588936 100644--- a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md+++ b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md@@ -463,6 +463,10 @@ and the refusal says whether any menu window it opened is still on screen. Ifthe refusal says the menu may still be open, press `escape` on the windowbefore other input.+A `get_window_state` read lists only the menu bar's titles and says+`menus collapsed`; pass `query` to find an item in any menu, or run it with+`invoke_menu`. A missed path segment lists the items at that level.+```bashcua-driver invoke_menu \'{"pid":844,"window_id":10725,"path":["Window","Move & Resize","Left"]}'@@ -545,4 +549,4 @@ starting point for new browser workflows.If the user instead asks to navigate _within_ an already-open Finderwindow, use the menu-bar flow from "Navigating native menu bars"-above (click Go → pick a menu item → re-snapshot → click it).+above (`invoke_menu` with a path such as `["Go", "Downloads"]`).diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rsindex 4e89f7e959..8505c3d1c9 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs@@ -279,6 +279,7 @@ pub fn apply(.and_then(Value::as_str).map(str::to_owned)else {+ drop_walk_diagnostics(opts, structured);return;};let snapshot_id = structured@@ -412,6 +413,26 @@ pub fn apply(}rewrite_tree_block(content, &md, &header_extra, &body);+ drop_walk_diagnostics(opts, structured);+}++/// Walk diagnostics cost every read context and no caller acts on them:+/// `truncated` and the truncation hint (built from them above) already say+/// when a walk stopped early. `verbose:true` keeps them.+fn drop_walk_diagnostics(opts: &ViewOptions, structured: &mut Value) {+ if opts.verbose {+ return;+ }+ if let Some(obj) = structured.as_object_mut() {+ for key in [+ "nodes_pending",+ "nodes_visited",+ "walk_elapsed_ms",+ "screenshot_refreshed_snapshot",+ ] {+ obj.remove(key);+ }+ }}/// `since` value that names the window's most recent read of the same view.@@ -514,6 +535,7 @@ fn trim_metadata(opts: &ViewOptions, structured: &mut Value) {// Linux restates the pixel-coordinate convention on every read; the// tool description already carries it.obj.remove("frame_note");+if !degraded {obj.remove("background_input");}@@ -1481,6 +1503,25 @@ mod tests {.contains("larger max_elements"));}+ #[test]+ fn walk_diagnostics_are_dropped_unless_verbose() {+ let (mut content, mut s) = payload("s000000d2", OLD);+ s["nodes_visited"] = json!(261);+ s["walk_elapsed_ms"] = json!(149);+ apply(&opts(json!({})), &ctx(7016), &mut content, &mut s);+ assert!(s.get("nodes_visited").is_none());+ assert!(s.get("walk_elapsed_ms").is_none());+ let (mut content, mut s) = payload("s000000d3", OLD);+ s["nodes_visited"] = json!(261);+ apply(+ &opts(json!({"verbose": true})),+ &ctx(7017),+ &mut content,+ &mut s,+ );+ assert_eq!(s["nodes_visited"], 261);+ }+#[test]fn degraded_snapshots_keep_background_input() {let (mut content, mut s) = payload("s000000e1", OLD);diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rsindex 540b105ae9..cedbbe404d 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs@@ -307,6 +307,7 @@ pub fn walk_tree_budgeted(}let walk = budget.outcome();+ prune_echo_text_rows(&mut lines);let raw_markdown = render_lines(&lines);let mut tree_markdown = if let Some(q) = query {filter_tree(&raw_markdown, q)@@ -617,9 +618,23 @@ unsafe fn walk_element(// indexed rows are addressable in click(element_index=N)).let next_parent = node.element_index.or(parent_index);- let line = format_node_line(&node);+ // A closed menu of the menu bar stays collapsed when the caller asked for+ // it: its items are reachable through invoke_menu or a query, and walking+ // them cost most of a read (94% of a Chrome window read in a VM check)+ // and ate the node budget before the window content.+ let collapse_here = role == "AXMenuBarItem"+ && MENUS_COLLAPSED.with(std::cell::Cell::get)+ && copy_bool_attr(element, "AXSelected") != Some(true);+ let line = if role == "AXMenuBar" && MENUS_COLLAPSED.with(std::cell::Cell::get) {+ format!("{} {COLLAPSED_MENU_NOTE}", format_node_line(&node))+ } else {+ format_node_line(&node)+ };lines.push((depth, line));nodes.push(node);+ if collapse_here {+ return;+ }let children = copy_children(element);for child in children {@@ -681,28 +696,37 @@ fn format_node_line(node: &AXNode) -> String {parts.push_str(&format!(" = \"{}\"", v));}// AXDescription → (description) — critical for Calculator digit buttons- // where AXTitle="" but AXDescription="2".- if let Some(d) = &node.description {+ // where AXTitle="" but AXDescription="2". A description that only repeats+ // the title (Chrome's `"Back" (Back)`) is not shown twice.+ if let Some(d) = node+ .description+ .as_ref()+ .filter(|d| node.title.as_ref() != Some(*d))+ {parts.push_str(&format!(" ({})", d));}// Bracketed metadata block (identifier, help, actions).if node.element_index.is_some() {let mut attrs: Vec<String> = Vec::new();- if let Some(id) = &node.identifier {+ if let Some(id) = node+ .identifier+ .as_deref()+ .filter(|id| !id.trim().is_empty())+ {attrs.push(format!("id={}", id));}if let Some(h) = &node.help {attrs.push(format!("help=\"{}\"", h));}- if !node.actions.is_empty() {- let action_str = node- .actions- .iter()- .map(|a| a.strip_prefix("AX").unwrap_or(a).to_lowercase())- .collect::<Vec<_>>()- .join(",");- attrs.push(format!("actions=[{}]", action_str));+ let shown: Vec<String> = node+ .actions+ .iter()+ .filter(|a| !UNLISTED_ACTIONS.contains(&a.as_str()))+ .map(|a| a.strip_prefix("AX").unwrap_or(a).to_lowercase())+ .collect();+ if !shown.is_empty() {+ attrs.push(format!("actions=[{}]", shown.join(",")));}if !attrs.is_empty() {parts.push_str(" [");@@ -714,6 +738,77 @@ fn format_node_line(node: &AXNode) -> String {parts}+thread_local! {+ static MENUS_COLLAPSED: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };+}++/// Shown on the menu bar row of a read whose menus were not walked.+pub(crate) const COLLAPSED_MENU_NOTE: &str =+ "(menus collapsed: run an item with invoke_menu path, or pass query to search menu items)";++/// Run `walk` with the menu bar's closed menus collapsed (or not) on this+/// thread. Only `get_window_state` collapses; every other walker keeps the+/// full menu tree.+pub fn with_menus_collapsed<T>(collapse: bool, walk: impl FnOnce() -> T) -> T {+ let previous = MENUS_COLLAPSED.with(|flag| flag.replace(collapse));+ let result = walk();+ MENUS_COLLAPSED.with(|flag| flag.set(previous));+ result+}++/// Actions every web-content element advertises. Listing them on each row+/// cost about a third of a web tree's text and told the caller nothing; the+/// structured `elements` still carry the full list.+const UNLISTED_ACTIONS: &[&str] = &["AXShowMenu", "AXScrollToVisible"];++/// Drop static-text leaf rows that only repeat text already shown on their+/// parent row (a web button "Save case" with a child text "Save case"). The+/// rows keep their element indices; only the rendering omits them, and a+/// `query` for the text still matches the parent.+fn prune_echo_text_rows(lines: &mut Vec<(usize, String)>) {+ let mut keep = vec![true; lines.len()];+ for i in 0..lines.len() {+ let (depth, line) = &lines[i];+ let Some(text) = echo_text_of(line) else {+ continue;+ };+ let is_leaf = lines.get(i + 1).is_none_or(|(next, _)| next <= depth);+ if !is_leaf || *depth == 0 {+ continue;+ }+ let parent = lines[..i].iter().rev().find(|(d, _)| *d + 1 == *depth);+ if parent.is_some_and(|(_, parent_line)| quoted_text_contains(parent_line, text)) {+ keep[i] = false;+ }+ }+ let mut index = 0;+ lines.retain(|_| {+ let kept = keep[index];+ index += 1;+ kept+ });+}++/// The text of a bare static-text row: `- [N] AXStaticText = "text"` with+/// nothing else on the row.+fn echo_text_of(line: &str) -> Option<&str> {+ let rest = line.strip_prefix("- ")?;+ let rest = match rest.strip_prefix('[') {+ Some(after) => after.split_once("] ")?.1,+ None => rest,+ };+ let text = rest.strip_prefix("AXStaticText = \"")?.strip_suffix('"')?;+ (!text.is_empty() && !text.contains('"')).then_some(text)+}++/// Whether one of the double-quoted strings on `line` contains `text`.+fn quoted_text_contains(line: &str, text: &str) -> bool {+ line.split('"')+ .skip(1)+ .step_by(2)+ .any(|quoted| quoted.contains(text))+}+fn render_lines(lines: &[(usize, String)]) -> String {let mut out = String::new();for (depth, line) in lines {@@ -836,3 +931,65 @@ mod tests {assert_eq!(actionable.enabled, Some(true));}}++#[cfg(test)]+mod lean_row_tests {+ use super::{echo_text_of, prune_echo_text_rows};++ fn rows(lines: &[(usize, &str)]) -> Vec<(usize, String)> {+ lines.iter().map(|(d, l)| (*d, l.to_string())).collect()+ }++ #[test]+ fn echo_text_rows_under_a_labelled_parent_are_dropped() {+ let mut lines = rows(&[+ (0, "- [0] AXWindow \"Mail\" [actions=[raise]]"),+ (+ 1,+ "- [8] AXButton \"Atrium plant service CM-199\" [actions=[press]]",+ ),+ (2, "- [9] AXStaticText = \"Atrium plant service\""),+ (2, "- [10] AXStaticText = \"CM-199\""),+ (1, "- [11] AXHeading \"Messages\""),+ (2, "- [12] AXStaticText = \"Inbox\""),+ (1, "- [13] AXStaticText = \"Mail\""),+ ]);+ prune_echo_text_rows(&mut lines);+ let kept: Vec<&str> = lines.iter().map(|(_, l)| l.as_str()).collect();+ assert_eq!(+ kept,+ [+ "- [0] AXWindow \"Mail\" [actions=[raise]]",+ "- [8] AXButton \"Atrium plant service CM-199\" [actions=[press]]",+ "- [11] AXHeading \"Messages\"",+ "- [12] AXStaticText = \"Inbox\"",+ ]+ );+ }++ #[test]+ fn menu_collapsing_is_scoped_to_the_wrapped_walk() {+ use super::{with_menus_collapsed, MENUS_COLLAPSED};+ assert!(!MENUS_COLLAPSED.with(std::cell::Cell::get));+ let inside = with_menus_collapsed(true, || {+ let nested = with_menus_collapsed(false, || MENUS_COLLAPSED.with(std::cell::Cell::get));+ (MENUS_COLLAPSED.with(std::cell::Cell::get), nested)+ });+ assert_eq!(inside, (true, false));+ assert!(!MENUS_COLLAPSED.with(std::cell::Cell::get));+ }++ #[test]+ fn rows_with_children_or_other_fields_are_kept() {+ let mut lines = rows(&[+ (0, "- [1] AXLink \"Docs\""),+ (1, "- [2] AXStaticText = \"Docs\" [actions=[press]]"),+ (1, "- [3] AXStaticText = \"Docs\""),+ (2, "- [4] AXStaticText = \"child\""),+ ]);+ prune_echo_text_rows(&mut lines);+ assert_eq!(lines.len(), 4);+ assert_eq!(echo_text_of("- AXStaticText = \"x\""), Some("x"));+ assert_eq!(echo_text_of("- [2] AXStaticText = \"\""), None);+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rsindex 1ce49017f3..37f817a5e5 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs@@ -253,6 +253,11 @@ impl Tool for GetWindowStateTool {.unwrap_or(crate::ax::tree::DEFAULT_MAX_DEPTH);let timeout_ms = cua_driver_core::tool_schema::resolve_timeout_ms(args.get("timeout_ms"));+ // A model-facing read lists the menu bar's top-level titles only; a+ // query or full_output walks every menu as before.+ let collapse_menus = query.as_deref().is_none_or(|q| q.trim().is_empty())+ && !view.full_output+ && !observation_only;let (tree_result, prepared_snapshot) = if want_tree {let q = query.clone();// `timeout_ms` bounds the walk itself: it returns the partial tree@@ -260,13 +265,15 @@ impl Tool for GetWindowStateTool {// for an AX call that ignores the per-element messaging timeout// (dropping a spawn_blocking JoinHandle cannot cancel it).let walk_future = tokio::task::spawn_blocking(move || {- let tree = crate::ax::tree::walk_tree_budgeted(- pid,- Some(window_id),- q.as_deref(),- max_depth,- cua_driver_core::walk_budget::WalkBudget::new(timeout_ms, max_elements),- );+ let tree = crate::ax::tree::with_menus_collapsed(collapse_menus, || {+ crate::ax::tree::walk_tree_budgeted(+ pid,+ Some(window_id),+ q.as_deref(),+ max_depth,+ cua_driver_core::walk_budget::WalkBudget::new(timeout_ms, max_elements),+ )+ });let payload = crate::ax::snapshot::AxSnapshot::from_nodes(&tree.nodes);(tree, payload)});