Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): run menu-bar item tokens as invoke_menu and keep rows for stale-token re-finds (#4886)
Francesco Bonacci4c2403fc5574
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rsindex a485379d7f..86886bdaa6 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs@@ -933,7 +933,14 @@ async fn run_step(let mut result = registry.invoke(tool, args.clone()).await;let mut refound_note = None;- if result.is_error == Some(true) {+ if result.is_error == Some(true) && tool == "click" {+ if let Some((menu_args, note)) = menu_bar_route(&args, &result) {+ report["refound"] = json!(note);+ result = registry.invoke("invoke_menu", menu_args).await;+ refound_note = Some(note);+ }+ }+ if result.is_error == Some(true) && refound_note.is_none() {if let Some((retry, note)) = refind_stale_target(locator, &args, &result).await {report["refound"] = json!(note);result = registry.invoke(tool, retry).await;@@ -1311,6 +1318,37 @@ async fn refind_stale_target())}+/// A click on a menu-bar item addressed by token is refused as outside the+/// window: menu-bar menus are the app's, not the window's. Send it as+/// invoke_menu with the path read off the item's ancestors instead.+fn menu_bar_route(args: &Value, result: &ToolResult) -> Option<(Value, String)> {+ if !stale_element(result) {+ return None;+ }+ let token = args.get("element_token").and_then(Value::as_str)?;+ let (snapshot_id, row) = token.split_once(':')?;+ let row: u64 = row.parse().ok()?;+ let path = crate::window_state_view::snapshot_menu_path(snapshot_id, row)?;+ let stored = crate::window_state_view::snapshot_row(snapshot_id, row)?;+ let pid = args+ .get("pid")+ .and_then(Value::as_i64)+ .unwrap_or(stored.pid);+ let window_id = args+ .get("window_id")+ .and_then(Value::as_u64)+ .unwrap_or(stored.window_id);+ let mut menu_args = json!({ "pid": pid, "window_id": window_id, "path": path });+ if let Some(session) = args.get("session") {+ menu_args["session"] = session.clone();+ }+ let note = format!(+ "{token} is a menu-bar item, so it ran as invoke_menu {}",+ path.join(" > ")+ );+ Some((menu_args, note))+}+/// Role and label of one tree row: `[14] AXButton "Save"` or/// `[17] AXButton (Delete draft)`. None for a row without a label.fn row_role_and_label(line: &str) -> Option<(String, String)> {@@ -1496,10 +1534,27 @@ fn parse_action(Some(value @ Value::Object(_)) => value.clone(),Some(_) => return Err(PlanError::step(index, "`args` must be an object")),};- let name = TOOL_ALIASES+ // "press" with a target and no key is an AX press of that element.+ let pressed_element = name == "press"+ && args.get("key").is_none()+ && [+ "element_token",+ "element_index",+ "role",+ "name",+ "label",+ "x",+ ].iter()- .find(|(alias, _)| *alias == name)- .map_or(name, |(_, tool)| *tool);+ .any(|key| args.get(*key).is_some());+ let name = if pressed_element {+ "click"+ } else {+ TOOL_ALIASES+ .iter()+ .find(|(alias, _)| *alias == name)+ .map_or(name, |(_, tool)| *tool)+ };let name = rewrite_step(name, &mut args);let tool = BATCHABLE_TOOLS.iter()diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rsindex 820f9b0017..3e2e482be8 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs@@ -179,6 +179,18 @@ async fn tool_name_slips_become_the_action_they_mean() {assert_eq!(harness.last("type_text")["text"], "*");assert_eq!(harness.last("hotkey")["keys"], json!(["ctrl", "pagedown"]));assert_eq!(harness.hits("move_cursor"), 1);++ // v037-full: "press" with an element target is a click on it; with a+ // key it stays a key press.+ let result = harness+ .run(json!({"pid": 9, "window_id": 4, "steps": [+ {"tool": "press", "args": {"element_token": "s00000001:33"}},+ {"tool": "press", "args": {"key": "tab"}}+ ]}))+ .await;+ assert_ne!(result.is_error, Some(true), "{}", text(&result));+ assert_eq!(harness.last("click")["element_token"], "s00000001:33");+ assert_eq!(harness.last("press_key")["key"], "tab");}// ── recovery after a failed step ─────────────────────────────────────────@@ -265,6 +277,36 @@ fn focus_holder_and_row_parsing() {);}+#[tokio::test]+async fn a_menu_bar_item_token_runs_as_invoke_menu() {+ let harness = Harness::new();+ // v037-full: a click on the LibreOffice menu item "To Next Sheet" by+ // token is refused as outside the window (menu-bar menus are the app's).+ crate::window_state_view::remember_for_test(+ "s0000a0e1",+ 80,+ 90,+ "- [0] AXWindow \"Doc\"\n- [27] AXMenuBar\n - [700] AXMenuBarItem \"Sheet\"\n - [701] AXMenu\n - [760] AXMenuItem \"Navigate\"\n - [761] AXMenu\n - [771] AXMenuItem \"To Next Sheet\"\n",+ );+ let refused = "Background input refused (element_outside_target_window): the addressed element could not be proven to belong to window 90";+ let result = harness+ .run(json!({"steps": [{"tool": "click", "args": {"pid": 80, "window_id": 90, "element_token": "s0000a0e1:771", "fail_message": refused}}]}))+ .await;+ // The probe invoke_menu carries no action record, so only the routing+ // is checked here; the real tool reports its own outcome.+ let menu = harness.last("invoke_menu");+ assert_eq!(menu["path"], json!(["Sheet", "Navigate", "To Next Sheet"]));+ assert_eq!(+ (menu["pid"].clone(), menu["window_id"].clone()),+ (json!(80), json!(90))+ );+ assert!(+ text(&result).contains("ran as invoke_menu Sheet > Navigate > To Next Sheet"),+ "{}",+ text(&result)+ );+}+// ── stale element targets ────────────────────────────────────────────────/// A window whose list re-renders on every click: element handles from andiff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rsindex bfa3712ced..3ad6ae3e6c 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs@@ -49,11 +49,11 @@ impl Harness {fn new() -> Self {let mut registry = ToolRegistry::new();let mut calls = std::collections::HashMap::new();- for name in super::BATCHABLE_TOOLS- .iter()- .copied()- .chain(["get_window_state", "zoom"])- {+ for name in super::BATCHABLE_TOOLS.iter().copied().chain([+ "get_window_state",+ "zoom",+ "invoke_menu",+ ]) {let log = Arc::new(Mutex::new(Vec::new()));calls.insert(name, log.clone());registry.register(Box::new(Probe {@@ -75,6 +75,7 @@ impl Harness {"modifiers": {"type": "array", "items": {"type": "string"}},"count": {"type": "integer", "minimum": 1},"query": {"type": "string"},+ "path": {"type": "array", "items": {"type": "string"}},"x1": {"type": "number"}, "y1": {"type": "number"},"x2": {"type": "number"}, "y2": {"type": "number"},"include_screenshot": {"type": "boolean"},diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rsindex 554206e907..4e89f7e959 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs@@ -583,6 +583,11 @@ fn store() -> &'static Mutex<VecDeque<StoredSnapshot>> {fn remember(snapshot_id: &str, ctx: &ViewContext<'_>, md: &str) {let mut guard = store().lock().unwrap_or_else(|e| e.into_inner());+ // A screenshot-only read keeps its snapshot id and returns no tree. It+ // must not erase the rows that snapshot's tokens were read from.+ if md.trim().is_empty() && guard.iter().any(|s| s.snapshot_id == snapshot_id) {+ return;+ }guard.retain(|s| s.snapshot_id != snapshot_id);guard.push_back(StoredSnapshot {snapshot_id: snapshot_id.to_owned(),@@ -637,8 +642,99 @@ pub(crate) fn snapshot_row(snapshot_id: &str, index: u64) -> Option<StoredRow> {})}+/// The menu path of row `[index]` of the read that produced `snapshot_id`,+/// when that row is an item in the app's menu bar: the labels of its+/// AXMenuBarItem and AXMenuItem ancestors, then its own. `None` for any+/// other row.+pub(crate) fn snapshot_menu_path(snapshot_id: &str, index: u64) -> Option<Vec<String>> {+ let markdown = {+ let guard = store().lock().unwrap_or_else(|e| e.into_inner());+ guard+ .iter()+ .find(|s| s.snapshot_id == snapshot_id && s.at.elapsed() <= STORE_TTL)+ .map(|s| s.markdown.clone())+ }?;+ menu_path_in(&markdown, index)+}++fn menu_path_in(markdown: &str, index: u64) -> Option<Vec<String>> {+ let lines: Vec<&str> = markdown.lines().collect();+ let marker = format!("[{index}] ");+ let at = lines.iter().position(|line| {+ line.trim_start()+ .strip_prefix("- ")+ .is_some_and(|rest| rest.starts_with(&marker))+ })?;+ let indent = |line: &str| line.len() - line.trim_start().len();+ let item = |line: &str| -> Option<(String, String)> {+ let rest = line.trim_start().strip_prefix("- ")?;+ let rest = rest+ .strip_prefix('[')+ .and_then(|r| r.split_once("] ").map(|(_, r)| r))+ .unwrap_or(rest);+ let (role, rest) = rest.split_once(' ').unwrap_or((rest, ""));+ let label = rest+ .trim_start()+ .strip_prefix('"')+ .and_then(|r| r.split_once('"').map(|(label, _)| label.to_owned()))+ .unwrap_or_default();+ Some((role.to_owned(), label))+ };+ let (role, label) = item(lines[at])?;+ if !matches!(role.as_str(), "AXMenuItem" | "AXMenuBarItem") || label.is_empty() {+ return None;+ }+ let mut path = vec![label];+ let mut level = indent(lines[at]);+ for line in lines[..at].iter().rev() {+ if indent(line) >= level {+ continue;+ }+ level = indent(line);+ let (role, label) = item(line)?;+ match role.as_str() {+ "AXMenuBar" => {+ path.reverse();+ return Some(path);+ }+ "AXMenuItem" | "AXMenuBarItem" if !label.is_empty() => path.push(label),+ "AXMenu" => {}+ _ => return None,+ }+ }+ None+}+/// Remember a read's markdown as `apply` does, for tests elsewhere in the/// crate that drive a fake get_window_state.+#[cfg(test)]+#[test]+fn menu_paths_come_from_the_menu_bar_ancestors() {+ let md = "- [0] AXWindow \"Doc\"\n - [3] AXButton \"OK\"\n- [27] AXMenuBar\n - [700] AXMenuBarItem \"Sheet\" [actions=[press]]\n - [701] AXMenu\n - [760] AXMenuItem \"Navigate\" [actions=[press]]\n - [761] AXMenu\n - [771] AXMenuItem \"To Next Sheet\" [actions=[press]]\n - [780] AXMenuItem \"Delete Sheet...\"\n";+ assert_eq!(+ menu_path_in(md, 771),+ Some(vec![+ "Sheet".to_owned(),+ "Navigate".to_owned(),+ "To Next Sheet".to_owned()+ ])+ );+ assert_eq!(+ menu_path_in(md, 780),+ Some(vec!["Sheet".to_owned(), "Delete Sheet...".to_owned()])+ );+ assert_eq!(menu_path_in(md, 3), None, "not a menu item");+ assert_eq!(menu_path_in(md, 999), None);+}++#[cfg(test)]+#[test]+fn a_screenshot_only_read_keeps_the_remembered_rows() {+ remember_for_test("s7e570001", 4, 5, "- [9] AXButton \"Send\"\n");+ remember_for_test("s7e570001", 4, 5, "");+ assert!(snapshot_row("s7e570001", 9).is_some_and(|row| row.line.contains("Send")));+}+#[cfg(test)]pub(crate) fn remember_for_test(snapshot_id: &str, pid: i64, window_id: u64, markdown: &str) {let ctx = ViewContext {