Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): run menu-bar item tokens as invoke_menu and keep rows for stale-token re-finds (#4886)

Francesco Bonacci4c2403fc5574
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
index a485379d7f..86886bdaa6 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
@@ -933,7 +933,14 @@ async fn run_step(
let mut result = registry.invoke(tool, args.clone()).await;
let mut refound_note = None;
- if result.is_error == Some(true) {
+ if result.is_error == Some(true) && tool == "click" {
+ if let Some((menu_args, note)) = menu_bar_route(&args, &result) {
+ report["refound"] = json!(note);
+ result = registry.invoke("invoke_menu", menu_args).await;
+ refound_note = Some(note);
+ }
+ }
+ if result.is_error == Some(true) && refound_note.is_none() {
if let Some((retry, note)) = refind_stale_target(locator, &args, &result).await {
report["refound"] = json!(note);
result = registry.invoke(tool, retry).await;
@@ -1311,6 +1318,37 @@ async fn refind_stale_target(
))
}
+/// A click on a menu-bar item addressed by token is refused as outside the
+/// window: menu-bar menus are the app's, not the window's. Send it as
+/// invoke_menu with the path read off the item's ancestors instead.
+fn menu_bar_route(args: &Value, result: &ToolResult) -> Option<(Value, String)> {
+ if !stale_element(result) {
+ return None;
+ }
+ let token = args.get("element_token").and_then(Value::as_str)?;
+ let (snapshot_id, row) = token.split_once(':')?;
+ let row: u64 = row.parse().ok()?;
+ let path = crate::window_state_view::snapshot_menu_path(snapshot_id, row)?;
+ let stored = crate::window_state_view::snapshot_row(snapshot_id, row)?;
+ let pid = args
+ .get("pid")
+ .and_then(Value::as_i64)
+ .unwrap_or(stored.pid);
+ let window_id = args
+ .get("window_id")
+ .and_then(Value::as_u64)
+ .unwrap_or(stored.window_id);
+ let mut menu_args = json!({ "pid": pid, "window_id": window_id, "path": path });
+ if let Some(session) = args.get("session") {
+ menu_args["session"] = session.clone();
+ }
+ let note = format!(
+ "{token} is a menu-bar item, so it ran as invoke_menu {}",
+ path.join(" > ")
+ );
+ Some((menu_args, note))
+}
+
/// Role and label of one tree row: `[14] AXButton "Save"` or
/// `[17] AXButton (Delete draft)`. None for a row without a label.
fn row_role_and_label(line: &str) -> Option<(String, String)> {
@@ -1496,10 +1534,27 @@ fn parse_action(
Some(value @ Value::Object(_)) => value.clone(),
Some(_) => return Err(PlanError::step(index, "`args` must be an object")),
};
- let name = TOOL_ALIASES
+ // "press" with a target and no key is an AX press of that element.
+ let pressed_element = name == "press"
+ && args.get("key").is_none()
+ && [
+ "element_token",
+ "element_index",
+ "role",
+ "name",
+ "label",
+ "x",
+ ]
.iter()
- .find(|(alias, _)| *alias == name)
- .map_or(name, |(_, tool)| *tool);
+ .any(|key| args.get(*key).is_some());
+ let name = if pressed_element {
+ "click"
+ } else {
+ TOOL_ALIASES
+ .iter()
+ .find(|(alias, _)| *alias == name)
+ .map_or(name, |(_, tool)| *tool)
+ };
let name = rewrite_step(name, &mut args);
let tool = BATCHABLE_TOOLS
.iter()
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs
index 820f9b0017..3e2e482be8 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs
@@ -179,6 +179,18 @@ async fn tool_name_slips_become_the_action_they_mean() {
assert_eq!(harness.last("type_text")["text"], "*");
assert_eq!(harness.last("hotkey")["keys"], json!(["ctrl", "pagedown"]));
assert_eq!(harness.hits("move_cursor"), 1);
+
+ // v037-full: "press" with an element target is a click on it; with a
+ // key it stays a key press.
+ let result = harness
+ .run(json!({"pid": 9, "window_id": 4, "steps": [
+ {"tool": "press", "args": {"element_token": "s00000001:33"}},
+ {"tool": "press", "args": {"key": "tab"}}
+ ]}))
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ assert_eq!(harness.last("click")["element_token"], "s00000001:33");
+ assert_eq!(harness.last("press_key")["key"], "tab");
}
// ── recovery after a failed step ─────────────────────────────────────────
@@ -265,6 +277,36 @@ fn focus_holder_and_row_parsing() {
);
}
+#[tokio::test]
+async fn a_menu_bar_item_token_runs_as_invoke_menu() {
+ let harness = Harness::new();
+ // v037-full: a click on the LibreOffice menu item "To Next Sheet" by
+ // token is refused as outside the window (menu-bar menus are the app's).
+ crate::window_state_view::remember_for_test(
+ "s0000a0e1",
+ 80,
+ 90,
+ "- [0] AXWindow \"Doc\"\n- [27] AXMenuBar\n - [700] AXMenuBarItem \"Sheet\"\n - [701] AXMenu\n - [760] AXMenuItem \"Navigate\"\n - [761] AXMenu\n - [771] AXMenuItem \"To Next Sheet\"\n",
+ );
+ let refused = "Background input refused (element_outside_target_window): the addressed element could not be proven to belong to window 90";
+ let result = harness
+ .run(json!({"steps": [{"tool": "click", "args": {"pid": 80, "window_id": 90, "element_token": "s0000a0e1:771", "fail_message": refused}}]}))
+ .await;
+ // The probe invoke_menu carries no action record, so only the routing
+ // is checked here; the real tool reports its own outcome.
+ let menu = harness.last("invoke_menu");
+ assert_eq!(menu["path"], json!(["Sheet", "Navigate", "To Next Sheet"]));
+ assert_eq!(
+ (menu["pid"].clone(), menu["window_id"].clone()),
+ (json!(80), json!(90))
+ );
+ assert!(
+ text(&result).contains("ran as invoke_menu Sheet > Navigate > To Next Sheet"),
+ "{}",
+ text(&result)
+ );
+}
+
// ── stale element targets ────────────────────────────────────────────────
/// A window whose list re-renders on every click: element handles from an
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs
index bfa3712ced..3ad6ae3e6c 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/tests.rs
@@ -49,11 +49,11 @@ impl Harness {
fn new() -> Self {
let mut registry = ToolRegistry::new();
let mut calls = std::collections::HashMap::new();
- for name in super::BATCHABLE_TOOLS
- .iter()
- .copied()
- .chain(["get_window_state", "zoom"])
- {
+ for name in super::BATCHABLE_TOOLS.iter().copied().chain([
+ "get_window_state",
+ "zoom",
+ "invoke_menu",
+ ]) {
let log = Arc::new(Mutex::new(Vec::new()));
calls.insert(name, log.clone());
registry.register(Box::new(Probe {
@@ -75,6 +75,7 @@ impl Harness {
"modifiers": {"type": "array", "items": {"type": "string"}},
"count": {"type": "integer", "minimum": 1},
"query": {"type": "string"},
+ "path": {"type": "array", "items": {"type": "string"}},
"x1": {"type": "number"}, "y1": {"type": "number"},
"x2": {"type": "number"}, "y2": {"type": "number"},
"include_screenshot": {"type": "boolean"},
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
index 554206e907..4e89f7e959 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
@@ -583,6 +583,11 @@ fn store() -> &'static Mutex<VecDeque<StoredSnapshot>> {
fn remember(snapshot_id: &str, ctx: &ViewContext<'_>, md: &str) {
let mut guard = store().lock().unwrap_or_else(|e| e.into_inner());
+ // A screenshot-only read keeps its snapshot id and returns no tree. It
+ // must not erase the rows that snapshot's tokens were read from.
+ if md.trim().is_empty() && guard.iter().any(|s| s.snapshot_id == snapshot_id) {
+ return;
+ }
guard.retain(|s| s.snapshot_id != snapshot_id);
guard.push_back(StoredSnapshot {
snapshot_id: snapshot_id.to_owned(),
@@ -637,8 +642,99 @@ pub(crate) fn snapshot_row(snapshot_id: &str, index: u64) -> Option<StoredRow> {
})
}
+/// The menu path of row `[index]` of the read that produced `snapshot_id`,
+/// when that row is an item in the app's menu bar: the labels of its
+/// AXMenuBarItem and AXMenuItem ancestors, then its own. `None` for any
+/// other row.
+pub(crate) fn snapshot_menu_path(snapshot_id: &str, index: u64) -> Option<Vec<String>> {
+ let markdown = {
+ let guard = store().lock().unwrap_or_else(|e| e.into_inner());
+ guard
+ .iter()
+ .find(|s| s.snapshot_id == snapshot_id && s.at.elapsed() <= STORE_TTL)
+ .map(|s| s.markdown.clone())
+ }?;
+ menu_path_in(&markdown, index)
+}
+
+fn menu_path_in(markdown: &str, index: u64) -> Option<Vec<String>> {
+ let lines: Vec<&str> = markdown.lines().collect();
+ let marker = format!("[{index}] ");
+ let at = lines.iter().position(|line| {
+ line.trim_start()
+ .strip_prefix("- ")
+ .is_some_and(|rest| rest.starts_with(&marker))
+ })?;
+ let indent = |line: &str| line.len() - line.trim_start().len();
+ let item = |line: &str| -> Option<(String, String)> {
+ let rest = line.trim_start().strip_prefix("- ")?;
+ let rest = rest
+ .strip_prefix('[')
+ .and_then(|r| r.split_once("] ").map(|(_, r)| r))
+ .unwrap_or(rest);
+ let (role, rest) = rest.split_once(' ').unwrap_or((rest, ""));
+ let label = rest
+ .trim_start()
+ .strip_prefix('"')
+ .and_then(|r| r.split_once('"').map(|(label, _)| label.to_owned()))
+ .unwrap_or_default();
+ Some((role.to_owned(), label))
+ };
+ let (role, label) = item(lines[at])?;
+ if !matches!(role.as_str(), "AXMenuItem" | "AXMenuBarItem") || label.is_empty() {
+ return None;
+ }
+ let mut path = vec![label];
+ let mut level = indent(lines[at]);
+ for line in lines[..at].iter().rev() {
+ if indent(line) >= level {
+ continue;
+ }
+ level = indent(line);
+ let (role, label) = item(line)?;
+ match role.as_str() {
+ "AXMenuBar" => {
+ path.reverse();
+ return Some(path);
+ }
+ "AXMenuItem" | "AXMenuBarItem" if !label.is_empty() => path.push(label),
+ "AXMenu" => {}
+ _ => return None,
+ }
+ }
+ None
+}
+
/// Remember a read's markdown as `apply` does, for tests elsewhere in the
/// crate that drive a fake get_window_state.
+#[cfg(test)]
+#[test]
+fn menu_paths_come_from_the_menu_bar_ancestors() {
+ let md = "- [0] AXWindow \"Doc\"\n - [3] AXButton \"OK\"\n- [27] AXMenuBar\n - [700] AXMenuBarItem \"Sheet\" [actions=[press]]\n - [701] AXMenu\n - [760] AXMenuItem \"Navigate\" [actions=[press]]\n - [761] AXMenu\n - [771] AXMenuItem \"To Next Sheet\" [actions=[press]]\n - [780] AXMenuItem \"Delete Sheet...\"\n";
+ assert_eq!(
+ menu_path_in(md, 771),
+ Some(vec![
+ "Sheet".to_owned(),
+ "Navigate".to_owned(),
+ "To Next Sheet".to_owned()
+ ])
+ );
+ assert_eq!(
+ menu_path_in(md, 780),
+ Some(vec!["Sheet".to_owned(), "Delete Sheet...".to_owned()])
+ );
+ assert_eq!(menu_path_in(md, 3), None, "not a menu item");
+ assert_eq!(menu_path_in(md, 999), None);
+}
+
+#[cfg(test)]
+#[test]
+fn a_screenshot_only_read_keeps_the_remembered_rows() {
+ remember_for_test("s7e570001", 4, 5, "- [9] AXButton \"Send\"\n");
+ remember_for_test("s7e570001", 4, 5, "");
+ assert!(snapshot_row("s7e570001", 9).is_some_and(|row| row.line.contains("Send")));
+}
+
#[cfg(test)]
pub(crate) fn remember_for_test(snapshot_id: &str, pid: i64, window_id: u64, markdown: &str) {
let ctx = ViewContext {