Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): break same-name ties by exact role and the acting tool in run_actions finds (#4863)
Francesco Bonacci2ad9d62a6634
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rsindex 134838fc54..a485379d7f 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs@@ -870,6 +870,10 @@ async fn run_step(window};if !element.is_empty() {+ let element = ElementSpec {+ action: Some(tool),+ ..element+ };match locator.find(&window_spec, &element, find_timeout).await {Ok(found) => {let object = args.as_object_mut().expect("object");@@ -1282,8 +1286,7 @@ async fn refind_stale_target(let element = ElementSpec {role: Some(role.clone()),name: Some(label.clone()),- text: None,- nth: None,+ ..Default::default()};let found = locator.find(&window, &element, REFIND_TIMEOUT).await.ok()?;let same_label = found@@ -1353,9 +1356,17 @@ fn parse_step(PlanError::step(index, "must be an object: {tool, args} or {<tool>: args}")})?;// The action: `{tool, args}`, or the shorthand `{<tool>: args}`.+ // An extra field that is empty (`"args2": {}`, `null`) says nothing; a+ // step is only refused for fields that carry something.let shorthand: Vec<&String> = object- .keys()- .filter(|key| !STEP_KEYS.contains(&key.as_str()))+ .iter()+ .filter(|(key, _)| !STEP_KEYS.contains(&key.as_str()))+ .filter(|(_, value)| match value {+ Value::Null => false,+ Value::Object(fields) => !fields.is_empty() || object.get("tool").is_none(),+ _ => true,+ })+ .map(|(key, _)| key).collect();let (name, raw_args) = match (object.get("tool"), shorthand.as_slice()) {(Some(_), [extra, ..]) => {diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rsindex f6d577192c..90cc29f381 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs@@ -109,6 +109,9 @@ pub(crate) struct ElementSpec {/// Any row of the tree containing this text, display-only rows included.pub text: Option<String>,pub nth: Option<usize>,+ /// The tool that will act on the element, when known. Breaks a tie+ /// between same-named rows: set_value means the field, not its label.+ pub action: Option<&'static str>,}impl ElementSpec {@@ -285,6 +288,7 @@ fn element_spec(object: &Map<String, Value>) -> Result<ElementSpec, String> {name,text: opt_string(object, "text")?,nth,+ action: None,})}@@ -943,8 +947,69 @@ pub(crate) fn matches<'r>(read: &'r Read, spec: &ElementSpec) -> Vec<&'r Value>}}+/// Lowercase role without the platform prefix: "AXTextField" -> "textfield".+fn bare_role(role: &str) -> String {+ let folded: String = role+ .chars()+ .filter(char::is_ascii_alphanumeric)+ .flat_map(|c| c.to_lowercase())+ .collect();+ folded+ .strip_prefix("ax")+ .map(str::to_owned)+ .unwrap_or(folded)+}++/// Narrow several matches to the ones the caller most plausibly meant: those+/// whose role is exactly the one named ("textfield" over a matching+/// AXTextArea), then, with no role named, those the acting tool can use (a+/// field for set_value rather than its label, a control for click rather+/// than static text). Returns the input unchanged when no rule narrows it.+fn narrow<'r>(found: Vec<&'r Value>, spec: &ElementSpec) -> Vec<&'r Value> {+ if found.len() < 2 || spec.nth.is_some() {+ return found;+ }+ let role_of = |element: &Value| {+ element+ .get("role")+ .and_then(Value::as_str)+ .unwrap_or_default()+ .to_owned()+ };+ let keep = |found: &Vec<&'r Value>, rule: &dyn Fn(&Value) -> bool| -> Option<Vec<&'r Value>> {+ let kept: Vec<&Value> = found.iter().copied().filter(|e| rule(e)).collect();+ (!kept.is_empty() && kept.len() < found.len()).then_some(kept)+ };+ let mut found = found;+ if let Some(role) = &spec.role {+ let wanted = bare_role(role);+ if let Some(kept) = keep(&found, &|e| bare_role(&role_of(e)) == wanted) {+ found = kept;+ }+ } else if let Some(action) = spec.action {+ let usable: &[&str] = match action {+ "set_value" | "type_text" => &["textfield", "popupbutton", "checkbox", "slider"],+ _ => &[],+ };+ let rule: Box<dyn Fn(&Value) -> bool> = if usable.is_empty() {+ Box::new(|e: &Value| {+ !matches!(+ role_family(&role_of(e)).as_str(),+ "statictext" | "image" | "group"+ )+ })+ } else {+ Box::new(move |e: &Value| usable.contains(&role_family(&role_of(e)).as_str()))+ };+ if let Some(kept) = keep(&found, &*rule) {+ found = kept;+ }+ }+ found+}+fn pick(read: &Read, spec: &ElementSpec) -> Result<Found, Miss> {- let found = matches(read, spec);+ let found = narrow(matches(read, spec), spec);let chosen = match (found.len(), spec.nth) {(0, _) => {return Err(Miss::new(@@ -1246,10 +1311,8 @@ fn row_matches(row: &str, spec: &ElementSpec) -> bool {fn near_misses(read: &Read, spec: &ElementSpec) -> String {// Same role, any name: the usual slip is a near-miss label.let relaxed = ElementSpec {- name: None,- text: None,- nth: None,role: spec.role.clone(),+ ..Default::default()};let pool: Vec<&Value> = if spec.role.is_some() {matches(read, &relaxed)@@ -1552,6 +1615,56 @@ mod tests {assert_eq!(pick(&read, &second).unwrap().element["element_index"], 9);}+ #[test]+ fn a_tie_goes_to_the_exact_role_or_what_the_tool_can_use() {+ // v037a: set_value name "Reason" matched the label and the field.+ let reason = read(+ vec![+ element(31, "AXStaticText", "Reason"),+ element(32, "AXTextArea", "Reason"),+ ],+ "",+ );+ let by_name = |action| ElementSpec {+ name: Some("Reason".into()),+ action,+ ..Default::default()+ };+ let set = pick(&reason, &by_name(Some("set_value"))).unwrap();+ assert_eq!(set.element["element_index"], 32);+ let click = pick(&reason, &by_name(Some("click"))).unwrap();+ assert_eq!(click.element["element_index"], 32);+ // Without a known tool the tie stays a tie.+ assert_eq!(pick(&reason, &by_name(None)).unwrap_err().code, "ambiguous");++ // v037a: role "textfield" matched AXTextField "A1" and AXTextArea "A1".+ let cells = read(+ vec![+ element(3, "AXTextField", "A1"),+ element(67, "AXTextArea", "A1"),+ ],+ "",+ );+ let field = ElementSpec {+ role: Some("textfield".into()),+ name: Some("A1".into()),+ ..Default::default()+ };+ assert_eq!(pick(&cells, &field).unwrap().element["element_index"], 3);+ // Two of the exact role stay ambiguous.+ let twins = read(+ vec![element(1, "AXButton", "OK"), element(2, "AXButton", "OK")],+ "",+ );+ let ok = ElementSpec {+ role: Some("button".into()),+ name: Some("OK".into()),+ action: Some("click"),+ ..Default::default()+ };+ assert_eq!(pick(&twins, &ok).unwrap_err().code, "ambiguous");+ }+#[test]fn a_miss_lists_labels_sharing_a_word_first_across_roles() {// v036: "Create event" had become "Update event"; the miss listeddiff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rsindex 23a13ee7a3..820f9b0017 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs@@ -96,6 +96,30 @@ async fn batch_level_pid_and_window_id_are_the_default_window() {);}+#[tokio::test]+async fn an_empty_extra_step_field_is_ignored() {+ let harness = Harness::new();+ // v037a: `"args2": {}` next to a complete step.+ let result = harness+ .run(json!({"steps": [+ {"tool": "click", "args": {"pid": 7, "window_id": 3, "x": 1, "y": 2}, "args2": {}},+ {"tool": "click", "args": {"pid": 7, "window_id": 3}, "note": null}+ ]}))+ .await;+ assert_ne!(result.is_error, Some(true), "{}", text(&result));+ assert_eq!(harness.hits("click"), 2);+ // One that carries something is still refused.+ let refused = harness+ .run(json!({"steps": [{"tool": "click", "args": {"pid": 7}, "args2": {"x": 1}}]}))+ .await;+ assert_eq!(refused.is_error, Some(true));+ assert!(+ text(&refused).contains("unknown field `args2`"),+ "{}",+ text(&refused)+ );+}+// ── read steps ───────────────────────────────────────────────────────────#[tokio::test]