Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): break same-name ties by exact role and the acting tool in run_actions finds (#4863)

Francesco Bonacci2ad9d62a6634
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
index 134838fc54..a485379d7f 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs
@@ -870,6 +870,10 @@ async fn run_step(
window
};
if !element.is_empty() {
+ let element = ElementSpec {
+ action: Some(tool),
+ ..element
+ };
match locator.find(&window_spec, &element, find_timeout).await {
Ok(found) => {
let object = args.as_object_mut().expect("object");
@@ -1282,8 +1286,7 @@ async fn refind_stale_target(
let element = ElementSpec {
role: Some(role.clone()),
name: Some(label.clone()),
- text: None,
- nth: None,
+ ..Default::default()
};
let found = locator.find(&window, &element, REFIND_TIMEOUT).await.ok()?;
let same_label = found
@@ -1353,9 +1356,17 @@ fn parse_step(
PlanError::step(index, "must be an object: {tool, args} or {<tool>: args}")
})?;
// The action: `{tool, args}`, or the shorthand `{<tool>: args}`.
+ // An extra field that is empty (`"args2": {}`, `null`) says nothing; a
+ // step is only refused for fields that carry something.
let shorthand: Vec<&String> = object
- .keys()
- .filter(|key| !STEP_KEYS.contains(&key.as_str()))
+ .iter()
+ .filter(|(key, _)| !STEP_KEYS.contains(&key.as_str()))
+ .filter(|(_, value)| match value {
+ Value::Null => false,
+ Value::Object(fields) => !fields.is_empty() || object.get("tool").is_none(),
+ _ => true,
+ })
+ .map(|(key, _)| key)
.collect();
let (name, raw_args) = match (object.get("tool"), shorthand.as_slice()) {
(Some(_), [extra, ..]) => {
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs
index f6d577192c..90cc29f381 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/locate.rs
@@ -109,6 +109,9 @@ pub(crate) struct ElementSpec {
/// Any row of the tree containing this text, display-only rows included.
pub text: Option<String>,
pub nth: Option<usize>,
+ /// The tool that will act on the element, when known. Breaks a tie
+ /// between same-named rows: set_value means the field, not its label.
+ pub action: Option<&'static str>,
}
impl ElementSpec {
@@ -285,6 +288,7 @@ fn element_spec(object: &Map<String, Value>) -> Result<ElementSpec, String> {
name,
text: opt_string(object, "text")?,
nth,
+ action: None,
})
}
@@ -943,8 +947,69 @@ pub(crate) fn matches<'r>(read: &'r Read, spec: &ElementSpec) -> Vec<&'r Value>
}
}
+/// Lowercase role without the platform prefix: "AXTextField" -> "textfield".
+fn bare_role(role: &str) -> String {
+ let folded: String = role
+ .chars()
+ .filter(char::is_ascii_alphanumeric)
+ .flat_map(|c| c.to_lowercase())
+ .collect();
+ folded
+ .strip_prefix("ax")
+ .map(str::to_owned)
+ .unwrap_or(folded)
+}
+
+/// Narrow several matches to the ones the caller most plausibly meant: those
+/// whose role is exactly the one named ("textfield" over a matching
+/// AXTextArea), then, with no role named, those the acting tool can use (a
+/// field for set_value rather than its label, a control for click rather
+/// than static text). Returns the input unchanged when no rule narrows it.
+fn narrow<'r>(found: Vec<&'r Value>, spec: &ElementSpec) -> Vec<&'r Value> {
+ if found.len() < 2 || spec.nth.is_some() {
+ return found;
+ }
+ let role_of = |element: &Value| {
+ element
+ .get("role")
+ .and_then(Value::as_str)
+ .unwrap_or_default()
+ .to_owned()
+ };
+ let keep = |found: &Vec<&'r Value>, rule: &dyn Fn(&Value) -> bool| -> Option<Vec<&'r Value>> {
+ let kept: Vec<&Value> = found.iter().copied().filter(|e| rule(e)).collect();
+ (!kept.is_empty() && kept.len() < found.len()).then_some(kept)
+ };
+ let mut found = found;
+ if let Some(role) = &spec.role {
+ let wanted = bare_role(role);
+ if let Some(kept) = keep(&found, &|e| bare_role(&role_of(e)) == wanted) {
+ found = kept;
+ }
+ } else if let Some(action) = spec.action {
+ let usable: &[&str] = match action {
+ "set_value" | "type_text" => &["textfield", "popupbutton", "checkbox", "slider"],
+ _ => &[],
+ };
+ let rule: Box<dyn Fn(&Value) -> bool> = if usable.is_empty() {
+ Box::new(|e: &Value| {
+ !matches!(
+ role_family(&role_of(e)).as_str(),
+ "statictext" | "image" | "group"
+ )
+ })
+ } else {
+ Box::new(move |e: &Value| usable.contains(&role_family(&role_of(e)).as_str()))
+ };
+ if let Some(kept) = keep(&found, &*rule) {
+ found = kept;
+ }
+ }
+ found
+}
+
fn pick(read: &Read, spec: &ElementSpec) -> Result<Found, Miss> {
- let found = matches(read, spec);
+ let found = narrow(matches(read, spec), spec);
let chosen = match (found.len(), spec.nth) {
(0, _) => {
return Err(Miss::new(
@@ -1246,10 +1311,8 @@ fn row_matches(row: &str, spec: &ElementSpec) -> bool {
fn near_misses(read: &Read, spec: &ElementSpec) -> String {
// Same role, any name: the usual slip is a near-miss label.
let relaxed = ElementSpec {
- name: None,
- text: None,
- nth: None,
role: spec.role.clone(),
+ ..Default::default()
};
let pool: Vec<&Value> = if spec.role.is_some() {
matches(read, &relaxed)
@@ -1552,6 +1615,56 @@ mod tests {
assert_eq!(pick(&read, &second).unwrap().element["element_index"], 9);
}
+ #[test]
+ fn a_tie_goes_to_the_exact_role_or_what_the_tool_can_use() {
+ // v037a: set_value name "Reason" matched the label and the field.
+ let reason = read(
+ vec![
+ element(31, "AXStaticText", "Reason"),
+ element(32, "AXTextArea", "Reason"),
+ ],
+ "",
+ );
+ let by_name = |action| ElementSpec {
+ name: Some("Reason".into()),
+ action,
+ ..Default::default()
+ };
+ let set = pick(&reason, &by_name(Some("set_value"))).unwrap();
+ assert_eq!(set.element["element_index"], 32);
+ let click = pick(&reason, &by_name(Some("click"))).unwrap();
+ assert_eq!(click.element["element_index"], 32);
+ // Without a known tool the tie stays a tie.
+ assert_eq!(pick(&reason, &by_name(None)).unwrap_err().code, "ambiguous");
+
+ // v037a: role "textfield" matched AXTextField "A1" and AXTextArea "A1".
+ let cells = read(
+ vec![
+ element(3, "AXTextField", "A1"),
+ element(67, "AXTextArea", "A1"),
+ ],
+ "",
+ );
+ let field = ElementSpec {
+ role: Some("textfield".into()),
+ name: Some("A1".into()),
+ ..Default::default()
+ };
+ assert_eq!(pick(&cells, &field).unwrap().element["element_index"], 3);
+ // Two of the exact role stay ambiguous.
+ let twins = read(
+ vec![element(1, "AXButton", "OK"), element(2, "AXButton", "OK")],
+ "",
+ );
+ let ok = ElementSpec {
+ role: Some("button".into()),
+ name: Some("OK".into()),
+ action: Some("click"),
+ ..Default::default()
+ };
+ assert_eq!(pick(&twins, &ok).unwrap_err().code, "ambiguous");
+ }
+
#[test]
fn a_miss_lists_labels_sharing_a_word_first_across_roles() {
// v036: "Create event" had become "Update event"; the miss listed
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs
index 23a13ee7a3..820f9b0017 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools/reliability_tests.rs
@@ -96,6 +96,30 @@ async fn batch_level_pid_and_window_id_are_the_default_window() {
);
}
+#[tokio::test]
+async fn an_empty_extra_step_field_is_ignored() {
+ let harness = Harness::new();
+ // v037a: `"args2": {}` next to a complete step.
+ let result = harness
+ .run(json!({"steps": [
+ {"tool": "click", "args": {"pid": 7, "window_id": 3, "x": 1, "y": 2}, "args2": {}},
+ {"tool": "click", "args": {"pid": 7, "window_id": 3}, "note": null}
+ ]}))
+ .await;
+ assert_ne!(result.is_error, Some(true), "{}", text(&result));
+ assert_eq!(harness.hits("click"), 2);
+ // One that carries something is still refused.
+ let refused = harness
+ .run(json!({"steps": [{"tool": "click", "args": {"pid": 7}, "args2": {"x": 1}}]}))
+ .await;
+ assert_eq!(refused.is_error, Some(true));
+ assert!(
+ text(&refused).contains("unknown field `args2`"),
+ "{}",
+ text(&refused)
+ );
+}
+
// ── read steps ───────────────────────────────────────────────────────────
#[tokio::test]