Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): count only the returned elements in an elements diff read (#5649)

injaneitye12ccfe29b5d
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
index 8505c3d1c9..60ba0d9b42 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs
@@ -562,6 +562,11 @@ fn retain_elements(
.and_then(Value::as_u64)
.is_some_and(|i| keep.contains(&i))
});
+ // `returned_element_count` counts the `elements` this response carries.
+ let returned = elements.len();
+ if let Some(count) = obj.get_mut("returned_element_count") {
+ *count = json!(returned);
+ }
}
}
@@ -1471,6 +1476,67 @@ mod tests {
assert_eq!(s5["since_status"], "other_window");
}
+ #[test]
+ fn an_elements_diff_counts_only_the_elements_it_returns() {
+ use cua_driver_contract::{ToolOutput, WindowStateOutput};
+ // A backend's read: every actionable row of OLD, counted as returned.
+ let read = |sid: &str, md: &str| {
+ let (content, mut s) = payload(sid, md);
+ s["elements"] = json!([
+ {"element_index": 1, "role": "AXTextField", "depth": 1},
+ {"element_index": 2, "role": "AXButton", "depth": 1},
+ {"element_index": 3, "role": "AXButton", "depth": 1}
+ ]);
+ s["total_element_count"] = json!(4);
+ s["returned_element_count"] = json!(3);
+ (content, s)
+ };
+ let edited = OLD.replace("\"a\"", "\"ab\"");
+ for (window, format, [id1, id2, id3]) in [
+ (7020, "elements", ["s000000g1", "s000000g2", "s000000g3"]),
+ (7021, "both", ["s000000h1", "s000000h2", "s000000h3"]),
+ ] {
+ let (mut c1, mut s1) = read(id1, OLD);
+ apply(
+ &opts(json!({ "tree_format": format })),
+ &ctx(window),
+ &mut c1,
+ &mut s1,
+ );
+ let (mut c2, mut s2) = read(id2, &edited);
+ apply(
+ &opts(json!({ "tree_format": format, "since": "latest" })),
+ &ctx(window),
+ &mut c2,
+ &mut s2,
+ );
+ let output: WindowStateOutput = serde_json::from_value(s2.clone()).unwrap();
+ assert_eq!(output.validate(), Ok(()), "{format}");
+ assert_eq!(s2["since_status"], "diff", "{format}");
+ assert_eq!(
+ s2["elements"],
+ json!([{"element_index": 1, "role": "AXTextField", "depth": 1}]),
+ "{format}"
+ );
+ assert_eq!(s2["returned_element_count"], 1, "{format}");
+ assert_eq!(s2["total_element_count"], 4, "{format}");
+
+ // No change: no elements, and a count of none.
+ let (mut c3, mut s3) = read(id3, &edited);
+ apply(
+ &opts(json!({ "tree_format": format, "since": "latest" })),
+ &ctx(window),
+ &mut c3,
+ &mut s3,
+ );
+ let output: WindowStateOutput = serde_json::from_value(s3.clone()).unwrap();
+ assert_eq!(output.validate(), Ok(()), "{format}");
+ assert_eq!(s3["since_status"], "no_change", "{format}");
+ assert_eq!(s3["elements"], json!([]), "{format}");
+ assert_eq!(s3["returned_element_count"], 0, "{format}");
+ }
+ }
+
#[test]
fn a_rewritten_tree_falls_back_to_full() {
let window = 7005;