Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

test(cua-driver): find Edge 156's quiet permission chip from the window tree (#6277)

injaneityd5aefdab5ae5
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
index eee8482076..67d2e26337 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
@@ -3881,53 +3881,106 @@ fn changed_pixel_count(before: &image::RgbaImage, after: &image::RgbaImage) -> u
.count() as u64
}
-fn browser_chrome_changed_pixel_center(
- before: &image::RgbaImage,
- after: &image::RgbaImage,
- desktop: &ToolResponse,
+/// Screen-point centers of the top-chrome buttons that a permission request
+/// added or relabelled, read from two `get_window_state` trees of one window.
+///
+/// Edge 155 adds a "Notifications blocked" button right of the omnibox. Edge
+/// 156 turns the left site-information button into an unlabelled chip and
+/// shifts the URL text, so a pixel diff of the toolbar is dominated by the
+/// text shift rather than the indicator.
+fn quiet_permission_indicator_centers(
+ before: &serde_json::Value,
+ after: &serde_json::Value,
bounds: (f64, f64, f64, f64),
-) -> Option<(f64, f64)> {
- assert_eq!(before.dimensions(), after.dimensions());
- let screen_width = desktop.structured()["screen_width"].as_f64()?;
- let screen_height = desktop.structured()["screen_height"].as_f64()?;
- let scale_x = f64::from(before.width()) / screen_width;
- let scale_y = f64::from(before.height()) / screen_height;
- let left = (bounds.0 * scale_x).round().max(0.0) as u32;
- let top = (bounds.1 * scale_y).round().max(0.0) as u32;
- let right = ((bounds.0 + bounds.2) * scale_x)
- .round()
- .min(f64::from(before.width())) as u32;
- // Browser permission indicators live in the top chrome. Limiting the
- // search excludes any unrelated page repaint below the toolbar.
- let bottom = ((bounds.1 + bounds.3.min(120.0)) * scale_y)
- .round()
- .min(f64::from(before.height())) as u32;
- let mut changed = 0_u64;
- let mut x_sum = 0_u64;
- let mut y_sum = 0_u64;
- for y in top..bottom {
- for x in left..right {
- let before = before.get_pixel(x, y);
- let after = after.get_pixel(x, y);
- if before
- .0
- .iter()
- .zip(after.0.iter())
- .take(3)
- .any(|(before, after)| before.abs_diff(*after) >= 32)
- {
- changed += 1;
- x_sum += u64::from(x);
- y_sum += u64::from(y);
- }
- }
- }
- (changed > 0).then(|| {
- (
- (x_sum as f64 / changed as f64) / scale_x,
- (y_sum as f64 / changed as f64) / scale_y,
- )
- })
+) -> Vec<(f64, f64)> {
+ let chrome_buttons = |tree: &serde_json::Value| -> Vec<(Option<String>, [f64; 4])> {
+ tree["elements"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .filter(|element| element["role"] == "Button")
+ .filter_map(|element| {
+ let frame = &element["frame"];
+ let rect = [
+ frame["x"].as_f64()?,
+ frame["y"].as_f64()?,
+ frame["w"].as_f64()?,
+ frame["h"].as_f64()?,
+ ];
+ // Browser permission indicators live in the top chrome.
+ (rect[1] + rect[3] <= bounds.1 + bounds.3.min(120.0))
+ .then(|| (element["label"].as_str().map(str::to_owned), rect))
+ })
+ .collect()
+ };
+ let unchanged = chrome_buttons(before);
+ chrome_buttons(after)
+ .into_iter()
+ .filter(|button| !unchanged.contains(button))
+ .map(|(_, [x, y, w, h])| (x + w / 2.0, y + h / 2.0))
+ .collect()
+}
+
+/// Toolbar trees from Windows CI recordings before and after
+/// `Notification.requestPermission()`: Edge 155 in run 41280330847 and Edge
+/// 156 in run 41332120340. Window bounds were (47, 40, 886, 633) in both.
+#[test]
+fn quiet_permission_indicator_follows_the_changed_toolbar_button() {
+ let button = |label: Option<&str>, x: f64, y: f64, w: f64, h: f64| {
+ serde_json::json!({
+ "role": "Button",
+ "label": label,
+ "frame": {"x": x, "y": y, "w": w, "h": h},
+ })
+ };
+ let tree = |site_information: serde_json::Value, extra: Option<serde_json::Value>| {
+ let mut elements = vec![
+ button(Some("Back"), 52.0, 84.0, 32.0, 32.0),
+ button(Some("Refresh"), 88.0, 84.0, 32.0, 32.0),
+ site_information,
+ serde_json::json!({
+ "role": "Edit",
+ "label": "Address and search bar",
+ "frame": {"x": 152.0, "y": 88.0, "w": 509.0, "h": 24.0},
+ }),
+ button(Some("Favorites"), 725.0, 84.0, 32.0, 32.0),
+ serde_json::json!({
+ "role": "Document",
+ "label": "cua-driver Web Harness",
+ "frame": {"x": 52.0, "y": 120.0, "w": 876.0, "h": 548.0},
+ }),
+ ];
+ elements.extend(extra);
+ serde_json::json!({ "elements": elements })
+ };
+ let bounds = (47.0, 40.0, 886.0, 633.0);
+ let before = tree(
+ button(Some("View site information"), 128.0, 88.0, 24.0, 24.0),
+ None,
+ );
+
+ let edge_155 = tree(
+ button(Some("View site information"), 128.0, 88.0, 24.0, 24.0),
+ Some(button(
+ Some("Notifications blocked"),
+ 637.0,
+ 88.0,
+ 24.0,
+ 24.0,
+ )),
+ );
+ assert_eq!(
+ quiet_permission_indicator_centers(&before, &edge_155, bounds),
+ [(649.0, 100.0)]
+ );
+
+ let edge_156 = tree(button(None, 128.0, 88.0, 24.0, 24.0), None);
+ assert_eq!(
+ quiet_permission_indicator_centers(&before, &edge_156, bounds),
+ [(140.0, 100.0)]
+ );
+
+ assert!(quiet_permission_indicator_centers(&before, &before, bounds).is_empty());
}
/// Screen-point center of the pixels that changed in the top browser chrome of
@@ -4166,28 +4219,38 @@ fn run_browser_owned_permission_prompt(spec: &BrowserSpec) {
let minimum_prompt_pixels = (compared_pixels / 1_000).max(1_000);
let initial_desktop_changed_pixels =
changed_pixel_count(&desktop_crop_before, &desktop_crop_after);
+ let minimum_desktop_to_window_ratio = 2_u64;
let mut quiet_prompt_expanded = false;
let mut quiet_prompt_indicator = None;
// Edge can collapse the notification request to a quiet indicator in
- // browser chrome. Expand the region that actually changed, then assess
- // the resulting browser-owned surface with the same pixel oracle.
+ // browser chrome. Expand it, then assess the resulting browser-owned
+ // surface with the same pixel oracle.
+ // On Windows the prompt is collapsed when the desktop shows no
+ // material change or nothing materially beyond the window: Edge 156's
+ // collapsed chip shifts the URL text by ~1650 px in both captures. The
+ // indicator is the toolbar button the request added or relabelled.
// macOS locates the indicator in the window capture and gates on its
// change count: a system notification banner can cover the toolbar in
// the desktop capture.
let quiet_indicator = if spec.name != "edge" {
None
} else if cfg!(target_os = "windows")
- && initial_desktop_changed_pixels < minimum_prompt_pixels
+ && (initial_desktop_changed_pixels < minimum_prompt_pixels
+ || changed_pixel_count(&window_before, &window_after)
+ .saturating_mul(minimum_desktop_to_window_ratio)
+ >= initial_desktop_changed_pixels)
{
- let full_desktop_before = load_capture(&desktop_before_path);
- let full_desktop_after = load_capture(&desktop_after_path);
- Some(browser_chrome_changed_pixel_center(
- &full_desktop_before,
- &full_desktop_after,
- &desktop,
+ let centers = quiet_permission_indicator_centers(
+ before.structured(),
+ window.structured(),
bounds,
- ))
+ );
+ assert!(
+ centers.len() <= 1,
+ "permission request changed more than one browser chrome button: {centers:?}"
+ );
+ Some(centers.first().copied())
} else if cfg!(target_os = "macos")
&& changed_pixel_count(&window_before, &window_after) < minimum_prompt_pixels
{
@@ -4260,7 +4323,6 @@ fn run_browser_owned_permission_prompt(spec: &BrowserSpec) {
let desktop_changed_pixels = changed_pixel_count(&desktop_crop_before, &desktop_crop_after);
let window_changed_pixels = changed_pixel_count(&window_before, &window_after);
- let minimum_desktop_to_window_ratio = 2_u64;
let desktop_has_materially_more_prompt_pixels = window_changed_pixels
.saturating_mul(minimum_desktop_to_window_ratio)
< desktop_changed_pixels;