Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): keep the pointer out of macOS window screenshots (#5480)

Francesco Bonacci757ae8cedb87
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
index 1e09774aa7..8181afa3bd 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
@@ -2374,3 +2374,128 @@ fn harness_appkit_context_menu_keeps_front_key_focus_background() {
},
);
}
+
+#[repr(C)]
+#[derive(Clone, Copy)]
+struct WarpPoint {
+ x: f64,
+ y: f64,
+}
+
+#[link(name = "CoreGraphics", kind = "framework")]
+extern "C" {
+ fn CGWarpMouseCursorPosition(point: WarpPoint) -> i32;
+}
+
+/// Regression for trycua/cua#5476. With the real pointer parked outside the
+/// target window, a window screenshot came back the size of the window but
+/// with the window drawn smaller (about 96.5%) and pushed down by ~31 px, so
+/// pixel targets read off it missed by ~37 px. The window must fill its own
+/// screenshot edge to edge, whatever the pointer is doing.
+#[test]
+#[ignore]
+fn harness_appkit_window_capture_ignores_pointer_outside_window() {
+ run_case(
+ native_readonly_case(
+ "appkit",
+ "window_capture_ignores_pointer_outside_window",
+ Targeting::NotApplicable,
+ DriverRoute::WindowState,
+ vec![OracleKind::Pixels],
+ ),
+ |pid, wid, driver| {
+ use base64::Engine as _;
+
+ // Offset the window from the screen origin and leave room above
+ // it. Keep its current size: the harness window is not resizable
+ // (no settable AXSize), and set_window_frame only writes AXSize
+ // when the size changes.
+ let listed = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 }));
+ assert!(!listed.is_error(), "list_windows: {}", listed.text());
+ let current = listed.structured()["windows"]
+ .as_array()
+ .and_then(|windows| {
+ windows
+ .iter()
+ .find(|w| w["window_id"].as_u64() == Some(wid))
+ })
+ .map(|w| w["bounds"].clone())
+ .expect("AppKit harness window bounds");
+ let width = current["width"].as_f64().expect("window width");
+ let height = current["height"].as_f64().expect("window height");
+ let (x, y) = (180.0, 260.0);
+ let moved = driver.call(
+ "set_window_frame",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "x": x, "y": y, "width": width, "height": height
+ }),
+ );
+ assert!(!moved.is_error(), "set_window_frame: {}", moved.text());
+
+ // Park the real pointer just above the window, outside its frame
+ // but close enough for ScreenCaptureKit to fold it into the
+ // capture (a benchmark parked it 27 pt above a window's top edge).
+ // SAFETY: plain CoreGraphics call with a by-value point.
+ unsafe {
+ CGWarpMouseCursorPosition(WarpPoint {
+ x: x + width / 2.0,
+ y: y - 24.0,
+ })
+ };
+ std::thread::sleep(Duration::from_millis(300));
+
+ let snapshot = snapshot_elements(driver, pid, wid);
+ assert!(!snapshot.is_error(), "capture: {}", snapshot.text());
+ let structured = snapshot.structured();
+ assert_eq!(structured["screenshot_frame_valid"], true, "{structured}");
+ let scale = structured["screenshot_scale"]
+ .as_f64()
+ .expect("screenshot scale");
+ let bounds = &structured["window_bounds"];
+ let data = snapshot.raw["result"]["content"]
+ .as_array()
+ .and_then(|parts| parts.iter().find(|part| part["type"] == "image"))
+ .and_then(|part| part["data"].as_str())
+ .expect("window screenshot image part");
+ let png = base64::engine::general_purpose::STANDARD
+ .decode(data)
+ .expect("decode screenshot");
+ let image = image::load_from_memory(&png)
+ .expect("load screenshot")
+ .to_rgba8();
+ let (w, h) = image.dimensions();
+ assert_eq!(
+ (f64::from(w), f64::from(h)),
+ (
+ (bounds["width"].as_f64().unwrap() * scale).round(),
+ (bounds["height"].as_f64().unwrap() * scale).round()
+ ),
+ "screenshot size must match the window bounds at the reported scale"
+ );
+
+ // The bug left a transparent band across the top and down the
+ // right side of the image. Sample just inside each edge, away
+ // from the rounded corners: every sample must be the window's own
+ // opaque pixels.
+ let inset = (4.0 * scale) as u32;
+ let samples = [
+ (w / 2, inset), // top centre (title bar)
+ (w / 4, inset), // top, left of centre
+ (3 * w / 4, inset), // top, right of centre
+ (w - 1 - inset, h / 2), // right edge
+ (w - 1 - inset, h / 4), // right edge, upper
+ (w / 2, h - 1 - inset), // bottom centre
+ ];
+ for (sx, sy) in samples {
+ let alpha = image.get_pixel(sx, sy).0[3];
+ assert_eq!(
+ alpha, 255,
+ "pixel ({sx},{sy}) of a {w}x{h} window screenshot is transparent: \
+ the window does not fill its own capture"
+ );
+ }
+ Observation::delivered(vec![OracleKind::Pixels], Evidence::default())
+ },
+ );
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
index fd519a81c9..dff4d82bd1 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
@@ -463,7 +463,7 @@ fn build_window_capture_plan(
window_id: u32,
expected_identity: WindowCaptureIdentity,
) -> anyhow::Result<std::sync::Arc<WindowCapturePlan>> {
- use screencapturekit::prelude::{SCContentFilter, SCShareableContent, SCStreamConfiguration};
+ use screencapturekit::prelude::{SCContentFilter, SCShareableContent};
let content = SCShareableContent::get()
.map_err(|e| anyhow::anyhow!("SCShareableContent::get failed: {e}"))?;
@@ -519,9 +519,7 @@ fn build_window_capture_plan(
let out_w = rounded_pixel_dim(width_pts * scale, "width")?;
let out_h = rounded_pixel_dim(height_pts * scale, "height")?;
- let config = SCStreamConfiguration::new()
- .with_width(out_w)
- .with_height(out_h);
+ let config = window_capture_config(out_w, out_h);
Ok(std::sync::Arc::new(WindowCapturePlan {
filter,
@@ -530,6 +528,26 @@ fn build_window_capture_plan(
}))
}
+/// Output configuration for a single-window capture.
+///
+/// The hardware pointer stays out. ScreenCaptureKit draws it by default, and
+/// with a desktop-independent window filter it grows the captured area to
+/// take in a pointer that sits outside the window, then shrinks that larger
+/// area into the configured `width`x`height`. The window then shows up scaled
+/// (for example to 96.5%) and offset inside an image whose size still matches
+/// its bounds, so the frame check passes with scale 1 and every pixel
+/// coordinate read off it lands off target. `screencapture -l` and the
+/// SkyLight path never include the pointer either.
+fn window_capture_config(
+ out_w: u32,
+ out_h: u32,
+) -> screencapturekit::prelude::SCStreamConfiguration {
+ screencapturekit::prelude::SCStreamConfiguration::new()
+ .with_width(out_w)
+ .with_height(out_h)
+ .with_shows_cursor(false)
+}
+
/// Single-frame capture + RGBA/PNG encode from an existing plan.
///
/// https://developer.apple.com/documentation/screencapturekit/scscreenshotmanager
@@ -1143,6 +1161,16 @@ mod tests {
use std::rc::Rc;
use std::sync::Arc;
+ /// Regression for trycua/cua#5476: with the pointer outside the window,
+ /// a pointer-showing window capture came back scaled and offset while
+ /// still sized like the window, so clicks aimed from it missed by ~37 px.
+ #[test]
+ fn window_capture_config_leaves_the_pointer_out_at_the_planned_size() {
+ let config = window_capture_config(1240, 900);
+ assert!(!config.shows_cursor());
+ assert_eq!((config.width(), config.height()), (1240, 900));
+ }
+
#[test]
fn off_space_window_uses_skylight_before_public_backends() {
let public_calls = Cell::new(0u32);
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index 562b9208b9..dc6d8cda90 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -382,6 +382,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_popup_set_value_keeps_front_key_focus_background \
harness_appkit_context_menu_keeps_front_key_focus_background \
harness_appkit_slider_drag_px_foreground \
+ harness_appkit_window_capture_ignores_pointer_outside_window \
harness_appkit_slider_drag_px_background; do
run_test "appkit-${appkit_test}" cargo test -p cua-driver-e2e --test harness_appkit_test -- \
--ignored --exact "${appkit_test}" --nocapture --test-threads=1