Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep the pointer out of macOS window screenshots (#5480)
Francesco Bonacci757ae8cedb87
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rsindex 1e09774aa7..8181afa3bd 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs@@ -2374,3 +2374,128 @@ fn harness_appkit_context_menu_keeps_front_key_focus_background() {},);}++#[repr(C)]+#[derive(Clone, Copy)]+struct WarpPoint {+ x: f64,+ y: f64,+}++#[link(name = "CoreGraphics", kind = "framework")]+extern "C" {+ fn CGWarpMouseCursorPosition(point: WarpPoint) -> i32;+}++/// Regression for trycua/cua#5476. With the real pointer parked outside the+/// target window, a window screenshot came back the size of the window but+/// with the window drawn smaller (about 96.5%) and pushed down by ~31 px, so+/// pixel targets read off it missed by ~37 px. The window must fill its own+/// screenshot edge to edge, whatever the pointer is doing.+#[test]+#[ignore]+fn harness_appkit_window_capture_ignores_pointer_outside_window() {+ run_case(+ native_readonly_case(+ "appkit",+ "window_capture_ignores_pointer_outside_window",+ Targeting::NotApplicable,+ DriverRoute::WindowState,+ vec![OracleKind::Pixels],+ ),+ |pid, wid, driver| {+ use base64::Engine as _;++ // Offset the window from the screen origin and leave room above+ // it. Keep its current size: the harness window is not resizable+ // (no settable AXSize), and set_window_frame only writes AXSize+ // when the size changes.+ let listed = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 }));+ assert!(!listed.is_error(), "list_windows: {}", listed.text());+ let current = listed.structured()["windows"]+ .as_array()+ .and_then(|windows| {+ windows+ .iter()+ .find(|w| w["window_id"].as_u64() == Some(wid))+ })+ .map(|w| w["bounds"].clone())+ .expect("AppKit harness window bounds");+ let width = current["width"].as_f64().expect("window width");+ let height = current["height"].as_f64().expect("window height");+ let (x, y) = (180.0, 260.0);+ let moved = driver.call(+ "set_window_frame",+ serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "x": x, "y": y, "width": width, "height": height+ }),+ );+ assert!(!moved.is_error(), "set_window_frame: {}", moved.text());++ // Park the real pointer just above the window, outside its frame+ // but close enough for ScreenCaptureKit to fold it into the+ // capture (a benchmark parked it 27 pt above a window's top edge).+ // SAFETY: plain CoreGraphics call with a by-value point.+ unsafe {+ CGWarpMouseCursorPosition(WarpPoint {+ x: x + width / 2.0,+ y: y - 24.0,+ })+ };+ std::thread::sleep(Duration::from_millis(300));++ let snapshot = snapshot_elements(driver, pid, wid);+ assert!(!snapshot.is_error(), "capture: {}", snapshot.text());+ let structured = snapshot.structured();+ assert_eq!(structured["screenshot_frame_valid"], true, "{structured}");+ let scale = structured["screenshot_scale"]+ .as_f64()+ .expect("screenshot scale");+ let bounds = &structured["window_bounds"];+ let data = snapshot.raw["result"]["content"]+ .as_array()+ .and_then(|parts| parts.iter().find(|part| part["type"] == "image"))+ .and_then(|part| part["data"].as_str())+ .expect("window screenshot image part");+ let png = base64::engine::general_purpose::STANDARD+ .decode(data)+ .expect("decode screenshot");+ let image = image::load_from_memory(&png)+ .expect("load screenshot")+ .to_rgba8();+ let (w, h) = image.dimensions();+ assert_eq!(+ (f64::from(w), f64::from(h)),+ (+ (bounds["width"].as_f64().unwrap() * scale).round(),+ (bounds["height"].as_f64().unwrap() * scale).round()+ ),+ "screenshot size must match the window bounds at the reported scale"+ );++ // The bug left a transparent band across the top and down the+ // right side of the image. Sample just inside each edge, away+ // from the rounded corners: every sample must be the window's own+ // opaque pixels.+ let inset = (4.0 * scale) as u32;+ let samples = [+ (w / 2, inset), // top centre (title bar)+ (w / 4, inset), // top, left of centre+ (3 * w / 4, inset), // top, right of centre+ (w - 1 - inset, h / 2), // right edge+ (w - 1 - inset, h / 4), // right edge, upper+ (w / 2, h - 1 - inset), // bottom centre+ ];+ for (sx, sy) in samples {+ let alpha = image.get_pixel(sx, sy).0[3];+ assert_eq!(+ alpha, 255,+ "pixel ({sx},{sy}) of a {w}x{h} window screenshot is transparent: \+ the window does not fill its own capture"+ );+ }+ Observation::delivered(vec![OracleKind::Pixels], Evidence::default())+ },+ );+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rsindex fd519a81c9..dff4d82bd1 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs@@ -463,7 +463,7 @@ fn build_window_capture_plan(window_id: u32,expected_identity: WindowCaptureIdentity,) -> anyhow::Result<std::sync::Arc<WindowCapturePlan>> {- use screencapturekit::prelude::{SCContentFilter, SCShareableContent, SCStreamConfiguration};+ use screencapturekit::prelude::{SCContentFilter, SCShareableContent};let content = SCShareableContent::get().map_err(|e| anyhow::anyhow!("SCShareableContent::get failed: {e}"))?;@@ -519,9 +519,7 @@ fn build_window_capture_plan(let out_w = rounded_pixel_dim(width_pts * scale, "width")?;let out_h = rounded_pixel_dim(height_pts * scale, "height")?;- let config = SCStreamConfiguration::new()- .with_width(out_w)- .with_height(out_h);+ let config = window_capture_config(out_w, out_h);Ok(std::sync::Arc::new(WindowCapturePlan {filter,@@ -530,6 +528,26 @@ fn build_window_capture_plan(}))}+/// Output configuration for a single-window capture.+///+/// The hardware pointer stays out. ScreenCaptureKit draws it by default, and+/// with a desktop-independent window filter it grows the captured area to+/// take in a pointer that sits outside the window, then shrinks that larger+/// area into the configured `width`x`height`. The window then shows up scaled+/// (for example to 96.5%) and offset inside an image whose size still matches+/// its bounds, so the frame check passes with scale 1 and every pixel+/// coordinate read off it lands off target. `screencapture -l` and the+/// SkyLight path never include the pointer either.+fn window_capture_config(+ out_w: u32,+ out_h: u32,+) -> screencapturekit::prelude::SCStreamConfiguration {+ screencapturekit::prelude::SCStreamConfiguration::new()+ .with_width(out_w)+ .with_height(out_h)+ .with_shows_cursor(false)+}+/// Single-frame capture + RGBA/PNG encode from an existing plan.////// https://developer.apple.com/documentation/screencapturekit/scscreenshotmanager@@ -1143,6 +1161,16 @@ mod tests {use std::rc::Rc;use std::sync::Arc;+ /// Regression for trycua/cua#5476: with the pointer outside the window,+ /// a pointer-showing window capture came back scaled and offset while+ /// still sized like the window, so clicks aimed from it missed by ~37 px.+ #[test]+ fn window_capture_config_leaves_the_pointer_out_at_the_planned_size() {+ let config = window_capture_config(1240, 900);+ assert!(!config.shows_cursor());+ assert_eq!((config.width(), config.height()), (1240, 900));+ }+#[test]fn off_space_window_uses_skylight_before_public_backends() {let public_calls = Cell::new(0u32);diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex 562b9208b9..dc6d8cda90 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -382,6 +382,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_popup_set_value_keeps_front_key_focus_background \harness_appkit_context_menu_keeps_front_key_focus_background \harness_appkit_slider_drag_px_foreground \+ harness_appkit_window_capture_ignores_pointer_outside_window \harness_appkit_slider_drag_px_background; dorun_test "appkit-${appkit_test}" cargo test -p cua-driver-e2e --test harness_appkit_test -- \--ignored --exact "${appkit_test}" --nocapture --test-threads=1