Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
perf(cua-driver): accelerate macOS window capture with ScreenCaptureKit (#2795)
Richard Bankolebc90373362cb
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rsindex 9220b481f8..88ea72f1ae 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs@@ -1,16 +1,172 @@-//! Window / display screenshot using the macOS `screencapture` CLI tool.+//! Window / display screenshot capture for macOS.//!-//! `screencapture -l <windowID> -x -o <file>` captures a single window by-//! CGWindowID to a PNG without any screen-recording permission dialog.+//! ## Window capture (primary: ScreenCaptureKit)//!-//! `screencapture -x <file>` captures the full main display.+//! Single-frame window capture prefers native ScreenCaptureKit via+//! `SCScreenshotManager::capture_image` with a desktop-independent window+//! filter, then encodes PNG in memory. No subprocess, temp file, or base64+//! on the native success path.//!-//! For production use, the ImageIO/CGWindowListCreateImageFromArray path-//! would give lower overhead (no subprocess + temp file), but the subprocess-//! approach is simpler to implement correctly and is reliable across OS versions.+//! A process-local bounded warm cache (TTL 2s, capacity 32) reuses+//! `SCContentFilter` + `SCStreamConfiguration` plans across rapid captures of+//! the same window id so warm hits skip `SCShareableContent::get` and plan+//! construction. This is not a persistent `SCStream` or frame cache.+//!+//! Sources:+//! - https://developer.apple.com/documentation/screencapturekit/scscreenshotmanager+//! - https://developer.apple.com/documentation/screencapturekit/sccontentfilter/init(desktopindependentwindow:)+//! - https://docs.rs/screencapturekit/6.0.1/screencapturekit/+//!+//! Compatibility fallback: `screencapture -l <windowID> -x -o <file>` when+//! the native path errors or returns empty bytes.+//!+//! ## Display capture+//!+//! `screencapture -x <file>` still captures the full main display (unchanged+//! in this slice).use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+use std::collections::HashMap;+use std::hash::Hash;use std::process::Command;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{mpsc, Mutex, OnceLock};+use std::time::{Duration, Instant};++/// Bounded TTL map keyed by insertion time. Std-only; used for the warm SCK+/// window filter/config cache and unit-tested independently of ScreenCaptureKit.+struct TimedCache<K, V> {+ ttl: Duration,+ capacity: usize,+ next_seq: u64,+ map: HashMap<K, TimedCacheEntry<V>>,+}++struct TimedCacheEntry<V> {+ value: V,+ inserted_at: Instant,+ /// Monotonic insertion sequence for oldest-first eviction (stable ties).+ seq: u64,+}++impl<K, V> TimedCache<K, V>+where+ K: Eq + Hash + Clone,+{+ fn new(ttl: Duration, capacity: usize) -> Self {+ Self {+ ttl,+ capacity,+ next_seq: 0,+ map: HashMap::new(),+ }+ }++ #[cfg(test)]+ fn len(&self) -> usize {+ self.map.len()+ }++ /// Insert or replace `key`. Capacity 0 stores nothing. At capacity, a new+ /// key evicts the oldest insertion. Replacing an existing key refreshes+ /// its timestamp and insertion order.+ fn insert_at(&mut self, key: K, value: V, now: Instant) {+ if self.capacity == 0 {+ return;+ }++ // Do not let expired entries occupy capacity merely because their+ // individual keys have not been queried again.+ self.map.retain(|_, entry| {+ now.checked_duration_since(entry.inserted_at)+ .unwrap_or(Duration::ZERO)+ < self.ttl+ });++ if self.map.contains_key(&key) {+ let seq = self.alloc_seq();+ // Key present: refresh value, timestamp, and insertion order.+ let entry = self.map.get_mut(&key).expect("contains_key just true");+ entry.value = value;+ entry.inserted_at = now;+ entry.seq = seq;+ return;+ }++ while self.map.len() >= self.capacity {+ let oldest_key = self+ .map+ .iter()+ .min_by_key(|(_, entry)| entry.seq)+ .map(|(k, _)| k.clone());+ match oldest_key {+ Some(k) => {+ self.map.remove(&k);+ }+ None => break,+ }+ }++ let seq = self.alloc_seq();+ self.map.insert(+ key,+ TimedCacheEntry {+ value,+ inserted_at: now,+ seq,+ },+ );+ }++ /// Clone a fresh value for `key`. Removes the entry when its age is+ /// greater than or equal to TTL. If `now` is earlier than insertion, age is+ /// treated as zero (still fresh).+ fn get_cloned_at(&mut self, key: &K, now: Instant) -> Option<V>+ where+ V: Clone,+ {+ let expired = match self.map.get(key) {+ Some(entry) => {+ let age = now+ .checked_duration_since(entry.inserted_at)+ .unwrap_or(Duration::ZERO);+ age >= self.ttl+ }+ None => return None,+ };++ if expired {+ self.map.remove(key);+ None+ } else {+ self.map.get(key).map(|entry| entry.value.clone())+ }+ }++ /// Remove `key` only when `pred` accepts the stored value. Used to drop a+ /// stale plan without clobbering a concurrently refreshed entry.+ fn remove_if<F>(&mut self, key: &K, pred: F) -> bool+ where+ F: FnOnce(&V) -> bool,+ {+ let should_remove = match self.map.get(key) {+ Some(entry) => pred(&entry.value),+ None => return false,+ };+ if should_remove {+ self.map.remove(key);+ true+ } else {+ false+ }+ }++ fn alloc_seq(&mut self) -> u64 {+ let seq = self.next_seq;+ self.next_seq = self.next_seq.wrapping_add(1);+ seq+ }+}struct SecureCapturePath {directory: std::path::PathBuf,@@ -39,9 +195,41 @@ impl Drop for SecureCapturePath {}}-/// Capture a window by its `window_id` (CGWindowID).-/// Returns raw PNG bytes or an error.-pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {+/// Prefer `native`; on native error or empty bytes, call `fallback`.+///+/// On non-empty native success, returns those bytes without invoking+/// fallback. When both paths fail, the error message preserves both+/// contexts. Closures are not required to be `Send`/`Sync`.+fn capture_window_with_backends<N, F>(+ window_id: u32,+ native: N,+ fallback: F,+) -> anyhow::Result<Vec<u8>>+where+ N: FnOnce(u32) -> anyhow::Result<Vec<u8>>,+ F: FnOnce(u32) -> anyhow::Result<Vec<u8>>,+{+ match native(window_id) {+ Ok(bytes) if !bytes.is_empty() => Ok(bytes),+ Ok(_) => match fallback(window_id) {+ Ok(bytes) => Ok(bytes),+ Err(fallback_err) => Err(anyhow::anyhow!(+ "window {window_id} capture failed: native produced empty bytes; \+ shell fallback: {fallback_err:#}"+ )),+ },+ Err(native_err) => match fallback(window_id) {+ Ok(bytes) => Ok(bytes),+ Err(fallback_err) => Err(anyhow::anyhow!(+ "window {window_id} capture failed: native: {native_err:#}; \+ shell fallback: {fallback_err:#}"+ )),+ },+ }+}++/// Shell compatibility path: `screencapture -l <id> -x -o <tmp.png>`.+fn screenshot_window_bytes_shell(window_id: u32) -> anyhow::Result<Vec<u8>> {let capture = SecureCapturePath::new("window.png")?;let tmp_path = capture.file.to_string_lossy().into_owned();@@ -77,6 +265,474 @@ pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {Ok(bytes)}+/// Hard ceiling on capture dimensions to avoid unbounded allocations.+const MAX_CAPTURE_DIM: u32 = 16384;++fn content_rect_usable(rect: screencapturekit::cg::CGRect) -> bool {+ let w = rect.size.width;+ let h = rect.size.height;+ w.is_finite() && h.is_finite() && w > 0.0 && h > 0.0+}++/// Round a positive finite pixel extent into `1..=MAX_CAPTURE_DIM` as `u32`.+fn rounded_pixel_dim(value: f64, label: &str) -> anyhow::Result<u32> {+ if !value.is_finite() || value <= 0.0 {+ anyhow::bail!("invalid capture {label}: {value}");+ }+ let rounded = value.round();+ if !(rounded.is_finite()) || rounded < 1.0 || rounded > f64::from(MAX_CAPTURE_DIM) {+ anyhow::bail!("capture {label} {rounded} out of allowed range 1..={MAX_CAPTURE_DIM}");+ }+ // After the range check the value fits in u32.+ #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]+ let dim = rounded as u32;+ if dim == 0 || dim > MAX_CAPTURE_DIM {+ anyhow::bail!("capture {label} {dim} out of allowed range 1..={MAX_CAPTURE_DIM}");+ }+ Ok(dim)+}++fn checked_image_dim(value: usize, label: &str) -> anyhow::Result<u32> {+ if value == 0 || value > MAX_CAPTURE_DIM as usize {+ anyhow::bail!("{label} {value} out of allowed range 1..={MAX_CAPTURE_DIM}");+ }+ u32::try_from(value).map_err(|_| anyhow::anyhow!("{label} {value} does not fit u32"))+}++/// Owned ScreenCaptureKit filter + stream config for one window id.+struct WindowCapturePlan {+ filter: screencapturekit::prelude::SCContentFilter,+ config: screencapturekit::prelude::SCStreamConfiguration,+ identity: WindowCaptureIdentity,+}++/// Cheap WindowServer fingerprint used to reject a cached filter after a+/// resize/move, owner change, layer change, or CGWindowID reuse. Float fields+/// are kept as their exact bit patterns because both CGWindowList and+/// ScreenCaptureKit report the same WindowServer frame.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+struct WindowCaptureIdentity {+ pid: i32,+ layer: i32,+ x: u64,+ y: u64,+ width: u64,+ height: u64,+}++impl WindowCaptureIdentity {+ fn new(pid: i32, layer: i32, x: f64, y: f64, width: f64, height: f64) -> Self {+ Self {+ pid,+ layer,+ x: x.to_bits(),+ y: y.to_bits(),+ width: width.to_bits(),+ height: height.to_bits(),+ }+ }+}++fn current_window_capture_identity(window_id: u32) -> anyhow::Result<WindowCaptureIdentity> {+ let info = crate::windows::window_info_by_id(window_id)+ .ok_or_else(|| anyhow::anyhow!("WindowServer no longer lists window id {window_id}"))?;+ Ok(WindowCaptureIdentity::new(+ info.pid,+ info.layer,+ info.bounds.x,+ info.bounds.y,+ info.bounds.width,+ info.bounds.height,+ ))+}++const WINDOW_PLAN_CACHE_TTL: Duration = Duration::from_secs(2);+const WINDOW_PLAN_CACHE_CAPACITY: usize = 32;+/// Keep native capture below the public observation deadline after the AX+/// walk's own 20-second bound. The shell fallback retains its prior behavior.+const WINDOW_CAPTURE_NATIVE_TIMEOUT: Duration = Duration::from_secs(3);++/// Permit at most one ScreenCaptureKit worker. If an Apple completion callback+/// never fires, the timed-out worker keeps this permit, and every later request+/// goes straight to the compatibility fallback instead of leaking more+/// blocked threads.+struct NativeCaptureGate {+ active: AtomicBool,+}++impl NativeCaptureGate {+ const fn new() -> Self {+ Self {+ active: AtomicBool::new(false),+ }+ }++ fn try_acquire(&'static self) -> Option<NativeCapturePermit> {+ self.active+ .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)+ .ok()+ .map(|_| NativeCapturePermit { gate: self })+ }+}++struct NativeCapturePermit {+ gate: &'static NativeCaptureGate,+}++impl Drop for NativeCapturePermit {+ fn drop(&mut self) {+ self.gate.active.store(false, Ordering::Release);+ }+}++fn native_capture_gate() -> &'static NativeCaptureGate {+ static GATE: NativeCaptureGate = NativeCaptureGate::new();+ &GATE+}++fn run_native_capture_worker<T, F>(+ gate: &'static NativeCaptureGate,+ timeout: Duration,+ work: F,+) -> anyhow::Result<T>+where+ T: Send + 'static,+ F: FnOnce() -> anyhow::Result<T> + Send + 'static,+{+ let permit = gate+ .try_acquire()+ .ok_or_else(|| anyhow::anyhow!("ScreenCaptureKit capture already in flight"))?;+ let (sender, receiver) = mpsc::sync_channel(1);+ std::thread::Builder::new()+ .name("cua-sck-window-capture".into())+ .spawn(move || {+ let result = work();+ drop(permit);+ let _ = sender.send(result);+ })+ .map_err(|error| anyhow::anyhow!("failed to start ScreenCaptureKit worker: {error}"))?;++ match receiver.recv_timeout(timeout) {+ Ok(result) => result,+ Err(mpsc::RecvTimeoutError::Timeout) => {+ anyhow::bail!(+ "ScreenCaptureKit capture timed out after {} ms",+ timeout.as_millis()+ )+ }+ Err(mpsc::RecvTimeoutError::Disconnected) => {+ anyhow::bail!("ScreenCaptureKit worker ended without a result")+ }+ }+}++fn window_plan_cache() -> &'static Mutex<TimedCache<u32, std::sync::Arc<WindowCapturePlan>>> {+ static CACHE: OnceLock<Mutex<TimedCache<u32, std::sync::Arc<WindowCapturePlan>>>> =+ OnceLock::new();+ CACHE.get_or_init(|| {+ Mutex::new(TimedCache::new(+ WINDOW_PLAN_CACHE_TTL,+ WINDOW_PLAN_CACHE_CAPACITY,+ ))+ })+}++fn lock_window_plan_cache(+) -> std::sync::MutexGuard<'static, TimedCache<u32, std::sync::Arc<WindowCapturePlan>>> {+ window_plan_cache()+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+}++/// Look up shareable content, build desktop-independent filter + pixel config.+/// Runs outside the cache lock.+fn build_window_capture_plan(+ window_id: u32,+ expected_identity: WindowCaptureIdentity,+) -> anyhow::Result<std::sync::Arc<WindowCapturePlan>> {+ use screencapturekit::prelude::{SCContentFilter, SCShareableContent, SCStreamConfiguration};++ let content = SCShareableContent::get()+ .map_err(|e| anyhow::anyhow!("SCShareableContent::get failed: {e}"))?;++ let window = content+ .windows()+ .into_iter()+ .find(|w| w.window_id() == window_id)+ .ok_or_else(|| {+ anyhow::anyhow!(+ "ScreenCaptureKit: window id {window_id} not found in shareable content \+ (closed, not capturable, or no longer listed)"+ )+ })?;++ let frame = window.frame();+ let owner_pid = window+ .owning_application()+ .map(|application| application.process_id())+ .ok_or_else(|| {+ anyhow::anyhow!("ScreenCaptureKit: window id {window_id} has no owning application")+ })?;+ let actual_identity = WindowCaptureIdentity::new(+ owner_pid,+ window.window_layer(),+ frame.origin.x,+ frame.origin.y,+ frame.size.width,+ frame.size.height,+ );+ if actual_identity != expected_identity {+ anyhow::bail!(+ "ScreenCaptureKit: window id {window_id} changed identity while capture was prepared"+ );+ }++ // Desktop-independent window filter (captures only the specified window):+ // https://developer.apple.com/documentation/screencapturekit/sccontentfilter/init(desktopindependentwindow:)+ let filter = SCContentFilter::create().with_window(&window).build();++ // Pixel output size = content_rect * point_pixel_scale (macOS 14+ filter info).+ // https://docs.rs/screencapturekit/6.0.1/screencapturekit/+ let scale = f64::from(filter.point_pixel_scale());+ let (width_pts, height_pts) = {+ let rect = filter.content_rect();+ if content_rect_usable(rect) {+ (rect.size.width, rect.size.height)+ } else {+ (frame.size.width, frame.size.height)+ }+ };++ let out_w = rounded_pixel_dim(width_pts * scale, "width")?;+ let out_h = rounded_pixel_dim(height_pts * scale, "height")?;++ let config = SCStreamConfiguration::new()+ .with_width(out_w)+ .with_height(out_h);++ Ok(std::sync::Arc::new(WindowCapturePlan {+ filter,+ config,+ identity: expected_identity,+ }))+}++/// Single-frame capture + RGBA/PNG encode from an existing plan.+///+/// https://developer.apple.com/documentation/screencapturekit/scscreenshotmanager+fn capture_window_from_plan(window_id: u32, plan: &WindowCapturePlan) -> anyhow::Result<Vec<u8>> {+ use screencapturekit::screenshot_manager::{CGImageExt, SCScreenshotManager};++ let image = SCScreenshotManager::capture_image(&plan.filter, &plan.config).map_err(|e| {+ anyhow::anyhow!("SCScreenshotManager::capture_image failed for window {window_id}: {e}")+ })?;++ let w = checked_image_dim(image.width(), "CGImage width")?;+ let h = checked_image_dim(image.height(), "CGImage height")?;++ let rgba = image+ .rgba_data()+ .map_err(|e| anyhow::anyhow!("CGImage::rgba_data failed for window {window_id}: {e}"))?;++ let expected_len = (w as u64)+ .checked_mul(h as u64)+ .and_then(|n| n.checked_mul(4))+ .ok_or_else(|| anyhow::anyhow!("RGBA byte length overflow for {w}x{h}"))?;+ if rgba.len() as u64 != expected_len {+ anyhow::bail!(+ "CGImage RGBA length {} != {w}*{h}*4 ({expected_len}) for window {window_id}",+ rgba.len()+ );+ }++ cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h)+}++enum CaptureIdentityValidation {+ Matched(Vec<u8>),+ Changed(WindowCaptureIdentity),+}++/// Do not release bytes until WindowServer confirms that the window still has+/// the identity the plan was built for. The injected form keeps the TOCTOU+/// contract deterministic and independently testable.+fn capture_then_validate_identity<C, I>(+ expected_identity: WindowCaptureIdentity,+ capture: C,+ current_identity: I,+) -> anyhow::Result<CaptureIdentityValidation>+where+ C: FnOnce() -> anyhow::Result<Vec<u8>>,+ I: FnOnce() -> anyhow::Result<WindowCaptureIdentity>,+{+ let bytes = capture()?;+ let actual_identity = current_identity()?;+ if actual_identity == expected_identity {+ Ok(CaptureIdentityValidation::Matched(bytes))+ } else {+ Ok(CaptureIdentityValidation::Changed(actual_identity))+ }+}++fn capture_window_from_plan_validated(+ window_id: u32,+ plan: &WindowCapturePlan,+) -> anyhow::Result<CaptureIdentityValidation> {+ capture_then_validate_identity(+ plan.identity,+ || capture_window_from_plan(window_id, plan),+ || current_window_capture_identity(window_id),+ )+}++fn evict_window_capture_plan(window_id: u32, plan: &std::sync::Arc<WindowCapturePlan>) {+ let mut cache = lock_window_plan_cache();+ cache.remove_if(&window_id, |stored| std::sync::Arc::ptr_eq(stored, plan));+}++/// A successful capture discovered that the window mutated. Rebuild once for+/// the observed identity; a second mutation fails closed so stale bytes can+/// never escape and the shell compatibility path can take over.+fn retry_after_identity_change(+ window_id: u32,+ stale_plan: &std::sync::Arc<WindowCapturePlan>,+ actual_identity: WindowCaptureIdentity,+) -> anyhow::Result<Vec<u8>> {+ evict_window_capture_plan(window_id, stale_plan);+ let rebuilt = build_window_capture_plan(window_id, actual_identity)?;+ {+ let mut cache = lock_window_plan_cache();+ cache.insert_at(window_id, std::sync::Arc::clone(&rebuilt), Instant::now());+ }++ match capture_window_from_plan_validated(window_id, &rebuilt) {+ Ok(CaptureIdentityValidation::Matched(bytes)) => Ok(bytes),+ Ok(CaptureIdentityValidation::Changed(_)) => {+ evict_window_capture_plan(window_id, &rebuilt);+ anyhow::bail!(+ "ScreenCaptureKit: window id {window_id} changed identity again during retry"+ )+ }+ Err(error) => {+ evict_window_capture_plan(window_id, &rebuilt);+ Err(error)+ }+ }+}++/// Native ScreenCaptureKit single-frame window capture (in-process PNG).+///+/// Uses a desktop-independent window filter and `SCScreenshotManager` —+/// see module docs for Apple + crate source URLs. No subprocess/temp/base64.+/// Warm hits reuse a bounded two-second filter/config plan cache.+fn screenshot_window_bytes_sck_inner(window_id: u32) -> anyhow::Result<Vec<u8>> {+ let identity = current_window_capture_identity(window_id)?;+ let cached = {+ let mut cache = lock_window_plan_cache();+ cache.get_cloned_at(&window_id, Instant::now())+ };++ if let Some(plan) = cached {+ if plan.identity != identity {+ let mut cache = lock_window_plan_cache();+ cache.remove_if(&window_id, |stored| std::sync::Arc::ptr_eq(stored, &plan));+ } else {+ match capture_window_from_plan_validated(window_id, &plan) {+ Ok(CaptureIdentityValidation::Matched(bytes)) => return Ok(bytes),+ Ok(CaptureIdentityValidation::Changed(actual_identity)) => {+ return retry_after_identity_change(window_id, &plan, actual_identity);+ }+ Err(first_err) => {+ // Drop only this Arc if it is still the cached value.+ evict_window_capture_plan(window_id, &plan);++ let rebuilt = match build_window_capture_plan(window_id, identity) {+ Ok(plan) => plan,+ Err(build_err) => {+ return Err(anyhow::anyhow!(+ "ScreenCaptureKit window {window_id}: cached plan capture failed: \+ {first_err:#}; rebuild failed: {build_err:#}"+ ));+ }+ };++ {+ let mut cache = lock_window_plan_cache();+ cache.insert_at(window_id, std::sync::Arc::clone(&rebuilt), Instant::now());+ }++ match capture_window_from_plan_validated(window_id, &rebuilt) {+ Ok(CaptureIdentityValidation::Matched(bytes)) => return Ok(bytes),+ Ok(CaptureIdentityValidation::Changed(_)) => {+ evict_window_capture_plan(window_id, &rebuilt);+ return Err(anyhow::anyhow!(+ "ScreenCaptureKit window {window_id}: cached plan capture failed: \+ {first_err:#}; window changed identity during retry"+ ));+ }+ Err(retry_err) => {+ evict_window_capture_plan(window_id, &rebuilt);+ return Err(anyhow::anyhow!(+ "ScreenCaptureKit window {window_id}: cached plan capture failed: \+ {first_err:#}; retry after rebuild failed: {retry_err:#}"+ ));+ }+ }+ }+ }+ }+ }++ // Cache miss: build outside the lock, then insert, then capture.+ // Failed builds are never inserted. Capture failure returns as-is (no+ // pointless second rebuild) so the shell fallback can run.+ let plan = build_window_capture_plan(window_id, identity)?;+ {+ let mut cache = lock_window_plan_cache();+ cache.insert_at(window_id, std::sync::Arc::clone(&plan), Instant::now());+ }+ match capture_window_from_plan_validated(window_id, &plan) {+ Ok(CaptureIdentityValidation::Matched(bytes)) => Ok(bytes),+ Ok(CaptureIdentityValidation::Changed(actual_identity)) => {+ retry_after_identity_change(window_id, &plan, actual_identity)+ }+ Err(error) => {+ evict_window_capture_plan(window_id, &plan);+ Err(error)+ }+ }+}++/// Bound the dependency's synchronous callback wrappers without permitting an+/// unbounded number of detached workers. Ownership of the gate permit and all+/// ScreenCaptureKit/CF objects stays on the worker until it really returns.+fn screenshot_window_bytes_sck(window_id: u32) -> anyhow::Result<Vec<u8>> {+ run_native_capture_worker(+ native_capture_gate(),+ WINDOW_CAPTURE_NATIVE_TIMEOUT,+ move || screenshot_window_bytes_sck_inner(window_id),+ )+ .map_err(|error| {+ anyhow::anyhow!(+ "ScreenCaptureKit capture failed for window {window_id}: {error:#}; \+ using shell fallback"+ )+ })+}++/// Capture a window by its `window_id` (CGWindowID).+/// Returns raw PNG bytes or an error.+///+/// Tries ScreenCaptureKit first; falls back to the `screencapture` CLI on+/// native error or empty output.+pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {+ capture_window_with_backends(+ window_id,+ screenshot_window_bytes_sck,+ screenshot_window_bytes_shell,+ )+}+/// Capture a window by its `window_id` (CGWindowID)./// Returns (base64-encoded PNG, width, height) or an error.pub fn screenshot_window(window_id: u32) -> anyhow::Result<(String, u32, u32)> {@@ -163,3 +819,289 @@ pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> anyhow::Resultpub fn png_dimensions(data: &[u8]) -> anyhow::Result<(u32, u32)> {cua_driver_core::image_utils::png_dimensions(data)}++#[cfg(test)]+mod tests {+ use super::*;+ use std::cell::Cell;+ use std::rc::Rc;+ use std::sync::Arc;++ #[test]+ fn native_window_capture_short_circuits_shell_fallback() {+ let png = cua_driver_core::image_utils::encode_rgba_to_png(&[0, 0, 0, 255], 1, 1)+ .expect("encode 1x1 PNG");+ assert!(!png.is_empty(), "PNG bytes must be non-empty");++ let native_calls = Rc::new(Cell::new(0u32));+ let fallback_calls = Rc::new(Cell::new(0u32));+ let native_window_id = Rc::new(Cell::new(None::<u32>));++ let native_calls_n = Rc::clone(&native_calls);+ let native_window_id_n = Rc::clone(&native_window_id);+ let png_n = png.clone();+ let fallback_calls_f = Rc::clone(&fallback_calls);++ let got = capture_window_with_backends(+ 42,+ move |window_id| {+ native_calls_n.set(native_calls_n.get() + 1);+ native_window_id_n.set(Some(window_id));+ Ok(png_n)+ },+ move |_window_id| {+ fallback_calls_f.set(fallback_calls_f.get() + 1);+ anyhow::bail!("shell fallback must not run when native succeeds");+ },+ )+ .expect("native capture should succeed");++ assert_eq!(native_calls.get(), 1, "native backend called once");+ assert_eq!(+ native_window_id.get(),+ Some(42),+ "native backend receives window id 42"+ );+ assert_eq!(fallback_calls.get(), 0, "shell fallback must not run");+ assert_eq!(got, png, "helper returns native PNG bytes verbatim");+ }++ #[test]+ fn empty_native_capture_uses_shell_fallback() {+ let got = capture_window_with_backends(+ 42,+ |_| Ok(Vec::new()),+ |window_id| Ok(format!("fallback-{window_id}").into_bytes()),+ )+ .expect("empty native capture should fall back");++ assert_eq!(got, b"fallback-42");+ }++ #[test]+ fn capture_error_preserves_native_and_fallback_contexts() {+ let error = capture_window_with_backends(+ 42,+ |_| anyhow::bail!("native denied"),+ |_| anyhow::bail!("fallback denied"),+ )+ .expect_err("both capture backends should fail")+ .to_string();++ assert!(error.contains("native denied"), "{error}");+ assert!(error.contains("fallback denied"), "{error}");+ }++ #[test]+ fn native_window_capture_cache_reuses_fresh_and_expires_stale_entries() {+ use std::time::{Duration, Instant};++ let t0 = Instant::now();+ let mut cache: TimedCache<u32, Arc<&'static str>> =+ TimedCache::new(Duration::from_secs(2), 4);++ let descriptor = Arc::new("descriptor");+ cache.insert_at(42, Arc::clone(&descriptor), t0);+ assert_eq!(cache.len(), 1);++ let fresh = cache+ .get_cloned_at(&42, t0 + Duration::from_millis(1500))+ .expect("fresh cache hit within TTL");+ assert!(+ Arc::ptr_eq(&fresh, &descriptor),+ "fresh hit must return the same Arc"+ );+ assert_eq!(cache.len(), 1);++ let stale = cache.get_cloned_at(&42, t0 + Duration::from_secs(2));+ assert!(stale.is_none(), "entry must expire at TTL");+ assert_eq!(cache.len(), 0, "stale entry must be removed on get");+ }++ #[test]+ fn native_window_capture_cache_purges_expired_before_capacity_eviction() {+ let t0 = Instant::now();+ let mut cache = TimedCache::new(Duration::from_secs(2), 2);+ cache.insert_at(1, "expired", t0);+ cache.insert_at(2, "fresh", t0 + Duration::from_millis(1500));++ cache.insert_at(3, "new", t0 + Duration::from_secs(2));++ assert_eq!(cache.len(), 2);+ assert!(cache+ .get_cloned_at(&1, t0 + Duration::from_secs(2))+ .is_none());+ assert_eq!(+ cache.get_cloned_at(&2, t0 + Duration::from_secs(2)),+ Some("fresh")+ );+ assert_eq!(+ cache.get_cloned_at(&3, t0 + Duration::from_secs(2)),+ Some("new")+ );+ }++ #[test]+ fn native_window_capture_cache_remove_if_does_not_remove_replacement() {+ let t0 = Instant::now();+ let mut cache = TimedCache::new(Duration::from_secs(2), 2);+ let old = Arc::new("old");+ let replacement = Arc::new("replacement");+ cache.insert_at(42, Arc::clone(&old), t0);+ cache.insert_at(42, Arc::clone(&replacement), t0);++ assert!(!cache.remove_if(&42, |stored| Arc::ptr_eq(stored, &old)));+ let stored = cache+ .get_cloned_at(&42, t0)+ .expect("replacement remains cached");+ assert!(Arc::ptr_eq(&stored, &replacement));+ }++ #[test]+ fn window_capture_identity_changes_with_owner_layer_or_frame() {+ let original = WindowCaptureIdentity::new(10, 0, 1.0, 2.0, 800.0, 600.0);+ assert_ne!(+ original,+ WindowCaptureIdentity::new(11, 0, 1.0, 2.0, 800.0, 600.0)+ );+ assert_ne!(+ original,+ WindowCaptureIdentity::new(10, 1, 1.0, 2.0, 800.0, 600.0)+ );+ assert_ne!(+ original,+ WindowCaptureIdentity::new(10, 0, 1.0, 2.0, 801.0, 600.0)+ );+ }++ #[test]+ fn post_capture_identity_change_never_returns_stale_bytes() {+ let before = WindowCaptureIdentity::new(10, 0, 1.0, 2.0, 800.0, 600.0);+ let after = WindowCaptureIdentity::new(10, 0, 5.0, 2.0, 800.0, 600.0);++ let outcome = capture_then_validate_identity(before, || Ok(vec![1, 2, 3]), || Ok(after))+ .expect("capture and identity read succeed");++ assert!(matches!(+ outcome,+ CaptureIdentityValidation::Changed(identity) if identity == after+ ));+ }++ #[test]+ fn second_identity_change_during_retry_also_fails_closed() {+ let first = WindowCaptureIdentity::new(10, 0, 1.0, 2.0, 800.0, 600.0);+ let second = WindowCaptureIdentity::new(10, 0, 5.0, 2.0, 800.0, 600.0);+ let third = WindowCaptureIdentity::new(10, 0, 5.0, 2.0, 900.0, 600.0);++ let first_attempt = capture_then_validate_identity(first, || Ok(vec![1]), || Ok(second))+ .expect("first attempt completes");+ assert!(matches!(+ first_attempt,+ CaptureIdentityValidation::Changed(identity) if identity == second+ ));++ let retry = capture_then_validate_identity(second, || Ok(vec![2]), || Ok(third))+ .expect("retry completes");+ assert!(matches!(+ retry,+ CaptureIdentityValidation::Changed(identity) if identity == third+ ));+ }++ #[test]+ fn identity_mismatch_eviction_does_not_clobber_replacement_cache_entry() {+ let now = Instant::now();+ let mut cache = TimedCache::new(Duration::from_secs(2), 1);+ let stale = std::sync::Arc::new(1u8);+ let replacement = std::sync::Arc::new(2u8);+ cache.insert_at(42, std::sync::Arc::clone(&stale), now);++ // Model another capture refreshing the same key before this stale+ // attempt handles its post-capture identity mismatch.+ cache.insert_at(42, std::sync::Arc::clone(&replacement), now);+ assert!(!cache.remove_if(&42, |stored| { std::sync::Arc::ptr_eq(stored, &stale) }));++ let stored = cache+ .get_cloned_at(&42, now)+ .expect("replacement remains cached");+ assert!(std::sync::Arc::ptr_eq(&stored, &replacement));+ }++ #[test]+ fn native_capture_gate_allows_only_one_worker_until_permit_drops() {+ static GATE: NativeCaptureGate = NativeCaptureGate::new();++ let permit = GATE.try_acquire().expect("first worker acquires gate");+ assert!(+ GATE.try_acquire().is_none(),+ "a second worker must fall back while the first is active"+ );++ drop(permit);+ assert!(+ GATE.try_acquire().is_some(),+ "gate reopens only when the worker-owned permit drops"+ );+ }++ #[test]+ fn timed_out_native_worker_blocks_replacement_until_original_returns() {+ use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};++ static GATE: NativeCaptureGate = NativeCaptureGate::new();+ static SPAWNED_WORK: AtomicUsize = AtomicUsize::new(0);+ let (started_sender, started_receiver) = mpsc::channel();+ let (release_sender, release_receiver) = mpsc::channel();++ let first_caller = std::thread::spawn(move || {+ run_native_capture_worker(&GATE, Duration::from_millis(10), move || {+ SPAWNED_WORK.fetch_add(1, AtomicOrdering::SeqCst);+ started_sender.send(()).expect("worker reports it started");+ release_receiver+ .recv()+ .expect("test releases stalled worker");+ Ok(1u8)+ })+ });+ started_receiver+ .recv()+ .expect("first worker starts before its timeout is observed");+ let first = first_caller.join().expect("first caller joins");+ assert!(first+ .expect_err("stalled worker times out")+ .to_string()+ .contains("timed out"));++ let second = run_native_capture_worker(&GATE, Duration::from_secs(1), || {+ SPAWNED_WORK.fetch_add(1, AtomicOrdering::SeqCst);+ Ok(2u8)+ });+ assert!(second+ .expect_err("replacement is refused while original remains blocked")+ .to_string()+ .contains("already in flight"));+ assert_eq!(+ SPAWNED_WORK.load(AtomicOrdering::SeqCst),+ 1,+ "refused replacement must not spawn"+ );++ release_sender.send(()).expect("release original worker");+ let deadline = Instant::now() + Duration::from_secs(1);+ while GATE.active.load(AtomicOrdering::Acquire) && Instant::now() < deadline {+ std::thread::yield_now();+ }+ assert!(!GATE.active.load(AtomicOrdering::Acquire));++ assert_eq!(+ run_native_capture_worker(&GATE, Duration::from_secs(1), || {+ SPAWNED_WORK.fetch_add(1, AtomicOrdering::SeqCst);+ Ok(3u8)+ })+ .expect("gate reopens after original worker returns"),+ 3+ );+ assert_eq!(SPAWNED_WORK.load(AtomicOrdering::SeqCst), 2);+ }+}