Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep the agent cursor out of desktop captures (#4199)
Francesco Bonaccifa5ded76dc40
diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.ymlindex f7ac382a5f..b0fba747ea 100644--- a/.github/workflows/ci-rust-linux.yml+++ b/.github/workflows/ci-rust-linux.yml@@ -161,6 +161,13 @@ jobs:cargo test -p platform-linux \x11_overlay_owner_stays_click_through_across_cursor_lifecycle_and_randr \--locked -- --ignored --nocapture+ - name: Run X11 desktop capture overlay-exclusion regression+ working-directory: libs/cua-driver/rust+ run: |+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \+ cargo test -p platform-linux \+ x11_desktop_capture_hides_the_resting_overlay_and_restores_it \+ --locked -- --ignored --nocapture- name: Run X11 clipboard round-trip regressionworking-directory: libs/cua-driver/rustrun: |diff --git a/docs/content/docs/reference/cua-driver/limits.mdx b/docs/content/docs/reference/cua-driver/limits.mdxindex 2542061964..2effdaaf99 100644--- a/docs/content/docs/reference/cua-driver/limits.mdx+++ b/docs/content/docs/reference/cua-driver/limits.mdx@@ -130,6 +130,23 @@ does not imply arbitrary raw background PX delivery on Wayland.---+## Desktop captures leave out the agent cursor++**Behavior:** `get_desktop_state` keeps the Driver's own agent cursor and session pill out of the PNG it returns, so a cursor resting on the control it just clicked does not cover that control's label. The on-screen cursor, external screen recorders, and the Driver's own recordings keep showing it. Every response reports the outcome in `structuredContent.agent_overlay_capture`:++- `excluded`, with the native `method`;+- `not_present`, when no overlay pixels were on screen;+- `not_excluded`, with a `reason`, when overlay pixels may be in the image. The text summary repeats the caveat.++**Platform scope:**++- **macOS:** the capture uses a ScreenCaptureKit display filter that leaves out the overlay window (macOS 14 or later). The capture falls back to `screencapture`, which reports `not_excluded`.+- **Windows:** the overlay is marked `WDA_EXCLUDEFROMCAPTURE` for the length of the capture (Windows 10 version 2004 or later). During that window, other capture tools also stop seeing the cursor. Older builds report `not_excluded`.+- **Linux X11:** the overlay hides for the length of the root grab, so the cursor blinks off briefly. Without a compositing manager, pixels that a slow window has not repainted are replaced with the desktop that the overlay recorded under them (`method` ends in `+x11_save_under`).+- **Linux Wayland:** the compositor draws the cursor into the output that the capture reads, and the Driver cannot leave it out. Captures report `not_excluded`.++---+## Off-Space SwiftUI windows strip their AX tree**Symptom:** `get_window_state({pid, window_id})` on a window on a different Space (e.g. System Settings parked on Space 2 while on Space 1) returns a minimal tree that contains only the menu bar, or just the `AXApplication` root.diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.jsonindex aa3e59fa27..b786ceef35 100644--- a/libs/cua-driver/contract/manifest.json+++ b/libs/cua-driver/contract/manifest.json@@ -1246,6 +1246,46 @@"success_output_schema": {"additionalProperties": true,"properties": {+ "agent_overlay_capture": {+ "additionalProperties": false,+ "properties": {+ "method": {+ "type": [+ "string",+ "null"+ ]+ },+ "reason": {+ "type": [+ "string",+ "null"+ ]+ },+ "status": {+ "oneOf": [+ {+ "const": "excluded",+ "type": "string"+ },+ {+ "const": "not_present",+ "type": "string"+ },+ {+ "const": "not_excluded",+ "type": "string"+ }+ ]+ }+ },+ "required": [+ "status"+ ],+ "type": [+ "object",+ "null"+ ]+ },"display": {"type": "string"},diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rsindex d0ef464eab..0cdde7377d 100644--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs@@ -44,13 +44,13 @@ pub use outputs::{refusal_envelope_schema, ActionDelivery, ActionDeliveryMode, ActionEffect, ActionError,ActionEscalation, ActionEscalationReason, ActionEscalationTarget, ActionEvidence,ActionEvidenceKind, ActionResult, ActionResultValidationError, ActionRoute,- ClipboardReadOutput, ClipboardWriteOutput, CursorMotionOutput, CursorPointOutput,- CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, DesktopStateOutput,- EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, ListSessionsOutput,- ScreenSizeOutput, SessionClientKindOutput, SessionLifecycleState, SessionOutput,- SessionStateOutput, SessionTransportOutput, SetAgentCursorEnabledOutput,- SetAgentCursorMotionOutput, SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput,- TOOL_INVOCATION_FAILED_CODE,+ AgentOverlayCapture, AgentOverlayCaptureStatus, ClipboardReadOutput, ClipboardWriteOutput,+ CursorMotionOutput, CursorPointOutput, CursorPositionOutput, CursorThemeOutput,+ CursorVisualOutput, DesktopStateOutput, EffectiveScope, EndSessionOutput,+ GetAgentCursorStateOutput, ListSessionsOutput, ScreenSizeOutput, SessionClientKindOutput,+ SessionLifecycleState, SessionOutput, SessionStateOutput, SessionTransportOutput,+ SetAgentCursorEnabledOutput, SetAgentCursorMotionOutput, SetAgentCursorThemeOutput,+ StartSessionOutput, ToolOutput, TOOL_INVOCATION_FAILED_CODE,};pub use verification::{BoundsExpectation, ElementPredicate, ElementSelector, PredicateOutcome, StatePredicate,diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rsindex 5ce0bdbfcf..7d05f009cd 100644--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs@@ -365,16 +365,92 @@ pub struct DesktopStateOutput {#[serde(default, skip_serializing_if = "Option::is_none")]#[schemars(schema_with = "string_schema")]pub screenshot_file_path: Option<String>,+ /// Whether the Driver's own overlay pixels (agent cursor, session pill)+ /// were kept out of this capture. Absent from producers that predate the+ /// report.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub agent_overlay_capture: Option<AgentOverlayCapture>,#[serde(flatten)]pub extensions: BTreeMap<String, Value>,}+/// How a Driver-owned desktop capture treated Driver-owned overlay pixels.+#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]+#[serde(rename_all = "snake_case")]+pub enum AgentOverlayCaptureStatus {+ /// Overlay windows were on screen and were kept out of the pixels.+ Excluded,+ /// No Driver overlay pixels were on screen, so there was nothing to keep out.+ NotPresent,+ /// Overlay pixels may be in the capture; `reason` says why.+ NotExcluded,+}++/// Report attached to desktop captures about Driver-owned overlay pixels.+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]+#[serde(deny_unknown_fields)]+pub struct AgentOverlayCapture {+ pub status: AgentOverlayCaptureStatus,+ /// Native mechanism that kept the overlay out, when `status` is `excluded`.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub method: Option<String>,+ /// Why the overlay could not be kept out, when `status` is `not_excluded`.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub reason: Option<String>,+}++impl AgentOverlayCapture {+ pub fn excluded(method: impl Into<String>) -> Self {+ Self {+ status: AgentOverlayCaptureStatus::Excluded,+ method: Some(method.into()),+ reason: None,+ }+ }++ pub fn not_present() -> Self {+ Self {+ status: AgentOverlayCaptureStatus::NotPresent,+ method: None,+ reason: None,+ }+ }++ pub fn not_excluded(reason: impl Into<String>) -> Self {+ Self {+ status: AgentOverlayCaptureStatus::NotExcluded,+ method: None,+ reason: Some(reason.into()),+ }+ }++ /// Whether the capture is free of Driver overlay pixels.+ pub fn is_clean(&self) -> bool {+ self.status != AgentOverlayCaptureStatus::NotExcluded+ }++ fn validate(&self) -> Result<(), String> {+ match (self.status, &self.method, &self.reason) {+ (AgentOverlayCaptureStatus::Excluded, Some(_), None)+ | (AgentOverlayCaptureStatus::NotPresent, None, None)+ | (AgentOverlayCaptureStatus::NotExcluded, None, Some(_)) => Ok(()),+ _ => Err(+ "agent_overlay_capture: excluded requires only method, not_excluded requires \+ only reason, not_present carries neither"+ .into(),+ ),+ }+ }+}+impl ToolOutput for DesktopStateOutput {fn validate(&self) -> Result<(), String> {- if self.screenshot_mime_type == "image/png" {- Ok(())- } else {- Err("screenshot_mime_type must be image/png".into())+ if self.screenshot_mime_type != "image/png" {+ return Err("screenshot_mime_type must be image/png".into());+ }+ match &self.agent_overlay_capture {+ Some(report) => report.validate(),+ None => Ok(()),}}}diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rsindex 95abd82341..7c4afdc718 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs@@ -118,6 +118,18 @@ fn semantic_cursor_showcase_records_session_and_action_states() {std::fs::write(&screenshot_path, cursor_png).expect("write cursor oracle screenshot");evidence.screenshot = Some(screenshot_path.display().to_string());+ // The cursor and pill keep resting there, yet the capture the Driver+ // hands the agent must not contain them (D-WL5).+ assert_driver_capture_excludes_overlay(+ &mut driver,+ &baseline,+ &cursor_frame,+ center_x - 180.0,+ center_y - 80.0,+ width,+ height,+ );+call_ok(&mut driver,"click",@@ -349,7 +361,121 @@ fn changed_pixels_in_rect(.count()}+/// Check the Driver's own desktop capture against the external cursor frame:+/// every pixel the overlay changed on screen must show the desktop there.+///+/// Native Wayland compositors draw the cursor into the captured output; there+/// the Driver must say so instead of claiming a clean capture.+fn assert_driver_capture_excludes_overlay(+ driver: &mut McpDriver,+ baseline: &RgbaImage,+ cursor_frame: &RgbaImage,+ logical_x: f64,+ logical_y: f64,+ logical_width: f64,+ logical_height: f64,+) {+ let (driver_png, _, _, report) = capture_desktop_png_with_report(driver);+ let status = report["status"].as_str().unwrap_or("missing");+ if wayland_session() {+ assert_eq!(+ status, "not_excluded",+ "a Wayland desktop capture must report that it could not exclude the overlay: \+ {report}"+ );+ return;+ }+ assert_eq!(+ status, "excluded",+ "desktop capture did not exclude the agent cursor overlay: {report}"+ );+ let driver_frame = image::load_from_memory(&driver_png)+ .expect("decode driver-owned desktop screenshot")+ .to_rgba8();+ assert_eq!(+ driver_frame.dimensions(),+ cursor_frame.dimensions(),+ "driver capture and cursor oracle are not in the same pixel frame"+ );+ let regions = cursor_oracle_regions(+ baseline.width(),+ baseline.height(),+ logical_x,+ logical_y,+ logical_width,+ logical_height,+ );+ let mut overlay_pixels = 0usize;+ let mut desktop_pixels = 0usize;+ for rect in [regions.pointer, regions.badge_left, regions.badge_right] {+ let (overlay, desktop) =+ overlay_pixels_resolved_to_desktop(baseline, cursor_frame, &driver_frame, rect);+ overlay_pixels += overlay;+ desktop_pixels += desktop;+ }+ assert!(+ overlay_pixels >= 36,+ "the cursor oracle no longer shows the overlay ({overlay_pixels} pixels)"+ );+ assert!(+ desktop_pixels * 10 >= overlay_pixels * 9,+ "driver desktop capture still shows the agent cursor overlay: only {desktop_pixels} \+ of {overlay_pixels} overlay pixels show the desktop; report={report}"+ );+}++/// Pixels in `rect` the overlay changed (baseline vs cursor frame), and how+/// many of those the driver capture shows closer to the desktop than to the+/// overlay. "Closer" keeps the check independent of small color differences+/// between capture pipelines.+fn overlay_pixels_resolved_to_desktop(+ baseline: &RgbaImage,+ cursor_frame: &RgbaImage,+ driver_frame: &RgbaImage,+ rect: PixelRect,+) -> (usize, usize) {+ let distance = |a: &image::Rgba<u8>, b: &image::Rgba<u8>| -> u16 {+ a.0.iter()+ .zip(b.0.iter())+ .take(3)+ .map(|(a, b)| u16::from(a.abs_diff(*b)))+ .sum()+ };+ let x0 = rect.x0.clamp(0, i64::from(baseline.width())) as u32;+ let x1 = rect.x1.clamp(0, i64::from(baseline.width())) as u32;+ let y0 = rect.y0.clamp(0, i64::from(baseline.height())) as u32;+ let y1 = rect.y1.clamp(0, i64::from(baseline.height())) as u32;+ let mut overlay = 0;+ let mut desktop = 0;+ for pixel_y in y0..y1 {+ for pixel_x in x0..x1 {+ let before = baseline.get_pixel(pixel_x, pixel_y);+ let on_screen = cursor_frame.get_pixel(pixel_x, pixel_y);+ if distance(before, on_screen) < 80 {+ continue;+ }+ overlay += 1;+ let captured = driver_frame.get_pixel(pixel_x, pixel_y);+ if distance(captured, before) < distance(captured, on_screen) {+ desktop += 1;+ }+ }+ }+ (overlay, desktop)+}++fn wayland_session() -> bool {+ cfg!(target_os = "linux") && std::env::var_os("WAYLAND_DISPLAY").is_some()+}+fn capture_desktop_png(driver: &mut McpDriver) -> (Vec<u8>, f64, f64) {+ let (png, width, height, _) = capture_desktop_png_with_report(driver);+ (png, width, height)+}++fn capture_desktop_png_with_report(+ driver: &mut McpDriver,+) -> (Vec<u8>, f64, f64, serde_json::Value) {let response = driver.call("get_desktop_state", serde_json::json!({}));assert!(!response.is_error(),@@ -378,7 +504,8 @@ fn capture_desktop_png(driver: &mut McpDriver) -> (Vec<u8>, f64, f64) {.as_f64().or_else(|| response.structured()["screenshot_height"].as_f64()).expect("desktop capture returned no logical height");- (png, width, height)+ let report = response.structured()["agent_overlay_capture"].clone();+ (png, width, height, report)}fn capture_cursor_oracle_png(driver: &mut McpDriver, width: f64, height: f64) -> Vec<u8> {@@ -426,10 +553,68 @@ fn capture_cursor_oracle_png(driver: &mut McpDriver, width: f64, height: f64) ->output.stdout}- #[cfg(not(target_os = "linux"))]+ // The Driver's own desktop capture leaves the overlay out, so the oracle+ // for what a user sees comes from another capture path: a BitBlt from this+ // (different) process, which the overlay's temporary capture exclusion+ // does not apply to.+ #[cfg(target_os = "windows")]+ {+ let _ = (driver, width, height);+ platform_windows::capture::screenshot_display_bytes()+ .expect("external Windows display capture failed")+ }++ // Only the Driver holds Screen Recording permission on the macOS lanes.+ // Its per-turn recording capture deliberately keeps the overlay, so the+ // move's post-action screenshot is what a user saw.+ #[cfg(target_os = "macos")]{let _ = (width, height);- capture_desktop_png(driver).0+ recorded_turn_screenshot(driver, "move_cursor")+ }+}++/// The newest recorded turn's post-action screenshot for `tool`.+#[cfg(target_os = "macos")]+fn recorded_turn_screenshot(driver: &McpDriver, tool: &str) -> Vec<u8> {+ let recording_dir = driver+ .recording_dir()+ .expect("showcase recording directory")+ .to_path_buf();+ let deadline = std::time::Instant::now() + Duration::from_secs(5);+ loop {+ let mut turns: Vec<_> = std::fs::read_dir(&recording_dir)+ .into_iter()+ .flatten()+ .flatten()+ .map(|entry| entry.path())+ .filter(|path| {+ path.file_name()+ .and_then(|name| name.to_str())+ .is_some_and(|name| name.starts_with("turn-"))+ })+ .collect();+ turns.sort();+ for turn in turns.iter().rev() {+ let Ok(action) = std::fs::read(turn.join("action.json")) else {+ continue;+ };+ let Ok(action) = serde_json::from_slice::<serde_json::Value>(&action) else {+ continue;+ };+ if action["tool"].as_str() != Some(tool) {+ continue;+ }+ if let Ok(png) = std::fs::read(turn.join("after.png")) {+ return png;+ }+ }+ assert!(+ std::time::Instant::now() < deadline,+ "no recorded {tool} turn with an after.png under {}",+ recording_dir.display()+ );+ settle(100);}}diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rsindex 3ee683820a..b727b6df52 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rs@@ -53,6 +53,14 @@ fn agent_cursor_overlay_obeys_untargeted_and_targeted_z_order() {let foreground_before = unsafe { GetForegroundWindow() };assert_eq!(foreground_before.0 as u64, target.native_id);let real_cursor_before = real_cursor_position();+ // The desktop under the cursor's resting point before any overlay+ // pixel exists, for the capture-exclusion oracle below.+ let baseline = image::load_from_memory(+ &platform_windows::capture::screenshot_display_bytes()+ .expect("baseline screenshot_display_bytes failed"),+ )+ .expect("decode baseline display screenshot")+ .to_rgba8();for (tool, arguments) in [(@@ -132,6 +140,58 @@ fn agent_cursor_overlay_obeys_untargeted_and_targeted_z_order() {"agent cursor not visible at ({x:.0},{y:.0}): only {visible_pixels} qualifying pixels");+ // D-WL5: the cursor keeps resting over the target, yet the Driver's+ // own desktop capture must not contain it, while the external capture+ // above (another process) still does.+ let capture_session = "windows-agent-cursor-desktop-capture";+ let started = driver.call(+ "start_session",+ serde_json::json!({"session": capture_session, "capture_scope": "desktop"}),+ );+ assert!(+ !started.is_error(),+ "desktop capture session failed: {}",+ started.text()+ );+ let capture_dir = tempfile::tempdir().expect("create desktop capture directory");+ let capture_path = capture_dir.path().join("desktop.png");+ let desktop = driver.call(+ "get_desktop_state",+ serde_json::json!({+ "session": capture_session,+ "screenshot_out_file": capture_path.to_string_lossy(),+ }),+ );+ assert!(+ !desktop.is_error(),+ "get_desktop_state failed: {}",+ desktop.text()+ );+ assert_eq!(+ desktop.structured()["agent_overlay_capture"]["status"].as_str(),+ Some("excluded"),+ "desktop capture did not exclude the agent cursor overlay: {}",+ desktop.text()+ );+ let driver_image = image::open(&capture_path)+ .expect("decode Driver desktop capture")+ .to_rgba8();+ let overlay_box = (x0, y0, x1, y1);+ let oracle = overlay_capture_oracle(&baseline, &image, &driver_image, overlay_box);+ eprintln!("agent cursor capture-exclusion oracle: {oracle:?}");+ assert!(+ oracle.overlay_pixels >= 20,+ "external capture barely differs from the baseline under the cursor: {oracle:?}"+ );+ assert!(+ oracle.left_in_driver_capture * 10 <= oracle.overlay_pixels,+ "Driver desktop capture still contains the agent cursor overlay: {oracle:?}"+ );+ assert!(+ oracle.desktop_in_driver_capture * 10 >= oracle.overlay_pixels * 8,+ "Driver desktop capture does not show the desktop under the cursor: {oracle:?}"+ );+// Capture the target while it is still foreground. Electron may prune// parts of its UIA tree once another full-size window covers it, but// the snapshot remains a stable handle for the later background click.@@ -423,6 +483,60 @@ fn wait_for_overlay() -> HWND {overlay}+/// Pixel accounting for one box: which pixels the overlay changed in an+/// external capture, and what the Driver's own capture shows there.+#[derive(Debug)]+struct OverlayCaptureOracle {+ /// Pixels where the external capture differs from the pre-cursor baseline.+ overlay_pixels: usize,+ /// Of those, pixels the Driver capture shows as the external one (overlay).+ left_in_driver_capture: usize,+ /// Of those, pixels the Driver capture shows as the baseline (desktop).+ desktop_in_driver_capture: usize,+}++fn overlay_capture_oracle(+ baseline: &image::RgbaImage,+ external: &image::RgbaImage,+ driver: &image::RgbaImage,+ (x0, y0, x1, y1): (u32, u32, u32, u32),+) -> OverlayCaptureOracle {+ assert_eq!(baseline.dimensions(), external.dimensions());+ assert_eq!(+ baseline.dimensions(),+ driver.dimensions(),+ "Driver desktop capture is not in the external capture's pixel frame"+ );+ let near = |a: &image::Rgba<u8>, b: &image::Rgba<u8>| {+ a.0.iter()+ .zip(b.0.iter())+ .take(3)+ .all(|(a, b)| a.abs_diff(*b) <= 24)+ };+ let mut oracle = OverlayCaptureOracle {+ overlay_pixels: 0,+ left_in_driver_capture: 0,+ desktop_in_driver_capture: 0,+ };+ for pixel_y in y0..y1 {+ for pixel_x in x0..x1 {+ let before = baseline.get_pixel(pixel_x, pixel_y);+ let outside = external.get_pixel(pixel_x, pixel_y);+ if near(before, outside) {+ continue;+ }+ oracle.overlay_pixels += 1;+ let inside = driver.get_pixel(pixel_x, pixel_y);+ if near(inside, outside) {+ oracle.left_in_driver_capture += 1;+ } else if near(inside, before) {+ oracle.desktop_in_driver_capture += 1;+ }+ }+ }+ oracle+}+fn window_center(window_id: u64) -> (f64, f64) {let mut rect = RECT::default();unsafe { GetWindowRect(HWND(window_id as *mut _), &mut rect) }diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/capture_exclusion.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/capture_exclusion.rsnew file mode 100644index 0000000000..66ed232b92--- /dev/null+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/capture_exclusion.rs@@ -0,0 +1,391 @@+//! Keep Driver-owned overlay pixels out of Driver-owned desktop captures.+//!+//! The agent cursor and its session pill are drawn in a Driver-owned overlay+//! window that sits above the desktop. A desktop capture taken for the agent+//! must not contain those pixels: after a click the cursor rests on the+//! target, so a capture would bake the arrow and the pill over the very label+//! the agent is trying to read.+//!+//! The exclusion is scoped to the Driver's own agent-facing capture. External+//! recorders and the Driver's recording feature keep seeing the cursor, so no+//! adapter may exclude the overlay from capture permanently.+//!+//! This module owns the cross-platform semantics:+//!+//! - one desktop capture excludes overlays at a time, so one capture can never+//! restore the overlay while another is still reading the screen;+//! - the overlay is restored exactly once after every exclusion, even when the+//! capture fails or panics;+//! - the result always says what happened ([`AgentOverlayCapture`]). A+//! platform that cannot exclude still returns the capture, but reports+//! `not_excluded` with a reason instead of pretending the pixels are clean.+//!+//! Adapters implement [`OverlayCaptureExcluder`] with the native mechanism+//! (window display affinity, an emptied X11 shape, a ScreenCaptureKit filter)+//! and nothing else.++use std::sync::Mutex;++pub use cua_driver_contract::{AgentOverlayCapture, AgentOverlayCaptureStatus};++/// Outcome of asking a platform to keep its overlays out of the next capture.+#[derive(Debug)]+pub enum ExclusionStart<H> {+ /// No Driver overlay pixels are on screen; the capture runs as is.+ NotPresent,+ /// The overlay is out of the captures that follow until+ /// [`OverlayCaptureExcluder::restore`] runs. `hidden` carries whatever the+ /// adapter needs to verify the capture or to restore the overlay.+ Excluded { method: &'static str, hidden: H },+ /// The overlay may be on screen and could not be kept out.+ Unsupported { reason: String },+}++/// What an adapter found when it inspected the captured pixels.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum ResidualCheck {+ /// Nothing of the overlay is left in the capture.+ Clean,+ /// Overlay pixels were still present and were replaced with the desktop+ /// the overlay last saw under them. `method` names that repair.+ Repaired { method: &'static str },+ /// Overlay pixels are still present in the capture.+ Residual { reason: String },+}++/// Native mechanism that hides or excludes a platform's overlay windows.+pub trait OverlayCaptureExcluder {+ /// Adapter state carried from [`Self::exclude`] to the capture and to+ /// [`Self::restore`].+ type Hidden;++ /// Keep every Driver overlay out of the captures that follow.+ fn exclude(&self) -> ExclusionStart<Self::Hidden>;++ /// Undo a successful [`Self::exclude`]. Called exactly once per+ /// `Excluded` start, after the capture, whether or not it succeeded.+ fn restore(&self, hidden: Self::Hidden);+}++/// Serializes overlay exclusion across concurrent desktop captures.+static EXCLUSION_GATE: Mutex<()> = Mutex::new(());++/// Run `capture` with every Driver overlay kept out of the screen pixels and+/// report what happened.+///+/// `capture` receives the adapter state when the overlay was excluded, and+/// returns the capture plus its own inspection of the pixels. An inspection is+/// only meaningful after an exclusion; for a capture that ran without one the+/// check is ignored.+pub fn capture_excluding_overlays<X, T, E>(+ excluder: &X,+ capture: impl FnOnce(Option<&X::Hidden>) -> Result<(T, ResidualCheck), E>,+) -> Result<(T, AgentOverlayCapture), E>+where+ X: OverlayCaptureExcluder,+{+ // A poisoned gate only means another capture panicked while holding it;+ // its restore already ran from the guard below, so the gate is still sound.+ let _gate = EXCLUSION_GATE+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());++ match excluder.exclude() {+ ExclusionStart::NotPresent => {+ let (value, _) = capture(None)?;+ Ok((value, AgentOverlayCapture::not_present()))+ }+ ExclusionStart::Unsupported { reason } => {+ let (value, _) = capture(None)?;+ Ok((value, AgentOverlayCapture::not_excluded(reason)))+ }+ ExclusionStart::Excluded { method, hidden } => {+ let restore = RestoreOnDrop {+ excluder,+ hidden: Some(hidden),+ };+ let result = capture(restore.hidden.as_ref());+ drop(restore);+ let (value, check) = result?;+ let report = match check {+ ResidualCheck::Clean => AgentOverlayCapture::excluded(method),+ ResidualCheck::Repaired { method: repair } => {+ AgentOverlayCapture::excluded(format!("{method}+{repair}"))+ }+ ResidualCheck::Residual { reason } => AgentOverlayCapture::not_excluded(reason),+ };+ Ok((value, report))+ }+ }+}++/// Restores the overlay when the capture returns or unwinds.+struct RestoreOnDrop<'a, X: OverlayCaptureExcluder> {+ excluder: &'a X,+ hidden: Option<X::Hidden>,+}++impl<X: OverlayCaptureExcluder> Drop for RestoreOnDrop<'_, X> {+ fn drop(&mut self) {+ if let Some(hidden) = self.hidden.take() {+ self.excluder.restore(hidden);+ }+ }+}++/// Append the one-line caveat an agent reading only the text content needs:+/// nothing for a clean capture, the reason when overlay pixels may be present.+pub fn append_summary_note(summary: &mut String, report: &AgentOverlayCapture) {+ if report.is_clean() {+ return;+ }+ summary.push_str("; the Driver's agent cursor overlay may appear in this image");+ if let Some(reason) = &report.reason {+ summary.push_str(" (");+ summary.push_str(reason);+ summary.push(')');+ }+}++/// Excluder for builds and platforms with no overlay window at all.+pub struct NoOverlay;++impl OverlayCaptureExcluder for NoOverlay {+ type Hidden = ();++ fn exclude(&self) -> ExclusionStart<()> {+ ExclusionStart::NotPresent+ }++ fn restore(&self, _hidden: ()) {}+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::sync::Arc;++ #[derive(Default)]+ struct Counters {+ excluded: AtomicUsize,+ restored: AtomicUsize,+ hidden_now: AtomicUsize,+ max_hidden: AtomicUsize,+ }++ enum Mode {+ Present,+ Absent,+ Unsupported,+ }++ struct Fake {+ mode: Mode,+ counters: Arc<Counters>,+ }++ impl Fake {+ fn new(mode: Mode) -> Self {+ Self {+ mode,+ counters: Arc::default(),+ }+ }+ }++ impl OverlayCaptureExcluder for Fake {+ type Hidden = u32;++ fn exclude(&self) -> ExclusionStart<u32> {+ match self.mode {+ Mode::Present => {+ self.counters.excluded.fetch_add(1, Ordering::SeqCst);+ let now = self.counters.hidden_now.fetch_add(1, Ordering::SeqCst) + 1;+ self.counters.max_hidden.fetch_max(now, Ordering::SeqCst);+ ExclusionStart::Excluded {+ method: "fake_hide",+ hidden: 7,+ }+ }+ Mode::Absent => ExclusionStart::NotPresent,+ Mode::Unsupported => ExclusionStart::Unsupported {+ reason: "fake platform cannot hide".into(),+ },+ }+ }++ fn restore(&self, hidden: u32) {+ assert_eq!(hidden, 7, "restore receives the state exclude produced");+ self.counters.hidden_now.fetch_sub(1, Ordering::SeqCst);+ self.counters.restored.fetch_add(1, Ordering::SeqCst);+ }+ }++ fn ok_clean(+ value: &'static str,+ ) -> impl FnOnce(Option<&u32>) -> Result<(&'static str, ResidualCheck), String> {+ move |_| Ok((value, ResidualCheck::Clean))+ }++ #[test]+ fn excluded_capture_restores_once_and_reports_the_method() {+ let fake = Fake::new(Mode::Present);+ let (value, report) = capture_excluding_overlays(&fake, |hidden| {+ assert_eq!(hidden, Some(&7));+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 0);+ Ok::<_, String>(("png", ResidualCheck::Clean))+ })+ .unwrap();+ assert_eq!(value, "png");+ assert_eq!(report, AgentOverlayCapture::excluded("fake_hide"));+ assert!(report.is_clean());+ assert_eq!(fake.counters.excluded.load(Ordering::SeqCst), 1);+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 1);+ }++ #[test]+ fn failed_capture_still_restores_the_overlay() {+ let fake = Fake::new(Mode::Present);+ let error = capture_excluding_overlays(&fake, |_| {+ Err::<(&str, ResidualCheck), _>("grab failed".to_string())+ })+ .unwrap_err();+ assert_eq!(error, "grab failed");+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 1);+ }++ #[test]+ fn panicking_capture_still_restores_the_overlay() {+ let fake = Fake::new(Mode::Present);+ let counters = Arc::clone(&fake.counters);+ let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {+ let _ = capture_excluding_overlays(&fake, |_| -> Result<((), ResidualCheck), ()> {+ panic!("capture backend panicked")+ });+ }));+ assert!(outcome.is_err());+ assert_eq!(counters.restored.load(Ordering::SeqCst), 1);+ // The gate is usable again after the panic.+ let (_, report) = capture_excluding_overlays(&fake, ok_clean("again")).unwrap();+ assert_eq!(report.status, AgentOverlayCaptureStatus::Excluded);+ assert_eq!(counters.restored.load(Ordering::SeqCst), 2);+ }++ #[test]+ fn absent_overlay_reports_not_present_without_restoring() {+ let fake = Fake::new(Mode::Absent);+ let (_, report) = capture_excluding_overlays(&fake, |hidden| {+ assert!(hidden.is_none());+ // A residual verdict without an exclusion carries no meaning.+ Ok::<_, String>((+ (),+ ResidualCheck::Residual {+ reason: "ignored".into(),+ },+ ))+ })+ .unwrap();+ assert_eq!(report, AgentOverlayCapture::not_present());+ assert!(report.is_clean());+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 0);+ }++ #[test]+ fn unsupported_platform_captures_and_publishes_the_limitation() {+ let fake = Fake::new(Mode::Unsupported);+ let (value, report) = capture_excluding_overlays(&fake, ok_clean("png")).unwrap();+ assert_eq!(value, "png");+ assert_eq!(+ report,+ AgentOverlayCapture::not_excluded("fake platform cannot hide")+ );+ assert!(!report.is_clean());+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 0);+ }++ #[test]+ fn residual_pixels_downgrade_the_report() {+ let fake = Fake::new(Mode::Present);+ let (_, report) = capture_excluding_overlays(&fake, |_| {+ Ok::<_, String>((+ (),+ ResidualCheck::Residual {+ reason: "window did not repaint".into(),+ },+ ))+ })+ .unwrap();+ assert_eq!(+ report,+ AgentOverlayCapture::not_excluded("window did not repaint")+ );+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 1);+ }++ #[test]+ fn repaired_pixels_name_both_mechanisms() {+ let fake = Fake::new(Mode::Present);+ let (_, report) = capture_excluding_overlays(&fake, |_| {+ Ok::<_, String>((+ (),+ ResidualCheck::Repaired {+ method: "save_under",+ },+ ))+ })+ .unwrap();+ assert_eq!(+ report,+ AgentOverlayCapture::excluded("fake_hide+save_under")+ );+ }++ #[test]+ fn concurrent_captures_never_overlap_their_exclusions() {+ let fake = Arc::new(Fake::new(Mode::Present));+ let threads: Vec<_> = (0..8)+ .map(|_| {+ let fake = Arc::clone(&fake);+ std::thread::spawn(move || {+ for _ in 0..25 {+ capture_excluding_overlays(fake.as_ref(), |_| {+ std::thread::yield_now();+ Ok::<_, String>(((), ResidualCheck::Clean))+ })+ .unwrap();+ }+ })+ })+ .collect();+ for thread in threads {+ thread.join().unwrap();+ }+ assert_eq!(fake.counters.excluded.load(Ordering::SeqCst), 200);+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 200);+ assert_eq!(fake.counters.max_hidden.load(Ordering::SeqCst), 1);+ }++ #[test]+ fn summary_note_is_added_only_when_overlay_pixels_may_remain() {+ let mut clean = String::from("desktop screenshot");+ append_summary_note(&mut clean, &AgentOverlayCapture::excluded("m"));+ append_summary_note(&mut clean, &AgentOverlayCapture::not_present());+ assert_eq!(clean, "desktop screenshot");++ let mut caveat = String::from("desktop screenshot");+ append_summary_note(&mut caveat, &AgentOverlayCapture::not_excluded("no API"));+ assert_eq!(+ caveat,+ "desktop screenshot; the Driver's agent cursor overlay may appear in this image (no API)"+ );+ }++ #[test]+ fn no_overlay_excluder_reports_not_present() {+ let (_, report) =+ capture_excluding_overlays(&NoOverlay, |_| Ok::<_, String>(((), ResidualCheck::Clean)))+ .unwrap();+ assert_eq!(report.status, AgentOverlayCaptureStatus::NotPresent);+ }+}diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rsindex dd90baaaa2..37d1655041 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs@@ -8,6 +8,7 @@pub mod badge_glyphs;pub mod bezier;+pub mod capture_exclusion;pub mod capture_utils;pub mod motion;pub mod path_planner;@@ -330,6 +331,10 @@ pub enum OverlayMsg {/// This deliberately does not recreate a cursor; the next command does so/// lazily after the successful `start_session` boundary.Revive(CursorKey),+ /// No state change: wakes a parked render loop so it services an+ /// out-of-band request (such as hiding for a Driver desktop capture)+ /// without waiting for its next maintenance tick.+ Wake,}/// Commands sent from MCP tool handlers to the overlay's render thread.diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rsindex bfe0243e1d..941edd9104 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs@@ -251,6 +251,7 @@ impl<S: RenderEntry, P> RenderMap<S, P> {OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd }) => self.apply_command(key, cmd),OverlayMsg::Remove(key) => self.remove(key),OverlayMsg::Revive(key) => self.revive(key),+ OverlayMsg::Wake => MsgOutcome::Ignored,}}@@ -454,6 +455,18 @@ mod tests {assert_eq!(map.cursors.len(), 2);}+ #[test]+ fn wake_changes_no_render_state() {+ let mut map = map();+ map.apply_msg(move_msg("sessA", 10.0, 10.0));+ let cursors = map.cursors.len();+ let last_active = map.last_active.clone();+ assert_eq!(map.apply_msg(OverlayMsg::Wake), MsgOutcome::Ignored);+ assert_eq!(map.cursors.len(), cursors);+ assert_eq!(map.last_active, last_active);+ assert!(map.ended.is_empty());+ }+#[test]fn revival_clears_the_tombstone_and_recreates_lazily() {let mut map = map();diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rsindex b29e82d2e8..c6ba57b5c7 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs@@ -14,6 +14,7 @@ use cua_driver_core::tool::ToolRegistry;pub mod health_report;pub mod overlay;+pub mod overlay_capture;pub mod pip;pub mod tools;diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rsindex 8d8ba00d29..f51e2a2dc0 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs@@ -120,6 +120,47 @@ fn should_start_x11_overlay(wayland_display_present: bool) -> bool {!wayland_display_present}+/// Set while the X11 owner thread is running its render loop over a mapped+/// overlay window; only then can a desktop capture ask it to hide.+static X11_OVERLAY_LIVE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);++/// Whether an X11 overlay window exists that a desktop capture must hide.+pub(crate) fn x11_overlay_live() -> bool {+ X11_OVERLAY_LIVE.load(std::sync::atomic::Ordering::Acquire)+}++/// Wake the X11 owner thread so it services a capture hold now instead of on+/// its next maintenance tick.+pub(crate) fn wake_x11_overlay() {+ let _ = try_send_x11_message(CMD_TX.get(), OverlayMsg::Wake);+}++/// The explicit limitation for Driver desktop captures on a Wayland session+/// whose agent cursor is drawn by the compositor (layer-shell surface or+/// GNOME Shell helper). `None` when no Wayland overlay is active.+pub(crate) fn wayland_capture_limitation() -> Option<String> {+ #[cfg(target_os = "linux")]+ {+ if !crate::wayland::is_wayland() {+ return None;+ }+ let backend = match WAYLAND_OVERLAY_BACKEND.get() {+ Some(WaylandOverlayBackend::LayerShell) => "layer-shell surface",+ Some(WaylandOverlayBackend::SemanticShellHelper)+ | Some(WaylandOverlayBackend::LegacyShellHelper) => "GNOME Shell helper",+ Some(WaylandOverlayBackend::None) | None => return None,+ };+ Some(format!(+ "on Wayland the agent cursor is drawn by the compositor ({backend}) into the same \+ output the desktop capture reads, and the Driver cannot leave it out"+ ))+ }+ #[cfg(not(target_os = "linux"))]+ {+ None+ }+}+#[derive(Clone, Copy, Debug, Eq, PartialEq)]enum WaylandOverlayBackend {SemanticShellHelper,@@ -337,7 +378,7 @@ fn dispatch_wayland_overlay_message(msg: &OverlayMsg) -> WaylandOverlayBackend {dispatch_shell_helper_command(&command.key, &command.cmd, semantic);}OverlayMsg::Remove(_) => crate::wayland::shell_helper::hide_cursor(),- OverlayMsg::Revive(_) => {}+ OverlayMsg::Revive(_) | OverlayMsg::Wake => {}}}WaylandOverlayBackend::LayerShell if !crate::wayland::overlay::forward(msg) => {@@ -1188,6 +1229,13 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMswin,root,};+ // Whether the last paint left a non-empty bounding shape on screen, so a+ // desktop capture knows there is something to hide and wait for.+ let mut overlay_shaped = false;+ // A Driver desktop capture is reading the root: paint nothing.+ let mut capture_held = false;+ let mut repaint_after_capture = false;+ let _live = X11OverlayLiveGuard::start();loop {// Idle fast path: no Pixmap allocation, RGBA→BGRA copy, XShape update,@@ -1350,17 +1398,55 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMsz_enforcer.reassert(pinned_wid);}+ // A Driver desktop capture wants the overlay out of the root image:+ // empty the shape, confirm the server applied it, and report what the+ // capture needs to find pixels a slow window has not repainted yet.+ if let Some(generation) = crate::overlay_capture::CAPTURE_HOLD.unanswered_request() {+ use crate::overlay_capture::{HiddenOverlay, HoldAnswer};+ let answer = if overlay_shaped {+ match blank_x11_overlay_shape(&conn, win, true) {+ Ok(()) => {+ overlay_shaped = false;+ HoldAnswer::Hidden(HiddenOverlay {+ was_shown: true,+ composited: compositor_present,+ residual: backdrop.residual_patches(Instant::now()),+ })+ }+ Err(e) => HoldAnswer::Failed(format!("{e:#}")),+ }+ } else {+ HoldAnswer::Hidden(HiddenOverlay {+ was_shown: false,+ composited: compositor_present,+ residual: Vec::new(),+ })+ };+ capture_held = matches!(answer, HoldAnswer::Hidden(_));+ crate::overlay_capture::CAPTURE_HOLD.answer(generation, answer);+ }+ if capture_held && !crate::overlay_capture::CAPTURE_HOLD.is_requested() {+ capture_held = false;+ repaint_after_capture = true;+ }+// Render after a command, during an active animation/fade, and once// more as the final active state settles. This leaves the X11 window in// its completed/cleared state before the next blocking receive.let mut paint_deferred = false;- if had_msg+ if capture_held {+ // Held for a capture: the frame is owed, so keep ticking and let+ // arrivals wait until it is actually on screen.+ paint_deferred = true;+ } else if had_msg|| screen_changed|| compositor_changed|| hover_changed|| frame_tick_needed|| next_frame_tick_needed+ || repaint_after_capture{+ repaint_after_capture = false;let tiles = {let guard = RENDER.lock().unwrap();guard.as_ref().map(render_x11_tiles)@@ -1379,7 +1465,12 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMsscreen_changed,&mut backdrop,) {- Ok(outcome) => paint_deferred = outcome == X11PaintOutcome::Deferred,+ Ok(outcome) => {+ paint_deferred = outcome == X11PaintOutcome::Deferred;+ if outcome == X11PaintOutcome::Painted {+ overlay_shaped = !tiles.is_empty();+ }+ }Err(e) => {// A broken X11 connection cannot recover inside this// owner thread. Exit instead of spinning at frame cadence@@ -1424,6 +1515,27 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMsconn.flush().ok();}+/// Publishes [`X11_OVERLAY_LIVE`] for the render loop's lifetime, however the+/// loop ends, and releases a capture waiting on a hold that will never be+/// answered.+#[cfg(target_os = "linux")]+struct X11OverlayLiveGuard;++#[cfg(target_os = "linux")]+impl X11OverlayLiveGuard {+ fn start() -> Self {+ X11_OVERLAY_LIVE.store(true, std::sync::atomic::Ordering::Release);+ Self+ }+}++#[cfg(target_os = "linux")]+impl Drop for X11OverlayLiveGuard {+ fn drop(&mut self) {+ X11_OVERLAY_LIVE.store(false, std::sync::atomic::Ordering::Release);+ }+}+// ── Z-order enforcer (Linux impl of cursor_overlay::ZOrderEnforcer) ──────/// X11 implementation of [`cursor_overlay::ZOrderEnforcer`].@@ -1876,6 +1988,24 @@ impl X11BackdropCache {}}+ /// Every retained painted rect, newest first, for a desktop capture to+ /// find pixels a window has not repainted since the overlay hid.+ fn residual_patches(&self, now: Instant) -> Vec<crate::overlay_capture::ResidualPatch> {+ self.frames+ .iter()+ .filter(|frame| !frame.expired(now))+ .flat_map(|frame| frame.saved.iter())+ .map(|saved| crate::overlay_capture::ResidualPatch {+ x: i32::from(saved.bounds.x),+ y: i32::from(saved.bounds.y),+ width: usize::from(saved.bounds.width),+ height: usize::from(saved.bounds.height),+ under: saved.under.clone(),+ uploaded: saved.uploaded.clone(),+ })+ .collect()+ }+fn record_frame(&mut self,at: Instant,@@ -2965,6 +3095,205 @@ mod tests {Ok(())}+ /// D-WL5: a desktop capture taken while the cursor and its session pill+ /// rest over a control must not contain them, while the screen (and any+ /// external capture) keeps showing them before and after.+ #[test]+ #[ignore = "requires a live X11 server (run under xvfb-run)"]+ fn x11_desktop_capture_hides_the_resting_overlay_and_restores_it() -> anyhow::Result<()> {+ use cursor_overlay::capture_exclusion::{+ capture_excluding_overlays, AgentOverlayCaptureStatus,+ };+ use x11rb::connection::Connection;+ use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK};+ use x11rb::protocol::xproto::{+ AtomEnum, ConnectionExt as XprotoConnectionExt, CreateWindowAux, ImageFormat, Window,+ WindowClass,+ };++ const BACKGROUND: u32 = 0x00c8_c8c8;+ const PROBE: (i16, i16, u16, u16) = (120, 120, 240, 120);++ fn find_named_window(+ conn: &impl Connection,+ root: Window,+ expected_name: &[u8],+ ) -> anyhow::Result<Window> {+ let deadline = Instant::now() + Duration::from_secs(3);+ while Instant::now() < deadline {+ for window in conn.query_tree(root)?.reply()?.children {+ let name = conn+ .get_property(+ false,+ window,+ AtomEnum::WM_NAME,+ AtomEnum::STRING,+ 0,+ u32::MAX,+ )?+ .reply()?;+ if name.value == expected_name {+ return Ok(window);+ }+ }+ std::thread::sleep(Duration::from_millis(10));+ }+ anyhow::bail!("timed out waiting for the overlay window")+ }++ fn wait_for_shape(+ conn: &impl Connection,+ overlay: Window,+ phase: &str,+ ) -> anyhow::Result<()> {+ let deadline = Instant::now() + Duration::from_secs(3);+ while Instant::now() < deadline {+ if !conn+ .shape_get_rectangles(overlay, SK::BOUNDING)?+ .reply()?+ .rectangles+ .is_empty()+ {+ return Ok(());+ }+ std::thread::sleep(Duration::from_millis(10));+ }+ anyhow::bail!("overlay bounding shape stayed empty during {phase}")+ }++ /// Pixels in the probe rect of a root-sized BGRX/RGBA buffer that are+ /// not the target window's background.+ fn foreign_pixels(pixel: impl Fn(usize, usize) -> [u8; 3]) -> usize {+ let (x0, y0, w, h) = PROBE;+ let mut count = 0;+ for y in y0 as usize..y0 as usize + h as usize {+ for x in x0 as usize..x0 as usize + w as usize {+ if pixel(x, y) != [0xc8, 0xc8, 0xc8] {+ count += 1;+ }+ }+ }+ count+ }++ fn live_foreign_pixels(conn: &impl Connection, root: Window) -> anyhow::Result<usize> {+ let (x0, y0, w, h) = PROBE;+ let image = conn+ .get_image(ImageFormat::Z_PIXMAP, root, x0, y0, w, h, !0u32)?+ .reply()?;+ let stride = usize::from(w) * 4;+ Ok(foreign_pixels(|x, y| {+ let offset = (y - y0 as usize) * stride + (x - x0 as usize) * 4;+ let bgrx = &image.data[offset..offset + 3];+ [bgrx[2], bgrx[1], bgrx[0]]+ }))+ }++ let (conn, screen_num) = x11rb::connect(None)?;+ let screen = &conn.setup().roots[screen_num];+ let root = screen.root;+ let target = conn.generate_id()?;+ conn.create_window(+ screen.root_depth,+ target,+ root,+ 0,+ 0,+ 480,+ 360,+ 0,+ WindowClass::INPUT_OUTPUT,+ screen.root_visual,+ &CreateWindowAux::new()+ .background_pixel(BACKGROUND)+ .override_redirect(1u32),+ )?+ .check()?;+ conn.map_window(target)?.check()?;+ conn.flush()?;+ anyhow::ensure!(+ live_foreign_pixels(&conn, root)? == 0,+ "the probe rect is not a solid target background before the cursor shows"+ );++ let cursor_id = "d-wl5-capture-exclusion-probe";+ init(CursorConfig {+ cursor_id: cursor_id.to_owned(),+ ..CursorConfig::default()+ });+ run_on_thread();+ let overlay = find_named_window(+ &conn,+ root,+ format!("Cua.AgentCursorOverlay.{cursor_id}").as_bytes(),+ )?;+ let deadline = Instant::now() + Duration::from_secs(3);+ while !crate::overlay::x11_overlay_live() {+ anyhow::ensure!(Instant::now() < deadline, "overlay loop never went live");+ std::thread::sleep(Duration::from_millis(10));+ }++ send_command(OverlayCommand::SetEnabled(true));+ send_command(OverlayCommand::SetSessionLabel(+ "ga-r5-window-772012bf".into(),+ ));+ send_command(OverlayCommand::SnapTo {+ x: 200.0,+ y: 160.0,+ heading_radians: None,+ });+ wait_for_shape(&conn, overlay, "cursor show")?;+ // Let the arrival and the pill settle into their resting frame.+ std::thread::sleep(Duration::from_millis(400));+ conn.configure_window(+ overlay,+ &x11rb::protocol::xproto::ConfigureWindowAux::new()+ .stack_mode(x11rb::protocol::xproto::StackMode::ABOVE),+ )?;+ conn.flush()?;+ let visible_before = live_foreign_pixels(&conn, root)?;+ anyhow::ensure!(+ visible_before >= 50,+ "the resting cursor and pill are not visible on screen ({visible_before} pixels)"+ );++ let (png, report) =+ capture_excluding_overlays(&crate::overlay_capture::OverlayExcluder, |hidden| {+ let png = crate::capture::screenshot_display_bytes()?;+ Ok::<_, anyhow::Error>(crate::overlay_capture::verify_hidden_capture(png, hidden))+ })?;+ anyhow::ensure!(+ report.status == AgentOverlayCaptureStatus::Excluded,+ "desktop capture did not exclude the overlay: {report:?}"+ );+ let image = image::load_from_memory(&png)?.to_rgba8();+ let in_capture = foreign_pixels(|x, y| {+ let [r, g, b, _] = image.get_pixel(x as u32, y as u32).0;+ [r, g, b]+ });+ eprintln!("visible_before={visible_before} in_capture={in_capture} report={report:?}");+ anyhow::ensure!(+ in_capture == 0,+ "desktop capture still contains {in_capture} overlay pixels over the target"+ );++ wait_for_shape(&conn, overlay, "restore after capture")?;+ let deadline = Instant::now() + Duration::from_secs(3);+ loop {+ let visible_after = live_foreign_pixels(&conn, root)?;+ if visible_after >= 50 {+ eprintln!("visible_after={visible_after}");+ break;+ }+ anyhow::ensure!(+ Instant::now() < deadline,+ "the cursor did not come back after the capture ({visible_after} pixels)"+ );+ std::thread::sleep(Duration::from_millis(20));+ }+ Ok(())+ }+#[test]#[ignore = "requires a live X11 server (run under xvfb-run)"]fn x11_overlay_input_shape_stays_empty_and_clicks_pass_through_after_resize(diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay_capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay_capture.rsnew file mode 100644index 0000000000..b01fa6cec3--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay_capture.rs@@ -0,0 +1,512 @@+//! Linux adapter of `cursor_overlay::capture_exclusion`.+//!+//! X11 has no per-window capture exclusion: a root `GetImage` returns whatever+//! the server is showing, our override-redirect overlay included. The adapter+//! therefore hides the overlay around the grab:+//!+//! 1. the capture thread raises a hold and wakes the overlay owner thread;+//! 2. the owner thread empties the overlay's bounding shape, round-trips the+//! server so the change is in effect, answers with what it knows about the+//! pixels it last painted, and paints nothing until the hold is released;+//! 3. the capture thread lets the uncovered windows repaint, grabs the root,+//! and releases the hold; the owner thread repaints the current frame.+//!+//! Under a compositing manager the uncovered pixels come back from the+//! windows' own buffers on the next composite. Without one the windows below+//! must answer Expose, and a slow client can leave our last frame in the+//! framebuffer. Those pixels are still exactly what the overlay uploaded, so+//! they are found by comparison and replaced with the desktop the overlay+//! recorded under them ([`scrub_residual_overlay`]) — the same save-under rule+//! the overlay's own compositing uses.+//!+//! Wayland overlays (layer-shell surface, GNOME Shell helper) are drawn by the+//! compositor into the same output the screencopy / portal capture reads, and+//! the Driver has no way to leave them out. That limitation is reported as+//! `not_excluded` instead of being papered over.++use std::sync::{Condvar, Mutex};+use std::time::Duration;++use cursor_overlay::capture_exclusion::{ExclusionStart, OverlayCaptureExcluder};++/// Method names reported in `agent_overlay_capture.method`.+pub const X11_METHOD: &str = "x11_unshaped_overlay";+pub const X11_COMPOSITED_METHOD: &str = "x11_unshaped_overlay_under_compositor";+pub const X11_SAVE_UNDER_REPAIR: &str = "x11_save_under";++/// How long the capture thread waits for the overlay owner to hide.+const HOLD_ACK_TIMEOUT: Duration = Duration::from_millis(500);+/// Time given to the compositor, or to the windows answering Expose, before+/// the root is read. Whatever is still ours afterwards is scrubbed.+const HIDE_SETTLE: Duration = Duration::from_millis(50);++/// One rect the overlay painted: the desktop it recorded under the rect and+/// the pixels it uploaded, both row-major BGRX (`width * height * 4`).+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct ResidualPatch {+ pub x: i32,+ pub y: i32,+ pub width: usize,+ pub height: usize,+ pub under: Vec<u8>,+ pub uploaded: Vec<u8>,+}++/// What the overlay owner reported after hiding.+#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct HiddenOverlay {+ /// Whether any overlay pixel was on screen before the hide.+ pub was_shown: bool,+ /// Whether a compositing manager blends the overlay (no residual check is+ /// possible or needed: the compositor repaints from window buffers).+ pub composited: bool,+ /// Painted rects that may still hold our pixels, newest first.+ pub residual: Vec<ResidualPatch>,+}++/// The owner thread's answer to a hold request.+#[derive(Clone, Debug, PartialEq, Eq)]+pub enum HoldAnswer {+ Hidden(HiddenOverlay),+ Failed(String),+}++#[derive(Default)]+struct HoldState {+ generation: u64,+ requested: bool,+ answer: Option<(u64, HoldAnswer)>,+}++/// Handshake between a capture thread and the overlay owner thread.+pub struct CaptureHold {+ state: Mutex<HoldState>,+ changed: Condvar,+}++impl CaptureHold {+ pub const fn new() -> Self {+ Self {+ state: Mutex::new(HoldState {+ generation: 0,+ requested: false,+ answer: None,+ }),+ changed: Condvar::new(),+ }+ }++ fn lock(&self) -> std::sync::MutexGuard<'_, HoldState> {+ self.state+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ }++ /// Capture side: ask the owner to hide. Returns the request generation.+ pub fn request(&self) -> u64 {+ let mut state = self.lock();+ state.generation = state.generation.wrapping_add(1);+ state.requested = true;+ state.answer = None;+ state.generation+ }++ /// Capture side: wait for the owner's answer to `generation`.+ pub fn wait_answer(&self, generation: u64, timeout: Duration) -> Option<HoldAnswer> {+ let state = self.lock();+ let (mut state, _) = self+ .changed+ .wait_timeout_while(+ state,+ timeout,+ |state| !matches!(&state.answer, Some((answered, _)) if *answered == generation),+ )+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ match state.answer.take() {+ Some((answered, answer)) if answered == generation => Some(answer),+ other => {+ state.answer = other;+ None+ }+ }+ }++ /// Capture side: let the overlay paint again.+ pub fn release(&self) {+ let mut state = self.lock();+ state.requested = false;+ state.answer = None;+ self.changed.notify_all();+ }++ /// Owner side: the request still waiting for an answer, if any.+ pub fn unanswered_request(&self) -> Option<u64> {+ let state = self.lock();+ (state.requested && state.answer.is_none()).then_some(state.generation)+ }++ /// Owner side: whether painting is still held.+ pub fn is_requested(&self) -> bool {+ self.lock().requested+ }++ /// Owner side: answer `generation`. A stale answer (the capture already+ /// gave up and released, or asked again) is dropped.+ pub fn answer(&self, generation: u64, answer: HoldAnswer) {+ let mut state = self.lock();+ if state.requested && state.generation == generation {+ state.answer = Some((generation, answer));+ self.changed.notify_all();+ }+ }+}++impl Default for CaptureHold {+ fn default() -> Self {+ Self::new()+ }+}++/// The process-wide hold shared by desktop captures and the X11 overlay owner.+pub static CAPTURE_HOLD: CaptureHold = CaptureHold::new();++/// Replace overlay pixels still present in `rgba` (row-major RGBA of the whole+/// root, `width * height * 4`) with the desktop recorded under them.+///+/// A patch only speaks for pixels it actually painted (`uploaded != under`).+/// For those, the newest patch decides: if the captured pixel still equals+/// what that patch uploaded, the window below never repainted and the pixel is+/// replaced with `under`; otherwise the window repainted and the capture is+/// the truth. Pixels a patch did not paint fall through to older patches.+///+/// Returns how many pixels were replaced.+pub fn scrub_residual_overlay(+ rgba: &mut [u8],+ width: usize,+ height: usize,+ patches: &[ResidualPatch],+) -> usize {+ if rgba.len() != width.saturating_mul(height).saturating_mul(4) {+ return 0;+ }+ let mut decided = vec![false; width * height];+ let mut replaced = 0;+ for patch in patches {+ let expected = patch.width.saturating_mul(patch.height).saturating_mul(4);+ if patch.under.len() != expected || patch.uploaded.len() != expected {+ continue;+ }+ for row in 0..patch.height {+ let Some(y) = offset(patch.y, row, height) else {+ continue;+ };+ for column in 0..patch.width {+ let Some(x) = offset(patch.x, column, width) else {+ continue;+ };+ let source = (row * patch.width + column) * 4;+ let uploaded = &patch.uploaded[source..source + 3];+ let under = &patch.under[source..source + 3];+ if uploaded == under {+ continue;+ }+ let index = y * width + x;+ if decided[index] {+ continue;+ }+ decided[index] = true;+ let target = index * 4;+ let captured = &rgba[target..target + 3];+ // BGRX patch vs RGBA capture.+ if captured[0] == uploaded[2]+ && captured[1] == uploaded[1]+ && captured[2] == uploaded[0]+ {+ rgba[target] = under[2];+ rgba[target + 1] = under[1];+ rgba[target + 2] = under[0];+ replaced += 1;+ }+ }+ }+ }+ replaced+}++fn offset(origin: i32, delta: usize, limit: usize) -> Option<usize> {+ let value = i64::from(origin) + i64::try_from(delta).ok()?;+ usize::try_from(value).ok().filter(|value| *value < limit)+}++/// Excluder used by the Linux `get_desktop_state`.+pub struct OverlayExcluder;++impl OverlayCaptureExcluder for OverlayExcluder {+ type Hidden = HiddenOverlay;++ fn exclude(&self) -> ExclusionStart<HiddenOverlay> {+ if let Some(reason) = crate::overlay::wayland_capture_limitation() {+ return ExclusionStart::Unsupported { reason };+ }+ if !crate::overlay::x11_overlay_live() {+ return ExclusionStart::NotPresent;+ }+ let generation = CAPTURE_HOLD.request();+ crate::overlay::wake_x11_overlay();+ match CAPTURE_HOLD.wait_answer(generation, HOLD_ACK_TIMEOUT) {+ Some(HoldAnswer::Hidden(hidden)) => {+ if hidden.was_shown {+ std::thread::sleep(HIDE_SETTLE);+ }+ ExclusionStart::Excluded {+ method: if hidden.composited {+ X11_COMPOSITED_METHOD+ } else {+ X11_METHOD+ },+ hidden,+ }+ }+ Some(HoldAnswer::Failed(error)) => {+ release_hold();+ ExclusionStart::Unsupported {+ reason: format!("the X11 agent cursor overlay could not be hidden: {error}"),+ }+ }+ None => {+ release_hold();+ ExclusionStart::Unsupported {+ reason: format!(+ "the X11 agent cursor overlay did not hide within {} ms",+ HOLD_ACK_TIMEOUT.as_millis()+ ),+ }+ }+ }+ }++ fn restore(&self, _hidden: HiddenOverlay) {+ release_hold();+ }+}++fn release_hold() {+ CAPTURE_HOLD.release();+ crate::overlay::wake_x11_overlay();+}++/// Inspect a PNG captured while the overlay was hidden, scrubbing any pixels+/// the overlay's last frame left behind. Returns the (possibly re-encoded) PNG+/// and the verdict.+#[cfg(target_os = "linux")]+pub fn verify_hidden_capture(+ png: Vec<u8>,+ hidden: Option<&HiddenOverlay>,+) -> (Vec<u8>, cursor_overlay::capture_exclusion::ResidualCheck) {+ use cursor_overlay::capture_exclusion::ResidualCheck;+ let Some(hidden) = hidden.filter(|hidden| !hidden.residual.is_empty()) else {+ return (png, ResidualCheck::Clean);+ };+ let decoded = match image::load_from_memory(&png) {+ Ok(decoded) => decoded.to_rgba8(),+ Err(error) => {+ return (+ png,+ ResidualCheck::Residual {+ reason: format!(+ "the desktop capture could not be checked for leftover overlay pixels: \+ {error}"+ ),+ },+ )+ }+ };+ let (width, height) = decoded.dimensions();+ let mut rgba = decoded.into_raw();+ let replaced =+ scrub_residual_overlay(&mut rgba, width as usize, height as usize, &hidden.residual);+ if replaced == 0 {+ return (png, ResidualCheck::Clean);+ }+ match cua_driver_core::image_utils::encode_rgba_to_png(&rgba, width, height) {+ Ok(repaired) => (+ repaired,+ ResidualCheck::Repaired {+ method: X11_SAVE_UNDER_REPAIR,+ },+ ),+ Err(error) => (+ png,+ ResidualCheck::Residual {+ reason: format!(+ "{replaced} leftover overlay pixels were found but the repaired capture \+ could not be encoded: {error}"+ ),+ },+ ),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn bgrx(pixels: &[[u8; 3]]) -> Vec<u8> {+ pixels+ .iter()+ .flat_map(|[r, g, b]| [*b, *g, *r, 0])+ .collect()+ }++ fn rgba(pixels: &[[u8; 3]]) -> Vec<u8> {+ pixels+ .iter()+ .flat_map(|[r, g, b]| [*r, *g, *b, 255])+ .collect()+ }++ const DESK: [u8; 3] = [200, 200, 200];+ const CURSOR: [u8; 3] = [120, 60, 220];+ const REPAINTED: [u8; 3] = [10, 120, 10];++ #[test]+ fn leftover_overlay_pixels_are_replaced_with_the_recorded_desktop() {+ // 3x1 screen; the patch painted the middle pixel.+ let mut capture = rgba(&[DESK, CURSOR, DESK]);+ let patch = ResidualPatch {+ x: 0,+ y: 0,+ width: 3,+ height: 1,+ under: bgrx(&[DESK, DESK, DESK]),+ uploaded: bgrx(&[DESK, CURSOR, DESK]),+ };+ assert_eq!(scrub_residual_overlay(&mut capture, 3, 1, &[patch]), 1);+ assert_eq!(capture, rgba(&[DESK, DESK, DESK]));+ }++ #[test]+ fn a_repainted_pixel_is_kept() {+ let mut capture = rgba(&[REPAINTED]);+ let patch = ResidualPatch {+ x: 0,+ y: 0,+ width: 1,+ height: 1,+ under: bgrx(&[DESK]),+ uploaded: bgrx(&[CURSOR]),+ };+ assert_eq!(scrub_residual_overlay(&mut capture, 1, 1, &[patch]), 0);+ assert_eq!(capture, rgba(&[REPAINTED]));+ }++ #[test]+ fn the_newest_painting_patch_decides_and_unpainted_pixels_fall_through() {+ // Newest patch painted pixel 0 only; pixel 1 is a backdrop copy there+ // (uploaded == under), so the older patch that painted pixel 1 speaks.+ let newest = ResidualPatch {+ x: 0,+ y: 0,+ width: 2,+ height: 1,+ under: bgrx(&[DESK, DESK]),+ uploaded: bgrx(&[CURSOR, DESK]),+ };+ let older = ResidualPatch {+ x: 0,+ y: 0,+ width: 2,+ height: 1,+ under: bgrx(&[DESK, DESK]),+ uploaded: bgrx(&[REPAINTED, CURSOR]),+ };+ // Pixel 0 shows the older frame's paint, which the newest frame+ // painted over: after the hide the window must have repainted it.+ let mut capture = rgba(&[REPAINTED, CURSOR]);+ assert_eq!(+ scrub_residual_overlay(&mut capture, 2, 1, &[newest, older]),+ 1+ );+ assert_eq!(capture, rgba(&[REPAINTED, DESK]));+ }++ #[test]+ fn patches_are_clipped_to_the_screen() {+ let mut capture = rgba(&[CURSOR, DESK]);+ let patch = ResidualPatch {+ x: -1,+ y: 0,+ width: 2,+ height: 2,+ under: bgrx(&[DESK, DESK, DESK, DESK]),+ uploaded: bgrx(&[CURSOR, CURSOR, CURSOR, CURSOR]),+ };+ assert_eq!(scrub_residual_overlay(&mut capture, 2, 1, &[patch]), 1);+ assert_eq!(capture, rgba(&[DESK, DESK]));+ }++ #[test]+ fn malformed_inputs_change_nothing() {+ let mut capture = rgba(&[CURSOR]);+ let short = ResidualPatch {+ x: 0,+ y: 0,+ width: 1,+ height: 1,+ under: vec![0; 3],+ uploaded: bgrx(&[CURSOR]),+ };+ assert_eq!(scrub_residual_overlay(&mut capture, 1, 1, &[short]), 0);+ assert_eq!(scrub_residual_overlay(&mut capture, 2, 2, &[]), 0);+ assert_eq!(capture, rgba(&[CURSOR]));+ }++ #[test]+ fn hold_answers_reach_only_the_current_request() {+ let hold = CaptureHold::new();+ assert_eq!(hold.unanswered_request(), None);+ let first = hold.request();+ assert_eq!(hold.unanswered_request(), Some(first));+ assert!(hold.is_requested());+ hold.answer(first, HoldAnswer::Hidden(HiddenOverlay::default()));+ assert_eq!(hold.unanswered_request(), None);+ assert_eq!(+ hold.wait_answer(first, Duration::ZERO),+ Some(HoldAnswer::Hidden(HiddenOverlay::default()))+ );+ hold.release();+ assert!(!hold.is_requested());++ // An answer that arrives after the capture gave up is dropped.+ let second = hold.request();+ hold.release();+ hold.answer(second, HoldAnswer::Failed("late".into()));+ assert_eq!(hold.wait_answer(second, Duration::ZERO), None);++ // An answer for an older request never satisfies a newer one.+ let third = hold.request();+ hold.answer(second, HoldAnswer::Failed("stale".into()));+ assert_eq!(hold.wait_answer(third, Duration::ZERO), None);+ assert_eq!(hold.unanswered_request(), Some(third));+ }++ #[test]+ fn hold_wait_wakes_when_the_owner_answers_from_another_thread() {+ let hold = std::sync::Arc::new(CaptureHold::new());+ let generation = hold.request();+ let owner = {+ let hold = std::sync::Arc::clone(&hold);+ std::thread::spawn(move || {+ while hold.unanswered_request().is_none() {+ std::thread::yield_now();+ }+ hold.answer(generation, HoldAnswer::Failed("owner".into()));+ })+ };+ assert_eq!(+ hold.wait_answer(generation, Duration::from_secs(5)),+ Some(HoldAnswer::Failed("owner".into()))+ );+ owner.join().unwrap();+ }+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex d3bdddda73..29e1939b77 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -12045,7 +12045,20 @@ impl Tool for GetDesktopStateTool {// Capture the full display at native size first. When the// compositor consumes logical input coordinates, normalize the// image below so screenshot pixels still land exactly.- let native_png = crate::capture::screenshot_display_bytes()?;+ //+ // The agent reads this image, so the Driver's own cursor and+ // session pill are hidden around the grab (or the limitation is+ // reported) instead of being baked over the controls it reads.+ let (native_png, overlay_capture) =+ cursor_overlay::capture_exclusion::capture_excluding_overlays(+ &crate::overlay_capture::OverlayExcluder,+ |hidden| {+ let png = crate::capture::screenshot_display_bytes()?;+ Ok::<_, anyhow::Error>(crate::overlay_capture::verify_hidden_capture(+ png, hidden,+ ))+ },+ )?;let (native_w, native_h) = crate::capture::png_dimensions_pub(&native_png)?;// True screen size. On a pure-Wayland session (native backend// opted in, no X11 DISPLAY) the capture above came from the@@ -12109,6 +12122,7 @@ impl Tool for GetDesktopStateTool {written,windows,capture_id,+ overlay_capture,))}).await;@@ -12124,6 +12138,7 @@ impl Tool for GetDesktopStateTool {written,windows,capture_id,+ overlay_capture,))) => {let frame_scale = if shot_w > 0 {f64::from(screen_w) / f64::from(shot_w)@@ -12143,6 +12158,7 @@ impl Tool for GetDesktopStateTool {"screenshot_mime_type": "image/png","windows": windows.iter().map(window_record_json).collect::<Vec<_>>(),"capture_id": capture_id,+ "agent_overlay_capture": overlay_capture,});if (frame_scale - 1.0).abs() > 0.001 {// Capped: the uncapped capture is the action frame.@@ -12153,7 +12169,7 @@ impl Tool for GetDesktopStateTool {content.push(cua_driver_core::protocol::Content::image_png(b64));}let window_lines = desktop_window_lines(&windows, frame_scale);- let frame_note = if (frame_scale - 1.0).abs() > 0.001 {+ let mut frame_note = if (frame_scale - 1.0).abs() > 0.001 {format!("; x/y for scope:\"desktop\" actions are pixels of THIS screenshot \(mapped ×{frame_scale:.2} back to the screen automatically)"@@ -12161,6 +12177,10 @@ impl Tool for GetDesktopStateTool {} else {String::new()};+ cursor_overlay::capture_exclusion::append_summary_note(+ &mut frame_note,+ &overlay_capture,+ );if let Some(path) = written {structured["screenshot_file_path"] = json!(path);content.push(cua_driver_core::protocol::Content::text(format!(diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rsindex c900542748..0c551da514 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs@@ -163,6 +163,7 @@ fn map_overlay_msg(msg: &OverlayMsg) -> Option<WlOverlayCmd> {cmd: kc.cmd.clone(),}),OverlayMsg::Revive(key) => Some(WlOverlayCmd::Revive(key.clone())),+ OverlayMsg::Wake => None,}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rsindex b0eea64447..32e1ca6ed8 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs@@ -22,8 +22,11 @@//!//! ## Display capture//!-//! `screencapture -x <file>` still captures the full main display (unchanged-//! in this slice).+//! `screencapture -x <file>` captures the full main display. The agent-facing+//! desktop capture instead uses a ScreenCaptureKit display filter that leaves+//! the Driver's cursor overlay window out+//! ([`screenshot_display_bytes_excluding_overlay`]); `screencapture` remains+//! its fallback and reports that it could not exclude the overlay.use base64::{engine::general_purpose::STANDARD as BASE64, Engine};use std::collections::HashMap;@@ -521,18 +524,28 @@ fn build_window_capture_plan(////// https://developer.apple.com/documentation/screencapturekit/scscreenshotmanagerfn capture_window_from_plan(window_id: u32, plan: &WindowCapturePlan) -> anyhow::Result<Vec<u8>> {- use screencapturekit::screenshot_manager::{CGImageExt, SCScreenshotManager};+ use screencapturekit::screenshot_manager::SCScreenshotManager;let image = SCScreenshotManager::capture_image(&plan.filter, &plan.config).map_err(|e| {anyhow::anyhow!("SCScreenshotManager::capture_image failed for window {window_id}: {e}")})?;+ cgimage_to_png(&image, &format!("window {window_id}"))+}++/// Encode a ScreenCaptureKit `CGImage` as PNG after checking its geometry.+fn cgimage_to_png(+ image: &screencapturekit::screenshot_manager::CGImage,+ label: &str,+) -> anyhow::Result<Vec<u8>> {+ use screencapturekit::screenshot_manager::CGImageExt;+let w = checked_image_dim(image.width(), "CGImage width")?;let h = checked_image_dim(image.height(), "CGImage height")?;let rgba = image.rgba_data()- .map_err(|e| anyhow::anyhow!("CGImage::rgba_data failed for window {window_id}: {e}"))?;+ .map_err(|e| anyhow::anyhow!("CGImage::rgba_data failed for {label}: {e}"))?;let expected_len = (w as u64).checked_mul(h as u64)@@ -540,7 +553,7 @@ fn capture_window_from_plan(window_id: u32, plan: &WindowCapturePlan) -> anyhow:.ok_or_else(|| anyhow::anyhow!("RGBA byte length overflow for {w}x{h}"))?;if rgba.len() as u64 != expected_len {anyhow::bail!(- "CGImage RGBA length {} != {w}*{h}*4 ({expected_len}) for window {window_id}",+ "CGImage RGBA length {} != {w}*{h}*4 ({expected_len}) for {label}",rgba.len());}@@ -774,6 +787,141 @@ pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {Ok(bytes)}+/// Name reported in `agent_overlay_capture.method` for the macOS mechanism.+pub const OVERLAY_EXCLUSION_METHOD: &str = "screencapturekit_excluding_windows";++/// Display captures run on their own single ScreenCaptureKit worker so a+/// window capture in flight never forces the agent's desktop capture onto the+/// fallback that cannot exclude the overlay.+const DISPLAY_CAPTURE_NATIVE_TIMEOUT: Duration = Duration::from_secs(5);++fn display_capture_gate() -> &'static NativeCaptureGate {+ static GATE: NativeCaptureGate = NativeCaptureGate::new();+ &GATE+}++/// macOS adapter of `cursor_overlay::capture_exclusion`: nothing is hidden on+/// screen; the capture itself leaves the overlay window out through a+/// ScreenCaptureKit display filter, so the cursor never flickers for the user+/// and external recorders keep seeing it.+struct OverlayExcluder;++impl cursor_overlay::capture_exclusion::OverlayCaptureExcluder for OverlayExcluder {+ type Hidden = u32;++ fn exclude(&self) -> cursor_overlay::capture_exclusion::ExclusionStart<u32> {+ use cursor_overlay::capture_exclusion::ExclusionStart;+ match crate::cursor::overlay::overlay_window_id() {+ Some(window_id) if crate::cursor::overlay::overlay_may_show_pixels() => {+ ExclusionStart::Excluded {+ method: OVERLAY_EXCLUSION_METHOD,+ hidden: window_id,+ }+ }+ // No overlay window, or nothing painted in it: the plain display+ // capture has no overlay pixels to leave out.+ _ => ExclusionStart::NotPresent,+ }+ }++ fn restore(&self, _window_id: u32) {}+}++/// Capture the main display for the agent with the Driver's cursor overlay+/// window left out, and report how that went.+///+/// `screencapture` cannot leave a window out, so it is only used when the+/// overlay shows nothing, or as a fallback that reports `not_excluded`.+pub fn screenshot_display_bytes_excluding_overlay() -> anyhow::Result<(+ Vec<u8>,+ cursor_overlay::capture_exclusion::AgentOverlayCapture,+)> {+ use cursor_overlay::capture_exclusion::{capture_excluding_overlays, ResidualCheck};+ capture_excluding_overlays(&OverlayExcluder, |overlay| {+ let Some(&overlay_window_id) = overlay else {+ return Ok((screenshot_display_bytes()?, ResidualCheck::Clean));+ };+ match run_native_capture_worker(+ display_capture_gate(),+ DISPLAY_CAPTURE_NATIVE_TIMEOUT,+ move || screenshot_main_display_sck_excluding(overlay_window_id),+ ) {+ Ok(bytes) => Ok((bytes, ResidualCheck::Clean)),+ Err(error) => {+ tracing::warn!(+ "ScreenCaptureKit display capture without the overlay failed: {error:#}"+ );+ let bytes = screenshot_display_bytes()?;+ Ok((+ bytes,+ ResidualCheck::Residual {+ reason: format!(+ "ScreenCaptureKit could not capture the display without the \+ overlay window ({error:#}); the screencapture fallback includes \+ every on-screen window"+ ),+ },+ ))+ }+ }+ })+}++/// Main display at native pixel size through ScreenCaptureKit, with one+/// window (the overlay) excluded. Requires macOS 14 for+/// `SCScreenshotManager`.+fn screenshot_main_display_sck_excluding(excluded_window_id: u32) -> anyhow::Result<Vec<u8>> {+ use core_graphics::display::CGMainDisplayID;+ use screencapturekit::prelude::{SCContentFilter, SCShareableContent, SCStreamConfiguration};+ use screencapturekit::screenshot_manager::SCScreenshotManager;++ let content = SCShareableContent::get()+ .map_err(|e| anyhow::anyhow!("SCShareableContent::get failed: {e}"))?;+ // SAFETY: CGMainDisplayID takes no arguments and only reads WindowServer state.+ let main_display_id = unsafe { CGMainDisplayID() };+ let display = content+ .displays()+ .into_iter()+ .find(|display| display.display_id() == main_display_id)+ .ok_or_else(|| {+ anyhow::anyhow!("ScreenCaptureKit does not list the main display {main_display_id}")+ })?;+ let windows = content.windows();+ let excluded = windows+ .iter()+ .find(|window| window.window_id() == excluded_window_id)+ .ok_or_else(|| {+ anyhow::anyhow!(+ "ScreenCaptureKit does not list the overlay window {excluded_window_id}"+ )+ })?;++ let mut filter = SCContentFilter::create()+ .with_display(&display)+ .with_excluding_windows(&[excluded])+ .build();+ // `screencapture` includes the menu bar; keep the same image.+ filter.set_include_menu_bar(true);++ let scale = f64::from(filter.point_pixel_scale());+ let rect = filter.content_rect();+ let (width_pts, height_pts) = if content_rect_usable(rect) {+ (rect.size.width, rect.size.height)+ } else {+ let frame = display.frame();+ (frame.size.width, frame.size.height)+ };+ let config = SCStreamConfiguration::new()+ .with_width(rounded_pixel_dim(width_pts * scale, "width")?)+ .with_height(rounded_pixel_dim(height_pts * scale, "height")?)+ // `screencapture -x` leaves the hardware pointer out too.+ .with_shows_cursor(false);++ let image = SCScreenshotManager::capture_image(&filter, &config)+ .map_err(|e| anyhow::anyhow!("SCScreenshotManager::capture_image failed: {e}"))?;+ cgimage_to_png(&image, "main display")+}+/// Capture the main display and return (base64-encoded PNG, width, height).pub fn screenshot_display() -> anyhow::Result<(String, u32, u32)> {let bytes = screenshot_display_bytes()?;diff --git a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rsindex 6a32feed0e..8a4c547156 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs@@ -31,7 +31,7 @@use std::collections::HashMap;use std::ffi::c_void;-use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};use std::sync::{Mutex, OnceLock};use std::time::{Duration, Instant};@@ -89,6 +89,53 @@ pub(crate) fn is_overlay_window(window_id: u32) -> bool {window_id != 0 && OVERLAY_WINDOW_ID.load(Ordering::Acquire) == window_id}+/// CGWindowID of the live overlay window, if one is on screen.+pub(crate) fn overlay_window_id() -> Option<u32> {+ match OVERLAY_WINDOW_ID.load(Ordering::Acquire) {+ 0 => None,+ window_id => Some(window_id),+ }+}++/// Whether the last composed frame could hold visible pixels, and when that+/// was last true. The layer update reaches WindowServer asynchronously, so a+/// frame that just cleared is still treated as visible for a short grace.+static OVERLAY_FRAME_PAINTS: AtomicBool = AtomicBool::new(false);+static OVERLAY_LAST_PAINT: Mutex<Option<Instant>> = Mutex::new(None);+const OVERLAY_CLEAR_GRACE: Duration = Duration::from_millis(250);++fn note_overlay_frame(paints: bool) {+ OVERLAY_FRAME_PAINTS.store(paints, Ordering::Release);+ if paints {+ if let Ok(mut last) = OVERLAY_LAST_PAINT.lock() {+ *last = Some(Instant::now());+ }+ }+}++/// Whether overlay pixels may be on screen right now: a cursor, its session+/// pill, a fade, or a focus rect. Desktop captures only need to exclude the+/// overlay window when this is true.+pub(crate) fn overlay_may_show_pixels() -> bool {+ if OVERLAY_FRAME_PAINTS.load(Ordering::Acquire) {+ return true;+ }+ OVERLAY_LAST_PAINT+ .lock()+ .ok()+ .and_then(|last| *last)+ .is_some_and(|last| last.elapsed() < OVERLAY_CLEAR_GRACE)+}++fn cursor_may_paint(state: &RenderState) -> bool {+ state.focus_rect.is_some()+ || (state.core.cfg.enabled+ && state.core.visible+ && state.core.idle_alpha > 0.0+ && state.core.pos.0 > -50.0+ && state.core.pos.1 > -50.0)+}+/// Screen-global geometry kept beside the shared keyed render map/// ([`cursor_overlay::RenderMap`], which owns the per-session lifecycle:/// lazy creation, stable z-order, tombstones, revival, and the default guard).@@ -799,6 +846,7 @@ fn render_loop(let mut pm = tiny_skia::Pixmap::new(w.max(1), h.max(1)).unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap());let backing_scale_f32 = scale as f32;+ note_overlay_frame(map.cursors.values().any(cursor_may_paint));for (_k, rs) in &map.cursors {let focus = rs.focus_rect.map(|rect| FocusRect {rect,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rsindex 99cbe2a3a5..1bc64766b6 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs@@ -88,11 +88,21 @@ impl Tool for GetDesktopStateTool {};// Capture the FULL display at native size — no resize. Run the- // blocking screencapture subprocess off the async runtime.+ // blocking display capture off the async runtime.let out_file = screenshot_out_file.clone();let res = tokio::task::spawn_blocking(- move || -> anyhow::Result<(Vec<u8>, Option<String>, u32, u32, (u32, u32))> {- let png = crate::capture::screenshot_display_bytes()?;+ move || -> anyhow::Result<(+ Vec<u8>,+ Option<String>,+ u32,+ u32,+ (u32, u32),+ cursor_overlay::capture_exclusion::AgentOverlayCapture,+ )> {+ // The agent reads this image, so the Driver's own cursor and+ // session pill must not sit over the controls it is reading.+ let (png, overlay_capture) =+ crate::capture::screenshot_display_bytes_excluding_overlay()?;let full = crate::capture::png_dimensions(&png)?;// Opt-in cap; later desktop-scope pixels from the capped// image are mapped back at dispatch and by its capture_id.@@ -105,19 +115,20 @@ impl Tool for GetDesktopStateTool {let (w, h) = crate::capture::png_dimensions(&png)?;if let Some(ref path) = out_file {std::fs::write(path, &png)?;- Ok((png, Some(path.clone()), w, h, full))+ Ok((png, Some(path.clone()), w, h, full, overlay_capture))} else {- Ok((png, None, w, h, full))+ Ok((png, None, w, h, full, overlay_capture))}},).await;- let (png, file_path, screenshot_width, screenshot_height, full_size) = match res {- Ok(Ok(v)) => v,- Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),- Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),- };+ let (png, file_path, screenshot_width, screenshot_height, full_size, overlay_capture) =+ match res {+ Ok(Ok(v)) => v,+ Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),+ Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),+ };let native_width = match u32::try_from(screen_width) {Ok(width) => width,@@ -142,10 +153,11 @@ impl Tool for GetDesktopStateTool {use base64::{engine::general_purpose::STANDARD as BASE64, Engine};content.push(Content::image_png(BASE64.encode(&png)));}- let summary = format!(+ let mut summary = format!("desktop screenshot {screenshot_width}x{screenshot_height} px \(screen {screen_width}x{screen_height} pts @ {scale_factor}x)");+ cursor_overlay::capture_exclusion::append_summary_note(&mut summary, &overlay_capture);content.push(Content::text(summary));let mut structured = serde_json::json!({@@ -158,6 +170,7 @@ impl Tool for GetDesktopStateTool {"scale_factor": scale_factor,"screenshot_mime_type": "image/png","capture_id": capture_id,+ "agent_overlay_capture": overlay_capture,});if full_size != (screenshot_width, screenshot_height) {structured["screenshot_original_width"] = serde_json::json!(full_size.0);diff --git a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs b/libs/cua-driver/rust/crates/platform-windows/src/capture.rsindex eb2e52c298..2941adb5ac 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs@@ -742,6 +742,18 @@ pub fn screenshot_display_bytes() -> Result<Vec<u8>> {}}+/// Capture the primary display for the agent, with the Driver's own cursor+/// overlay kept out of the pixels, and report how that went.+pub fn screenshot_display_bytes_excluding_overlay() -> Result<(+ Vec<u8>,+ cursor_overlay::capture_exclusion::AgentOverlayCapture,+)> {+ use cursor_overlay::capture_exclusion::{capture_excluding_overlays, ResidualCheck};+ capture_excluding_overlays(&crate::overlay::CaptureExcluder, |_| {+ Ok((screenshot_display_bytes()?, ResidualCheck::Clean))+ })+}+/// Capture primary display, returning (base64_png, width, height).pub fn screenshot_display() -> Result<(String, u32, u32)> {let png_bytes = screenshot_display_bytes()?;diff --git a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rsindex 800749fd22..b7dfcd7c8c 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs@@ -596,7 +596,7 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMsset_timer_resolution_raised(true);// Store hwnd and rx globally for the wnd_proc callback.- OVERLAY_HWND.store(hwnd.0 as isize, std::sync::atomic::Ordering::Relaxed);+ OVERLAY_HWND.store(hwnd.0 as isize, std::sync::atomic::Ordering::Release);*CMD_RX_WIN.lock().unwrap() = Some(rx);LAST_ZTICK.store(0, std::sync::atomic::Ordering::Relaxed);let _ = Z_ORDER.set(WinZOrderEnforcer {@@ -1397,6 +1397,93 @@ impl ZOrderEnforcer for WinZOrderEnforcer {}}+// ── Capture exclusion (Windows adapter of cursor_overlay::capture_exclusion) ─++/// Name reported in `agent_overlay_capture.method` for the Windows mechanism.+pub const CAPTURE_EXCLUSION_METHOD: &str = "win32_display_affinity_exclude_from_capture";++/// Keeps the overlay out of the Driver's own desktop BitBlt.+///+/// `WDA_EXCLUDEFROMCAPTURE` removes the window from every capture path DWM+/// serves — GDI `BitBlt` of the screen DC included — while it stays on the+/// monitor. It is applied only for the duration of one Driver desktop capture:+/// a permanent affinity would also hide the cursor from screen recorders and+/// from the Driver's own recordings, which must keep showing it.+///+/// The affinity is Windows 10 2004+ only. Older builds reject it; the capture+/// then runs anyway and reports `not_excluded` with the system error.+pub struct CaptureExcluder;++impl cursor_overlay::capture_exclusion::OverlayCaptureExcluder for CaptureExcluder {+ type Hidden = isize;++ fn exclude(&self) -> cursor_overlay::capture_exclusion::ExclusionStart<isize> {+ use cursor_overlay::capture_exclusion::ExclusionStart;+ let hwnd = OVERLAY_HWND.load(std::sync::atomic::Ordering::Acquire);+ if hwnd == 0 {+ return ExclusionStart::NotPresent;+ }+ match set_capture_excluded(hwnd, true) {+ Ok(()) => ExclusionStart::Excluded {+ method: CAPTURE_EXCLUSION_METHOD,+ hidden: hwnd,+ },+ Err(reason) => ExclusionStart::Unsupported { reason },+ }+ }++ fn restore(&self, hwnd: isize) {+ if let Err(error) = set_capture_excluded(hwnd, false) {+ tracing::warn!("Win32 overlay: could not restore capture visibility: {error}");+ }+ }+}++/// Apply or clear `WDA_EXCLUDEFROMCAPTURE` and wait for DWM to compose a frame+/// with the new affinity, so the very next screen read already honors it.+/// `SetWindowDisplayAffinity` accepts any window of the calling process, not+/// only windows of the calling thread.+#[cfg(target_os = "windows")]+fn set_capture_excluded(hwnd_isize: isize, excluded: bool) -> Result<(), String> {+ use windows::Win32::Foundation::HWND;+ use windows::Win32::Graphics::Dwm::DwmFlush;+ use windows::Win32::UI::WindowsAndMessaging::{+ SetWindowDisplayAffinity, WDA_EXCLUDEFROMCAPTURE, WDA_NONE,+ };+ let hwnd = HWND(hwnd_isize as *mut _);+ let affinity = if excluded {+ WDA_EXCLUDEFROMCAPTURE+ } else {+ WDA_NONE+ };+ unsafe { SetWindowDisplayAffinity(hwnd, affinity) }.map_err(|error| {+ format!(+ "SetWindowDisplayAffinity({}) failed on the agent cursor overlay: {error}; \+ excluding it from desktop captures requires Windows 10 version 2004 or later",+ if excluded {+ "WDA_EXCLUDEFROMCAPTURE"+ } else {+ "WDA_NONE"+ }+ )+ })?;+ if excluded {+ // One flush can land on a composition pass that started before the+ // affinity changed; the second is the first frame composed after it.+ for _ in 0..2 {+ if unsafe { DwmFlush() }.is_err() {+ break;+ }+ }+ }+ Ok(())+}++#[cfg(not(target_os = "windows"))]+fn set_capture_excluded(_hwnd_isize: isize, _excluded: bool) -> Result<(), String> {+ Err("window display affinity is only available on Windows".into())+}+// ── Headless unit tests for the Windows adapter ───────────────────────────//// The keyed lifecycle (ownership, removal, default guard, tombstone, revival,diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 7ca5f8a66e..483bd2cd98 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -7786,9 +7786,12 @@ impl Tool for GetDesktopStateTool {// blocking GDI capture off the async runtime.let out_file = screenshot_out_file.clone();let res = tokio::task::spawn_blocking(- move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32, Vec<u8>, (u32, u32))> {+ move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32, Vec<u8>, (u32, u32), cursor_overlay::capture_exclusion::AgentOverlayCapture)> {use base64::{engine::general_purpose::STANDARD as BASE64, Engine};- let png = crate::capture::screenshot_display_bytes()?;+ // The agent reads this image, so the Driver's own cursor and+ // session pill must not sit over the controls it is reading.+ let (png, overlay_capture) =+ crate::capture::screenshot_display_bytes_excluding_overlay()?;let full = crate::capture::png_dimensions_pub(&png)?;// Opt-in cap; later desktop-scope pixels from the capped// image are mapped back at dispatch and by its capture_id.@@ -7801,15 +7804,23 @@ impl Tool for GetDesktopStateTool {let (w, h) = crate::capture::png_dimensions_pub(&png)?;if let Some(ref path) = out_file {std::fs::write(path, &png)?;- Ok((None, Some(path.clone()), w, h, png, full))+ Ok((None, Some(path.clone()), w, h, png, full, overlay_capture))} else {- Ok((Some(BASE64.encode(&png)), None, w, h, png, full))+ Ok((Some(BASE64.encode(&png)), None, w, h, png, full, overlay_capture))}},).await;- let (b64_opt, file_path, screenshot_width, screenshot_height, png, full_size) = match res {+ let (+ b64_opt,+ file_path,+ screenshot_width,+ screenshot_height,+ png,+ full_size,+ overlay_capture,+ ) = match res {Ok(Ok(v)) => v,Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),@@ -7819,10 +7830,11 @@ impl Tool for GetDesktopStateTool {if let Some(b64) = b64_opt {content.push(Content::image_png(b64));}- let summary = format!(+ let mut summary = format!("desktop screenshot {screenshot_width}x{screenshot_height} px \(screen {screen_width}x{screen_height} px)");+ cursor_overlay::capture_exclusion::append_summary_note(&mut summary, &overlay_capture);content.push(Content::text(summary));let mut structured = json!({@@ -7837,6 +7849,7 @@ impl Tool for GetDesktopStateTool {if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 }},"screenshot_mime_type": "image/png",+ "agent_overlay_capture": overlay_capture,});if full_size != (screenshot_width, screenshot_height) {structured["screenshot_original_width"] = json!(full_size.0);