Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): keep the agent cursor out of desktop captures (#4199)

Francesco Bonaccifa5ded76dc40
diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.yml
index f7ac382a5f..b0fba747ea 100644
--- a/.github/workflows/ci-rust-linux.yml
+++ b/.github/workflows/ci-rust-linux.yml
@@ -161,6 +161,13 @@ jobs:
cargo test -p platform-linux \
x11_overlay_owner_stays_click_through_across_cursor_lifecycle_and_randr \
--locked -- --ignored --nocapture
+ - name: Run X11 desktop capture overlay-exclusion regression
+ working-directory: libs/cua-driver/rust
+ run: |
+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ cargo test -p platform-linux \
+ x11_desktop_capture_hides_the_resting_overlay_and_restores_it \
+ --locked -- --ignored --nocapture
- name: Run X11 clipboard round-trip regression
working-directory: libs/cua-driver/rust
run: |
diff --git a/docs/content/docs/reference/cua-driver/limits.mdx b/docs/content/docs/reference/cua-driver/limits.mdx
index 2542061964..2effdaaf99 100644
--- a/docs/content/docs/reference/cua-driver/limits.mdx
+++ b/docs/content/docs/reference/cua-driver/limits.mdx
@@ -130,6 +130,23 @@ does not imply arbitrary raw background PX delivery on Wayland.
---
+## Desktop captures leave out the agent cursor
+
+**Behavior:** `get_desktop_state` keeps the Driver's own agent cursor and session pill out of the PNG it returns, so a cursor resting on the control it just clicked does not cover that control's label. The on-screen cursor, external screen recorders, and the Driver's own recordings keep showing it. Every response reports the outcome in `structuredContent.agent_overlay_capture`:
+
+- `excluded`, with the native `method`;
+- `not_present`, when no overlay pixels were on screen;
+- `not_excluded`, with a `reason`, when overlay pixels may be in the image. The text summary repeats the caveat.
+
+**Platform scope:**
+
+- **macOS:** the capture uses a ScreenCaptureKit display filter that leaves out the overlay window (macOS 14 or later). The capture falls back to `screencapture`, which reports `not_excluded`.
+- **Windows:** the overlay is marked `WDA_EXCLUDEFROMCAPTURE` for the length of the capture (Windows 10 version 2004 or later). During that window, other capture tools also stop seeing the cursor. Older builds report `not_excluded`.
+- **Linux X11:** the overlay hides for the length of the root grab, so the cursor blinks off briefly. Without a compositing manager, pixels that a slow window has not repainted are replaced with the desktop that the overlay recorded under them (`method` ends in `+x11_save_under`).
+- **Linux Wayland:** the compositor draws the cursor into the output that the capture reads, and the Driver cannot leave it out. Captures report `not_excluded`.
+
+---
+
## Off-Space SwiftUI windows strip their AX tree
**Symptom:** `get_window_state({pid, window_id})` on a window on a different Space (e.g. System Settings parked on Space 2 while on Space 1) returns a minimal tree that contains only the menu bar, or just the `AXApplication` root.
diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.json
index aa3e59fa27..b786ceef35 100644
--- a/libs/cua-driver/contract/manifest.json
+++ b/libs/cua-driver/contract/manifest.json
@@ -1246,6 +1246,46 @@
"success_output_schema": {
"additionalProperties": true,
"properties": {
+ "agent_overlay_capture": {
+ "additionalProperties": false,
+ "properties": {
+ "method": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "reason": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "status": {
+ "oneOf": [
+ {
+ "const": "excluded",
+ "type": "string"
+ },
+ {
+ "const": "not_present",
+ "type": "string"
+ },
+ {
+ "const": "not_excluded",
+ "type": "string"
+ }
+ ]
+ }
+ },
+ "required": [
+ "status"
+ ],
+ "type": [
+ "object",
+ "null"
+ ]
+ },
"display": {
"type": "string"
},
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
index d0ef464eab..0cdde7377d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
@@ -44,13 +44,13 @@ pub use outputs::{
refusal_envelope_schema, ActionDelivery, ActionDeliveryMode, ActionEffect, ActionError,
ActionEscalation, ActionEscalationReason, ActionEscalationTarget, ActionEvidence,
ActionEvidenceKind, ActionResult, ActionResultValidationError, ActionRoute,
- ClipboardReadOutput, ClipboardWriteOutput, CursorMotionOutput, CursorPointOutput,
- CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, DesktopStateOutput,
- EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, ListSessionsOutput,
- ScreenSizeOutput, SessionClientKindOutput, SessionLifecycleState, SessionOutput,
- SessionStateOutput, SessionTransportOutput, SetAgentCursorEnabledOutput,
- SetAgentCursorMotionOutput, SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput,
- TOOL_INVOCATION_FAILED_CODE,
+ AgentOverlayCapture, AgentOverlayCaptureStatus, ClipboardReadOutput, ClipboardWriteOutput,
+ CursorMotionOutput, CursorPointOutput, CursorPositionOutput, CursorThemeOutput,
+ CursorVisualOutput, DesktopStateOutput, EffectiveScope, EndSessionOutput,
+ GetAgentCursorStateOutput, ListSessionsOutput, ScreenSizeOutput, SessionClientKindOutput,
+ SessionLifecycleState, SessionOutput, SessionStateOutput, SessionTransportOutput,
+ SetAgentCursorEnabledOutput, SetAgentCursorMotionOutput, SetAgentCursorThemeOutput,
+ StartSessionOutput, ToolOutput, TOOL_INVOCATION_FAILED_CODE,
};
pub use verification::{
BoundsExpectation, ElementPredicate, ElementSelector, PredicateOutcome, StatePredicate,
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
index 5ce0bdbfcf..7d05f009cd 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
@@ -365,16 +365,92 @@ pub struct DesktopStateOutput {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub screenshot_file_path: Option<String>,
+ /// Whether the Driver's own overlay pixels (agent cursor, session pill)
+ /// were kept out of this capture. Absent from producers that predate the
+ /// report.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub agent_overlay_capture: Option<AgentOverlayCapture>,
#[serde(flatten)]
pub extensions: BTreeMap<String, Value>,
}
+/// How a Driver-owned desktop capture treated Driver-owned overlay pixels.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum AgentOverlayCaptureStatus {
+ /// Overlay windows were on screen and were kept out of the pixels.
+ Excluded,
+ /// No Driver overlay pixels were on screen, so there was nothing to keep out.
+ NotPresent,
+ /// Overlay pixels may be in the capture; `reason` says why.
+ NotExcluded,
+}
+
+/// Report attached to desktop captures about Driver-owned overlay pixels.
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct AgentOverlayCapture {
+ pub status: AgentOverlayCaptureStatus,
+ /// Native mechanism that kept the overlay out, when `status` is `excluded`.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub method: Option<String>,
+ /// Why the overlay could not be kept out, when `status` is `not_excluded`.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub reason: Option<String>,
+}
+
+impl AgentOverlayCapture {
+ pub fn excluded(method: impl Into<String>) -> Self {
+ Self {
+ status: AgentOverlayCaptureStatus::Excluded,
+ method: Some(method.into()),
+ reason: None,
+ }
+ }
+
+ pub fn not_present() -> Self {
+ Self {
+ status: AgentOverlayCaptureStatus::NotPresent,
+ method: None,
+ reason: None,
+ }
+ }
+
+ pub fn not_excluded(reason: impl Into<String>) -> Self {
+ Self {
+ status: AgentOverlayCaptureStatus::NotExcluded,
+ method: None,
+ reason: Some(reason.into()),
+ }
+ }
+
+ /// Whether the capture is free of Driver overlay pixels.
+ pub fn is_clean(&self) -> bool {
+ self.status != AgentOverlayCaptureStatus::NotExcluded
+ }
+
+ fn validate(&self) -> Result<(), String> {
+ match (self.status, &self.method, &self.reason) {
+ (AgentOverlayCaptureStatus::Excluded, Some(_), None)
+ | (AgentOverlayCaptureStatus::NotPresent, None, None)
+ | (AgentOverlayCaptureStatus::NotExcluded, None, Some(_)) => Ok(()),
+ _ => Err(
+ "agent_overlay_capture: excluded requires only method, not_excluded requires \
+ only reason, not_present carries neither"
+ .into(),
+ ),
+ }
+ }
+}
+
impl ToolOutput for DesktopStateOutput {
fn validate(&self) -> Result<(), String> {
- if self.screenshot_mime_type == "image/png" {
- Ok(())
- } else {
- Err("screenshot_mime_type must be image/png".into())
+ if self.screenshot_mime_type != "image/png" {
+ return Err("screenshot_mime_type must be image/png".into());
+ }
+ match &self.agent_overlay_capture {
+ Some(report) => report.validate(),
+ None => Ok(()),
}
}
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs
index 95abd82341..7c4afdc718 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_showcase_test.rs
@@ -118,6 +118,18 @@ fn semantic_cursor_showcase_records_session_and_action_states() {
std::fs::write(&screenshot_path, cursor_png).expect("write cursor oracle screenshot");
evidence.screenshot = Some(screenshot_path.display().to_string());
+ // The cursor and pill keep resting there, yet the capture the Driver
+ // hands the agent must not contain them (D-WL5).
+ assert_driver_capture_excludes_overlay(
+ &mut driver,
+ &baseline,
+ &cursor_frame,
+ center_x - 180.0,
+ center_y - 80.0,
+ width,
+ height,
+ );
+
call_ok(
&mut driver,
"click",
@@ -349,7 +361,121 @@ fn changed_pixels_in_rect(
.count()
}
+/// Check the Driver's own desktop capture against the external cursor frame:
+/// every pixel the overlay changed on screen must show the desktop there.
+///
+/// Native Wayland compositors draw the cursor into the captured output; there
+/// the Driver must say so instead of claiming a clean capture.
+fn assert_driver_capture_excludes_overlay(
+ driver: &mut McpDriver,
+ baseline: &RgbaImage,
+ cursor_frame: &RgbaImage,
+ logical_x: f64,
+ logical_y: f64,
+ logical_width: f64,
+ logical_height: f64,
+) {
+ let (driver_png, _, _, report) = capture_desktop_png_with_report(driver);
+ let status = report["status"].as_str().unwrap_or("missing");
+ if wayland_session() {
+ assert_eq!(
+ status, "not_excluded",
+ "a Wayland desktop capture must report that it could not exclude the overlay: \
+ {report}"
+ );
+ return;
+ }
+ assert_eq!(
+ status, "excluded",
+ "desktop capture did not exclude the agent cursor overlay: {report}"
+ );
+ let driver_frame = image::load_from_memory(&driver_png)
+ .expect("decode driver-owned desktop screenshot")
+ .to_rgba8();
+ assert_eq!(
+ driver_frame.dimensions(),
+ cursor_frame.dimensions(),
+ "driver capture and cursor oracle are not in the same pixel frame"
+ );
+ let regions = cursor_oracle_regions(
+ baseline.width(),
+ baseline.height(),
+ logical_x,
+ logical_y,
+ logical_width,
+ logical_height,
+ );
+ let mut overlay_pixels = 0usize;
+ let mut desktop_pixels = 0usize;
+ for rect in [regions.pointer, regions.badge_left, regions.badge_right] {
+ let (overlay, desktop) =
+ overlay_pixels_resolved_to_desktop(baseline, cursor_frame, &driver_frame, rect);
+ overlay_pixels += overlay;
+ desktop_pixels += desktop;
+ }
+ assert!(
+ overlay_pixels >= 36,
+ "the cursor oracle no longer shows the overlay ({overlay_pixels} pixels)"
+ );
+ assert!(
+ desktop_pixels * 10 >= overlay_pixels * 9,
+ "driver desktop capture still shows the agent cursor overlay: only {desktop_pixels} \
+ of {overlay_pixels} overlay pixels show the desktop; report={report}"
+ );
+}
+
+/// Pixels in `rect` the overlay changed (baseline vs cursor frame), and how
+/// many of those the driver capture shows closer to the desktop than to the
+/// overlay. "Closer" keeps the check independent of small color differences
+/// between capture pipelines.
+fn overlay_pixels_resolved_to_desktop(
+ baseline: &RgbaImage,
+ cursor_frame: &RgbaImage,
+ driver_frame: &RgbaImage,
+ rect: PixelRect,
+) -> (usize, usize) {
+ let distance = |a: &image::Rgba<u8>, b: &image::Rgba<u8>| -> u16 {
+ a.0.iter()
+ .zip(b.0.iter())
+ .take(3)
+ .map(|(a, b)| u16::from(a.abs_diff(*b)))
+ .sum()
+ };
+ let x0 = rect.x0.clamp(0, i64::from(baseline.width())) as u32;
+ let x1 = rect.x1.clamp(0, i64::from(baseline.width())) as u32;
+ let y0 = rect.y0.clamp(0, i64::from(baseline.height())) as u32;
+ let y1 = rect.y1.clamp(0, i64::from(baseline.height())) as u32;
+ let mut overlay = 0;
+ let mut desktop = 0;
+ for pixel_y in y0..y1 {
+ for pixel_x in x0..x1 {
+ let before = baseline.get_pixel(pixel_x, pixel_y);
+ let on_screen = cursor_frame.get_pixel(pixel_x, pixel_y);
+ if distance(before, on_screen) < 80 {
+ continue;
+ }
+ overlay += 1;
+ let captured = driver_frame.get_pixel(pixel_x, pixel_y);
+ if distance(captured, before) < distance(captured, on_screen) {
+ desktop += 1;
+ }
+ }
+ }
+ (overlay, desktop)
+}
+
+fn wayland_session() -> bool {
+ cfg!(target_os = "linux") && std::env::var_os("WAYLAND_DISPLAY").is_some()
+}
+
fn capture_desktop_png(driver: &mut McpDriver) -> (Vec<u8>, f64, f64) {
+ let (png, width, height, _) = capture_desktop_png_with_report(driver);
+ (png, width, height)
+}
+
+fn capture_desktop_png_with_report(
+ driver: &mut McpDriver,
+) -> (Vec<u8>, f64, f64, serde_json::Value) {
let response = driver.call("get_desktop_state", serde_json::json!({}));
assert!(
!response.is_error(),
@@ -378,7 +504,8 @@ fn capture_desktop_png(driver: &mut McpDriver) -> (Vec<u8>, f64, f64) {
.as_f64()
.or_else(|| response.structured()["screenshot_height"].as_f64())
.expect("desktop capture returned no logical height");
- (png, width, height)
+ let report = response.structured()["agent_overlay_capture"].clone();
+ (png, width, height, report)
}
fn capture_cursor_oracle_png(driver: &mut McpDriver, width: f64, height: f64) -> Vec<u8> {
@@ -426,10 +553,68 @@ fn capture_cursor_oracle_png(driver: &mut McpDriver, width: f64, height: f64) ->
output.stdout
}
- #[cfg(not(target_os = "linux"))]
+ // The Driver's own desktop capture leaves the overlay out, so the oracle
+ // for what a user sees comes from another capture path: a BitBlt from this
+ // (different) process, which the overlay's temporary capture exclusion
+ // does not apply to.
+ #[cfg(target_os = "windows")]
+ {
+ let _ = (driver, width, height);
+ platform_windows::capture::screenshot_display_bytes()
+ .expect("external Windows display capture failed")
+ }
+
+ // Only the Driver holds Screen Recording permission on the macOS lanes.
+ // Its per-turn recording capture deliberately keeps the overlay, so the
+ // move's post-action screenshot is what a user saw.
+ #[cfg(target_os = "macos")]
{
let _ = (width, height);
- capture_desktop_png(driver).0
+ recorded_turn_screenshot(driver, "move_cursor")
+ }
+}
+
+/// The newest recorded turn's post-action screenshot for `tool`.
+#[cfg(target_os = "macos")]
+fn recorded_turn_screenshot(driver: &McpDriver, tool: &str) -> Vec<u8> {
+ let recording_dir = driver
+ .recording_dir()
+ .expect("showcase recording directory")
+ .to_path_buf();
+ let deadline = std::time::Instant::now() + Duration::from_secs(5);
+ loop {
+ let mut turns: Vec<_> = std::fs::read_dir(&recording_dir)
+ .into_iter()
+ .flatten()
+ .flatten()
+ .map(|entry| entry.path())
+ .filter(|path| {
+ path.file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| name.starts_with("turn-"))
+ })
+ .collect();
+ turns.sort();
+ for turn in turns.iter().rev() {
+ let Ok(action) = std::fs::read(turn.join("action.json")) else {
+ continue;
+ };
+ let Ok(action) = serde_json::from_slice::<serde_json::Value>(&action) else {
+ continue;
+ };
+ if action["tool"].as_str() != Some(tool) {
+ continue;
+ }
+ if let Ok(png) = std::fs::read(turn.join("after.png")) {
+ return png;
+ }
+ }
+ assert!(
+ std::time::Instant::now() < deadline,
+ "no recorded {tool} turn with an after.png under {}",
+ recording_dir.display()
+ );
+ settle(100);
}
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rs
index 3ee683820a..b727b6df52 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/agent_cursor_windows_test.rs
@@ -53,6 +53,14 @@ fn agent_cursor_overlay_obeys_untargeted_and_targeted_z_order() {
let foreground_before = unsafe { GetForegroundWindow() };
assert_eq!(foreground_before.0 as u64, target.native_id);
let real_cursor_before = real_cursor_position();
+ // The desktop under the cursor's resting point before any overlay
+ // pixel exists, for the capture-exclusion oracle below.
+ let baseline = image::load_from_memory(
+ &platform_windows::capture::screenshot_display_bytes()
+ .expect("baseline screenshot_display_bytes failed"),
+ )
+ .expect("decode baseline display screenshot")
+ .to_rgba8();
for (tool, arguments) in [
(
@@ -132,6 +140,58 @@ fn agent_cursor_overlay_obeys_untargeted_and_targeted_z_order() {
"agent cursor not visible at ({x:.0},{y:.0}): only {visible_pixels} qualifying pixels"
);
+ // D-WL5: the cursor keeps resting over the target, yet the Driver's
+ // own desktop capture must not contain it, while the external capture
+ // above (another process) still does.
+ let capture_session = "windows-agent-cursor-desktop-capture";
+ let started = driver.call(
+ "start_session",
+ serde_json::json!({"session": capture_session, "capture_scope": "desktop"}),
+ );
+ assert!(
+ !started.is_error(),
+ "desktop capture session failed: {}",
+ started.text()
+ );
+ let capture_dir = tempfile::tempdir().expect("create desktop capture directory");
+ let capture_path = capture_dir.path().join("desktop.png");
+ let desktop = driver.call(
+ "get_desktop_state",
+ serde_json::json!({
+ "session": capture_session,
+ "screenshot_out_file": capture_path.to_string_lossy(),
+ }),
+ );
+ assert!(
+ !desktop.is_error(),
+ "get_desktop_state failed: {}",
+ desktop.text()
+ );
+ assert_eq!(
+ desktop.structured()["agent_overlay_capture"]["status"].as_str(),
+ Some("excluded"),
+ "desktop capture did not exclude the agent cursor overlay: {}",
+ desktop.text()
+ );
+ let driver_image = image::open(&capture_path)
+ .expect("decode Driver desktop capture")
+ .to_rgba8();
+ let overlay_box = (x0, y0, x1, y1);
+ let oracle = overlay_capture_oracle(&baseline, &image, &driver_image, overlay_box);
+ eprintln!("agent cursor capture-exclusion oracle: {oracle:?}");
+ assert!(
+ oracle.overlay_pixels >= 20,
+ "external capture barely differs from the baseline under the cursor: {oracle:?}"
+ );
+ assert!(
+ oracle.left_in_driver_capture * 10 <= oracle.overlay_pixels,
+ "Driver desktop capture still contains the agent cursor overlay: {oracle:?}"
+ );
+ assert!(
+ oracle.desktop_in_driver_capture * 10 >= oracle.overlay_pixels * 8,
+ "Driver desktop capture does not show the desktop under the cursor: {oracle:?}"
+ );
+
// Capture the target while it is still foreground. Electron may prune
// parts of its UIA tree once another full-size window covers it, but
// the snapshot remains a stable handle for the later background click.
@@ -423,6 +483,60 @@ fn wait_for_overlay() -> HWND {
overlay
}
+/// Pixel accounting for one box: which pixels the overlay changed in an
+/// external capture, and what the Driver's own capture shows there.
+#[derive(Debug)]
+struct OverlayCaptureOracle {
+ /// Pixels where the external capture differs from the pre-cursor baseline.
+ overlay_pixels: usize,
+ /// Of those, pixels the Driver capture shows as the external one (overlay).
+ left_in_driver_capture: usize,
+ /// Of those, pixels the Driver capture shows as the baseline (desktop).
+ desktop_in_driver_capture: usize,
+}
+
+fn overlay_capture_oracle(
+ baseline: &image::RgbaImage,
+ external: &image::RgbaImage,
+ driver: &image::RgbaImage,
+ (x0, y0, x1, y1): (u32, u32, u32, u32),
+) -> OverlayCaptureOracle {
+ assert_eq!(baseline.dimensions(), external.dimensions());
+ assert_eq!(
+ baseline.dimensions(),
+ driver.dimensions(),
+ "Driver desktop capture is not in the external capture's pixel frame"
+ );
+ let near = |a: &image::Rgba<u8>, b: &image::Rgba<u8>| {
+ a.0.iter()
+ .zip(b.0.iter())
+ .take(3)
+ .all(|(a, b)| a.abs_diff(*b) <= 24)
+ };
+ let mut oracle = OverlayCaptureOracle {
+ overlay_pixels: 0,
+ left_in_driver_capture: 0,
+ desktop_in_driver_capture: 0,
+ };
+ for pixel_y in y0..y1 {
+ for pixel_x in x0..x1 {
+ let before = baseline.get_pixel(pixel_x, pixel_y);
+ let outside = external.get_pixel(pixel_x, pixel_y);
+ if near(before, outside) {
+ continue;
+ }
+ oracle.overlay_pixels += 1;
+ let inside = driver.get_pixel(pixel_x, pixel_y);
+ if near(inside, outside) {
+ oracle.left_in_driver_capture += 1;
+ } else if near(inside, before) {
+ oracle.desktop_in_driver_capture += 1;
+ }
+ }
+ }
+ oracle
+}
+
fn window_center(window_id: u64) -> (f64, f64) {
let mut rect = RECT::default();
unsafe { GetWindowRect(HWND(window_id as *mut _), &mut rect) }
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/capture_exclusion.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/capture_exclusion.rs
new file mode 100644
index 0000000000..66ed232b92
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/capture_exclusion.rs
@@ -0,0 +1,391 @@
+//! Keep Driver-owned overlay pixels out of Driver-owned desktop captures.
+//!
+//! The agent cursor and its session pill are drawn in a Driver-owned overlay
+//! window that sits above the desktop. A desktop capture taken for the agent
+//! must not contain those pixels: after a click the cursor rests on the
+//! target, so a capture would bake the arrow and the pill over the very label
+//! the agent is trying to read.
+//!
+//! The exclusion is scoped to the Driver's own agent-facing capture. External
+//! recorders and the Driver's recording feature keep seeing the cursor, so no
+//! adapter may exclude the overlay from capture permanently.
+//!
+//! This module owns the cross-platform semantics:
+//!
+//! - one desktop capture excludes overlays at a time, so one capture can never
+//! restore the overlay while another is still reading the screen;
+//! - the overlay is restored exactly once after every exclusion, even when the
+//! capture fails or panics;
+//! - the result always says what happened ([`AgentOverlayCapture`]). A
+//! platform that cannot exclude still returns the capture, but reports
+//! `not_excluded` with a reason instead of pretending the pixels are clean.
+//!
+//! Adapters implement [`OverlayCaptureExcluder`] with the native mechanism
+//! (window display affinity, an emptied X11 shape, a ScreenCaptureKit filter)
+//! and nothing else.
+
+use std::sync::Mutex;
+
+pub use cua_driver_contract::{AgentOverlayCapture, AgentOverlayCaptureStatus};
+
+/// Outcome of asking a platform to keep its overlays out of the next capture.
+#[derive(Debug)]
+pub enum ExclusionStart<H> {
+ /// No Driver overlay pixels are on screen; the capture runs as is.
+ NotPresent,
+ /// The overlay is out of the captures that follow until
+ /// [`OverlayCaptureExcluder::restore`] runs. `hidden` carries whatever the
+ /// adapter needs to verify the capture or to restore the overlay.
+ Excluded { method: &'static str, hidden: H },
+ /// The overlay may be on screen and could not be kept out.
+ Unsupported { reason: String },
+}
+
+/// What an adapter found when it inspected the captured pixels.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ResidualCheck {
+ /// Nothing of the overlay is left in the capture.
+ Clean,
+ /// Overlay pixels were still present and were replaced with the desktop
+ /// the overlay last saw under them. `method` names that repair.
+ Repaired { method: &'static str },
+ /// Overlay pixels are still present in the capture.
+ Residual { reason: String },
+}
+
+/// Native mechanism that hides or excludes a platform's overlay windows.
+pub trait OverlayCaptureExcluder {
+ /// Adapter state carried from [`Self::exclude`] to the capture and to
+ /// [`Self::restore`].
+ type Hidden;
+
+ /// Keep every Driver overlay out of the captures that follow.
+ fn exclude(&self) -> ExclusionStart<Self::Hidden>;
+
+ /// Undo a successful [`Self::exclude`]. Called exactly once per
+ /// `Excluded` start, after the capture, whether or not it succeeded.
+ fn restore(&self, hidden: Self::Hidden);
+}
+
+/// Serializes overlay exclusion across concurrent desktop captures.
+static EXCLUSION_GATE: Mutex<()> = Mutex::new(());
+
+/// Run `capture` with every Driver overlay kept out of the screen pixels and
+/// report what happened.
+///
+/// `capture` receives the adapter state when the overlay was excluded, and
+/// returns the capture plus its own inspection of the pixels. An inspection is
+/// only meaningful after an exclusion; for a capture that ran without one the
+/// check is ignored.
+pub fn capture_excluding_overlays<X, T, E>(
+ excluder: &X,
+ capture: impl FnOnce(Option<&X::Hidden>) -> Result<(T, ResidualCheck), E>,
+) -> Result<(T, AgentOverlayCapture), E>
+where
+ X: OverlayCaptureExcluder,
+{
+ // A poisoned gate only means another capture panicked while holding it;
+ // its restore already ran from the guard below, so the gate is still sound.
+ let _gate = EXCLUSION_GATE
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+
+ match excluder.exclude() {
+ ExclusionStart::NotPresent => {
+ let (value, _) = capture(None)?;
+ Ok((value, AgentOverlayCapture::not_present()))
+ }
+ ExclusionStart::Unsupported { reason } => {
+ let (value, _) = capture(None)?;
+ Ok((value, AgentOverlayCapture::not_excluded(reason)))
+ }
+ ExclusionStart::Excluded { method, hidden } => {
+ let restore = RestoreOnDrop {
+ excluder,
+ hidden: Some(hidden),
+ };
+ let result = capture(restore.hidden.as_ref());
+ drop(restore);
+ let (value, check) = result?;
+ let report = match check {
+ ResidualCheck::Clean => AgentOverlayCapture::excluded(method),
+ ResidualCheck::Repaired { method: repair } => {
+ AgentOverlayCapture::excluded(format!("{method}+{repair}"))
+ }
+ ResidualCheck::Residual { reason } => AgentOverlayCapture::not_excluded(reason),
+ };
+ Ok((value, report))
+ }
+ }
+}
+
+/// Restores the overlay when the capture returns or unwinds.
+struct RestoreOnDrop<'a, X: OverlayCaptureExcluder> {
+ excluder: &'a X,
+ hidden: Option<X::Hidden>,
+}
+
+impl<X: OverlayCaptureExcluder> Drop for RestoreOnDrop<'_, X> {
+ fn drop(&mut self) {
+ if let Some(hidden) = self.hidden.take() {
+ self.excluder.restore(hidden);
+ }
+ }
+}
+
+/// Append the one-line caveat an agent reading only the text content needs:
+/// nothing for a clean capture, the reason when overlay pixels may be present.
+pub fn append_summary_note(summary: &mut String, report: &AgentOverlayCapture) {
+ if report.is_clean() {
+ return;
+ }
+ summary.push_str("; the Driver's agent cursor overlay may appear in this image");
+ if let Some(reason) = &report.reason {
+ summary.push_str(" (");
+ summary.push_str(reason);
+ summary.push(')');
+ }
+}
+
+/// Excluder for builds and platforms with no overlay window at all.
+pub struct NoOverlay;
+
+impl OverlayCaptureExcluder for NoOverlay {
+ type Hidden = ();
+
+ fn exclude(&self) -> ExclusionStart<()> {
+ ExclusionStart::NotPresent
+ }
+
+ fn restore(&self, _hidden: ()) {}
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+
+ #[derive(Default)]
+ struct Counters {
+ excluded: AtomicUsize,
+ restored: AtomicUsize,
+ hidden_now: AtomicUsize,
+ max_hidden: AtomicUsize,
+ }
+
+ enum Mode {
+ Present,
+ Absent,
+ Unsupported,
+ }
+
+ struct Fake {
+ mode: Mode,
+ counters: Arc<Counters>,
+ }
+
+ impl Fake {
+ fn new(mode: Mode) -> Self {
+ Self {
+ mode,
+ counters: Arc::default(),
+ }
+ }
+ }
+
+ impl OverlayCaptureExcluder for Fake {
+ type Hidden = u32;
+
+ fn exclude(&self) -> ExclusionStart<u32> {
+ match self.mode {
+ Mode::Present => {
+ self.counters.excluded.fetch_add(1, Ordering::SeqCst);
+ let now = self.counters.hidden_now.fetch_add(1, Ordering::SeqCst) + 1;
+ self.counters.max_hidden.fetch_max(now, Ordering::SeqCst);
+ ExclusionStart::Excluded {
+ method: "fake_hide",
+ hidden: 7,
+ }
+ }
+ Mode::Absent => ExclusionStart::NotPresent,
+ Mode::Unsupported => ExclusionStart::Unsupported {
+ reason: "fake platform cannot hide".into(),
+ },
+ }
+ }
+
+ fn restore(&self, hidden: u32) {
+ assert_eq!(hidden, 7, "restore receives the state exclude produced");
+ self.counters.hidden_now.fetch_sub(1, Ordering::SeqCst);
+ self.counters.restored.fetch_add(1, Ordering::SeqCst);
+ }
+ }
+
+ fn ok_clean(
+ value: &'static str,
+ ) -> impl FnOnce(Option<&u32>) -> Result<(&'static str, ResidualCheck), String> {
+ move |_| Ok((value, ResidualCheck::Clean))
+ }
+
+ #[test]
+ fn excluded_capture_restores_once_and_reports_the_method() {
+ let fake = Fake::new(Mode::Present);
+ let (value, report) = capture_excluding_overlays(&fake, |hidden| {
+ assert_eq!(hidden, Some(&7));
+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 0);
+ Ok::<_, String>(("png", ResidualCheck::Clean))
+ })
+ .unwrap();
+ assert_eq!(value, "png");
+ assert_eq!(report, AgentOverlayCapture::excluded("fake_hide"));
+ assert!(report.is_clean());
+ assert_eq!(fake.counters.excluded.load(Ordering::SeqCst), 1);
+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn failed_capture_still_restores_the_overlay() {
+ let fake = Fake::new(Mode::Present);
+ let error = capture_excluding_overlays(&fake, |_| {
+ Err::<(&str, ResidualCheck), _>("grab failed".to_string())
+ })
+ .unwrap_err();
+ assert_eq!(error, "grab failed");
+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn panicking_capture_still_restores_the_overlay() {
+ let fake = Fake::new(Mode::Present);
+ let counters = Arc::clone(&fake.counters);
+ let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ let _ = capture_excluding_overlays(&fake, |_| -> Result<((), ResidualCheck), ()> {
+ panic!("capture backend panicked")
+ });
+ }));
+ assert!(outcome.is_err());
+ assert_eq!(counters.restored.load(Ordering::SeqCst), 1);
+ // The gate is usable again after the panic.
+ let (_, report) = capture_excluding_overlays(&fake, ok_clean("again")).unwrap();
+ assert_eq!(report.status, AgentOverlayCaptureStatus::Excluded);
+ assert_eq!(counters.restored.load(Ordering::SeqCst), 2);
+ }
+
+ #[test]
+ fn absent_overlay_reports_not_present_without_restoring() {
+ let fake = Fake::new(Mode::Absent);
+ let (_, report) = capture_excluding_overlays(&fake, |hidden| {
+ assert!(hidden.is_none());
+ // A residual verdict without an exclusion carries no meaning.
+ Ok::<_, String>((
+ (),
+ ResidualCheck::Residual {
+ reason: "ignored".into(),
+ },
+ ))
+ })
+ .unwrap();
+ assert_eq!(report, AgentOverlayCapture::not_present());
+ assert!(report.is_clean());
+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 0);
+ }
+
+ #[test]
+ fn unsupported_platform_captures_and_publishes_the_limitation() {
+ let fake = Fake::new(Mode::Unsupported);
+ let (value, report) = capture_excluding_overlays(&fake, ok_clean("png")).unwrap();
+ assert_eq!(value, "png");
+ assert_eq!(
+ report,
+ AgentOverlayCapture::not_excluded("fake platform cannot hide")
+ );
+ assert!(!report.is_clean());
+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 0);
+ }
+
+ #[test]
+ fn residual_pixels_downgrade_the_report() {
+ let fake = Fake::new(Mode::Present);
+ let (_, report) = capture_excluding_overlays(&fake, |_| {
+ Ok::<_, String>((
+ (),
+ ResidualCheck::Residual {
+ reason: "window did not repaint".into(),
+ },
+ ))
+ })
+ .unwrap();
+ assert_eq!(
+ report,
+ AgentOverlayCapture::not_excluded("window did not repaint")
+ );
+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn repaired_pixels_name_both_mechanisms() {
+ let fake = Fake::new(Mode::Present);
+ let (_, report) = capture_excluding_overlays(&fake, |_| {
+ Ok::<_, String>((
+ (),
+ ResidualCheck::Repaired {
+ method: "save_under",
+ },
+ ))
+ })
+ .unwrap();
+ assert_eq!(
+ report,
+ AgentOverlayCapture::excluded("fake_hide+save_under")
+ );
+ }
+
+ #[test]
+ fn concurrent_captures_never_overlap_their_exclusions() {
+ let fake = Arc::new(Fake::new(Mode::Present));
+ let threads: Vec<_> = (0..8)
+ .map(|_| {
+ let fake = Arc::clone(&fake);
+ std::thread::spawn(move || {
+ for _ in 0..25 {
+ capture_excluding_overlays(fake.as_ref(), |_| {
+ std::thread::yield_now();
+ Ok::<_, String>(((), ResidualCheck::Clean))
+ })
+ .unwrap();
+ }
+ })
+ })
+ .collect();
+ for thread in threads {
+ thread.join().unwrap();
+ }
+ assert_eq!(fake.counters.excluded.load(Ordering::SeqCst), 200);
+ assert_eq!(fake.counters.restored.load(Ordering::SeqCst), 200);
+ assert_eq!(fake.counters.max_hidden.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn summary_note_is_added_only_when_overlay_pixels_may_remain() {
+ let mut clean = String::from("desktop screenshot");
+ append_summary_note(&mut clean, &AgentOverlayCapture::excluded("m"));
+ append_summary_note(&mut clean, &AgentOverlayCapture::not_present());
+ assert_eq!(clean, "desktop screenshot");
+
+ let mut caveat = String::from("desktop screenshot");
+ append_summary_note(&mut caveat, &AgentOverlayCapture::not_excluded("no API"));
+ assert_eq!(
+ caveat,
+ "desktop screenshot; the Driver's agent cursor overlay may appear in this image (no API)"
+ );
+ }
+
+ #[test]
+ fn no_overlay_excluder_reports_not_present() {
+ let (_, report) =
+ capture_excluding_overlays(&NoOverlay, |_| Ok::<_, String>(((), ResidualCheck::Clean)))
+ .unwrap();
+ assert_eq!(report.status, AgentOverlayCaptureStatus::NotPresent);
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs
index dd90baaaa2..37d1655041 100644
--- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs
@@ -8,6 +8,7 @@
pub mod badge_glyphs;
pub mod bezier;
+pub mod capture_exclusion;
pub mod capture_utils;
pub mod motion;
pub mod path_planner;
@@ -330,6 +331,10 @@ pub enum OverlayMsg {
/// This deliberately does not recreate a cursor; the next command does so
/// lazily after the successful `start_session` boundary.
Revive(CursorKey),
+ /// No state change: wakes a parked render loop so it services an
+ /// out-of-band request (such as hiding for a Driver desktop capture)
+ /// without waiting for its next maintenance tick.
+ Wake,
}
/// Commands sent from MCP tool handlers to the overlay's render thread.
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs
index bfe0243e1d..941edd9104 100644
--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs
+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_map.rs
@@ -251,6 +251,7 @@ impl<S: RenderEntry, P> RenderMap<S, P> {
OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd }) => self.apply_command(key, cmd),
OverlayMsg::Remove(key) => self.remove(key),
OverlayMsg::Revive(key) => self.revive(key),
+ OverlayMsg::Wake => MsgOutcome::Ignored,
}
}
@@ -454,6 +455,18 @@ mod tests {
assert_eq!(map.cursors.len(), 2);
}
+ #[test]
+ fn wake_changes_no_render_state() {
+ let mut map = map();
+ map.apply_msg(move_msg("sessA", 10.0, 10.0));
+ let cursors = map.cursors.len();
+ let last_active = map.last_active.clone();
+ assert_eq!(map.apply_msg(OverlayMsg::Wake), MsgOutcome::Ignored);
+ assert_eq!(map.cursors.len(), cursors);
+ assert_eq!(map.last_active, last_active);
+ assert!(map.ended.is_empty());
+ }
+
#[test]
fn revival_clears_the_tombstone_and_recreates_lazily() {
let mut map = map();
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
index b29e82d2e8..c6ba57b5c7 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
@@ -14,6 +14,7 @@ use cua_driver_core::tool::ToolRegistry;
pub mod health_report;
pub mod overlay;
+pub mod overlay_capture;
pub mod pip;
pub mod tools;
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
index 8d8ba00d29..f51e2a2dc0 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
@@ -120,6 +120,47 @@ fn should_start_x11_overlay(wayland_display_present: bool) -> bool {
!wayland_display_present
}
+/// Set while the X11 owner thread is running its render loop over a mapped
+/// overlay window; only then can a desktop capture ask it to hide.
+static X11_OVERLAY_LIVE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
+/// Whether an X11 overlay window exists that a desktop capture must hide.
+pub(crate) fn x11_overlay_live() -> bool {
+ X11_OVERLAY_LIVE.load(std::sync::atomic::Ordering::Acquire)
+}
+
+/// Wake the X11 owner thread so it services a capture hold now instead of on
+/// its next maintenance tick.
+pub(crate) fn wake_x11_overlay() {
+ let _ = try_send_x11_message(CMD_TX.get(), OverlayMsg::Wake);
+}
+
+/// The explicit limitation for Driver desktop captures on a Wayland session
+/// whose agent cursor is drawn by the compositor (layer-shell surface or
+/// GNOME Shell helper). `None` when no Wayland overlay is active.
+pub(crate) fn wayland_capture_limitation() -> Option<String> {
+ #[cfg(target_os = "linux")]
+ {
+ if !crate::wayland::is_wayland() {
+ return None;
+ }
+ let backend = match WAYLAND_OVERLAY_BACKEND.get() {
+ Some(WaylandOverlayBackend::LayerShell) => "layer-shell surface",
+ Some(WaylandOverlayBackend::SemanticShellHelper)
+ | Some(WaylandOverlayBackend::LegacyShellHelper) => "GNOME Shell helper",
+ Some(WaylandOverlayBackend::None) | None => return None,
+ };
+ Some(format!(
+ "on Wayland the agent cursor is drawn by the compositor ({backend}) into the same \
+ output the desktop capture reads, and the Driver cannot leave it out"
+ ))
+ }
+ #[cfg(not(target_os = "linux"))]
+ {
+ None
+ }
+}
+
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum WaylandOverlayBackend {
SemanticShellHelper,
@@ -337,7 +378,7 @@ fn dispatch_wayland_overlay_message(msg: &OverlayMsg) -> WaylandOverlayBackend {
dispatch_shell_helper_command(&command.key, &command.cmd, semantic);
}
OverlayMsg::Remove(_) => crate::wayland::shell_helper::hide_cursor(),
- OverlayMsg::Revive(_) => {}
+ OverlayMsg::Revive(_) | OverlayMsg::Wake => {}
}
}
WaylandOverlayBackend::LayerShell if !crate::wayland::overlay::forward(msg) => {
@@ -1188,6 +1229,13 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
win,
root,
};
+ // Whether the last paint left a non-empty bounding shape on screen, so a
+ // desktop capture knows there is something to hide and wait for.
+ let mut overlay_shaped = false;
+ // A Driver desktop capture is reading the root: paint nothing.
+ let mut capture_held = false;
+ let mut repaint_after_capture = false;
+ let _live = X11OverlayLiveGuard::start();
loop {
// Idle fast path: no Pixmap allocation, RGBA→BGRA copy, XShape update,
@@ -1350,17 +1398,55 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
z_enforcer.reassert(pinned_wid);
}
+ // A Driver desktop capture wants the overlay out of the root image:
+ // empty the shape, confirm the server applied it, and report what the
+ // capture needs to find pixels a slow window has not repainted yet.
+ if let Some(generation) = crate::overlay_capture::CAPTURE_HOLD.unanswered_request() {
+ use crate::overlay_capture::{HiddenOverlay, HoldAnswer};
+ let answer = if overlay_shaped {
+ match blank_x11_overlay_shape(&conn, win, true) {
+ Ok(()) => {
+ overlay_shaped = false;
+ HoldAnswer::Hidden(HiddenOverlay {
+ was_shown: true,
+ composited: compositor_present,
+ residual: backdrop.residual_patches(Instant::now()),
+ })
+ }
+ Err(e) => HoldAnswer::Failed(format!("{e:#}")),
+ }
+ } else {
+ HoldAnswer::Hidden(HiddenOverlay {
+ was_shown: false,
+ composited: compositor_present,
+ residual: Vec::new(),
+ })
+ };
+ capture_held = matches!(answer, HoldAnswer::Hidden(_));
+ crate::overlay_capture::CAPTURE_HOLD.answer(generation, answer);
+ }
+ if capture_held && !crate::overlay_capture::CAPTURE_HOLD.is_requested() {
+ capture_held = false;
+ repaint_after_capture = true;
+ }
+
// Render after a command, during an active animation/fade, and once
// more as the final active state settles. This leaves the X11 window in
// its completed/cleared state before the next blocking receive.
let mut paint_deferred = false;
- if had_msg
+ if capture_held {
+ // Held for a capture: the frame is owed, so keep ticking and let
+ // arrivals wait until it is actually on screen.
+ paint_deferred = true;
+ } else if had_msg
|| screen_changed
|| compositor_changed
|| hover_changed
|| frame_tick_needed
|| next_frame_tick_needed
+ || repaint_after_capture
{
+ repaint_after_capture = false;
let tiles = {
let guard = RENDER.lock().unwrap();
guard.as_ref().map(render_x11_tiles)
@@ -1379,7 +1465,12 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
screen_changed,
&mut backdrop,
) {
- Ok(outcome) => paint_deferred = outcome == X11PaintOutcome::Deferred,
+ Ok(outcome) => {
+ paint_deferred = outcome == X11PaintOutcome::Deferred;
+ if outcome == X11PaintOutcome::Painted {
+ overlay_shaped = !tiles.is_empty();
+ }
+ }
Err(e) => {
// A broken X11 connection cannot recover inside this
// owner thread. Exit instead of spinning at frame cadence
@@ -1424,6 +1515,27 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
conn.flush().ok();
}
+/// Publishes [`X11_OVERLAY_LIVE`] for the render loop's lifetime, however the
+/// loop ends, and releases a capture waiting on a hold that will never be
+/// answered.
+#[cfg(target_os = "linux")]
+struct X11OverlayLiveGuard;
+
+#[cfg(target_os = "linux")]
+impl X11OverlayLiveGuard {
+ fn start() -> Self {
+ X11_OVERLAY_LIVE.store(true, std::sync::atomic::Ordering::Release);
+ Self
+ }
+}
+
+#[cfg(target_os = "linux")]
+impl Drop for X11OverlayLiveGuard {
+ fn drop(&mut self) {
+ X11_OVERLAY_LIVE.store(false, std::sync::atomic::Ordering::Release);
+ }
+}
+
// ── Z-order enforcer (Linux impl of cursor_overlay::ZOrderEnforcer) ──────
/// X11 implementation of [`cursor_overlay::ZOrderEnforcer`].
@@ -1876,6 +1988,24 @@ impl X11BackdropCache {
}
}
+ /// Every retained painted rect, newest first, for a desktop capture to
+ /// find pixels a window has not repainted since the overlay hid.
+ fn residual_patches(&self, now: Instant) -> Vec<crate::overlay_capture::ResidualPatch> {
+ self.frames
+ .iter()
+ .filter(|frame| !frame.expired(now))
+ .flat_map(|frame| frame.saved.iter())
+ .map(|saved| crate::overlay_capture::ResidualPatch {
+ x: i32::from(saved.bounds.x),
+ y: i32::from(saved.bounds.y),
+ width: usize::from(saved.bounds.width),
+ height: usize::from(saved.bounds.height),
+ under: saved.under.clone(),
+ uploaded: saved.uploaded.clone(),
+ })
+ .collect()
+ }
+
fn record_frame(
&mut self,
at: Instant,
@@ -2965,6 +3095,205 @@ mod tests {
Ok(())
}
+ /// D-WL5: a desktop capture taken while the cursor and its session pill
+ /// rest over a control must not contain them, while the screen (and any
+ /// external capture) keeps showing them before and after.
+ #[test]
+ #[ignore = "requires a live X11 server (run under xvfb-run)"]
+ fn x11_desktop_capture_hides_the_resting_overlay_and_restores_it() -> anyhow::Result<()> {
+ use cursor_overlay::capture_exclusion::{
+ capture_excluding_overlays, AgentOverlayCaptureStatus,
+ };
+ use x11rb::connection::Connection;
+ use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK};
+ use x11rb::protocol::xproto::{
+ AtomEnum, ConnectionExt as XprotoConnectionExt, CreateWindowAux, ImageFormat, Window,
+ WindowClass,
+ };
+
+ const BACKGROUND: u32 = 0x00c8_c8c8;
+ const PROBE: (i16, i16, u16, u16) = (120, 120, 240, 120);
+
+ fn find_named_window(
+ conn: &impl Connection,
+ root: Window,
+ expected_name: &[u8],
+ ) -> anyhow::Result<Window> {
+ let deadline = Instant::now() + Duration::from_secs(3);
+ while Instant::now() < deadline {
+ for window in conn.query_tree(root)?.reply()?.children {
+ let name = conn
+ .get_property(
+ false,
+ window,
+ AtomEnum::WM_NAME,
+ AtomEnum::STRING,
+ 0,
+ u32::MAX,
+ )?
+ .reply()?;
+ if name.value == expected_name {
+ return Ok(window);
+ }
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ anyhow::bail!("timed out waiting for the overlay window")
+ }
+
+ fn wait_for_shape(
+ conn: &impl Connection,
+ overlay: Window,
+ phase: &str,
+ ) -> anyhow::Result<()> {
+ let deadline = Instant::now() + Duration::from_secs(3);
+ while Instant::now() < deadline {
+ if !conn
+ .shape_get_rectangles(overlay, SK::BOUNDING)?
+ .reply()?
+ .rectangles
+ .is_empty()
+ {
+ return Ok(());
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ anyhow::bail!("overlay bounding shape stayed empty during {phase}")
+ }
+
+ /// Pixels in the probe rect of a root-sized BGRX/RGBA buffer that are
+ /// not the target window's background.
+ fn foreign_pixels(pixel: impl Fn(usize, usize) -> [u8; 3]) -> usize {
+ let (x0, y0, w, h) = PROBE;
+ let mut count = 0;
+ for y in y0 as usize..y0 as usize + h as usize {
+ for x in x0 as usize..x0 as usize + w as usize {
+ if pixel(x, y) != [0xc8, 0xc8, 0xc8] {
+ count += 1;
+ }
+ }
+ }
+ count
+ }
+
+ fn live_foreign_pixels(conn: &impl Connection, root: Window) -> anyhow::Result<usize> {
+ let (x0, y0, w, h) = PROBE;
+ let image = conn
+ .get_image(ImageFormat::Z_PIXMAP, root, x0, y0, w, h, !0u32)?
+ .reply()?;
+ let stride = usize::from(w) * 4;
+ Ok(foreign_pixels(|x, y| {
+ let offset = (y - y0 as usize) * stride + (x - x0 as usize) * 4;
+ let bgrx = &image.data[offset..offset + 3];
+ [bgrx[2], bgrx[1], bgrx[0]]
+ }))
+ }
+
+ let (conn, screen_num) = x11rb::connect(None)?;
+ let screen = &conn.setup().roots[screen_num];
+ let root = screen.root;
+ let target = conn.generate_id()?;
+ conn.create_window(
+ screen.root_depth,
+ target,
+ root,
+ 0,
+ 0,
+ 480,
+ 360,
+ 0,
+ WindowClass::INPUT_OUTPUT,
+ screen.root_visual,
+ &CreateWindowAux::new()
+ .background_pixel(BACKGROUND)
+ .override_redirect(1u32),
+ )?
+ .check()?;
+ conn.map_window(target)?.check()?;
+ conn.flush()?;
+ anyhow::ensure!(
+ live_foreign_pixels(&conn, root)? == 0,
+ "the probe rect is not a solid target background before the cursor shows"
+ );
+
+ let cursor_id = "d-wl5-capture-exclusion-probe";
+ init(CursorConfig {
+ cursor_id: cursor_id.to_owned(),
+ ..CursorConfig::default()
+ });
+ run_on_thread();
+ let overlay = find_named_window(
+ &conn,
+ root,
+ format!("Cua.AgentCursorOverlay.{cursor_id}").as_bytes(),
+ )?;
+ let deadline = Instant::now() + Duration::from_secs(3);
+ while !crate::overlay::x11_overlay_live() {
+ anyhow::ensure!(Instant::now() < deadline, "overlay loop never went live");
+ std::thread::sleep(Duration::from_millis(10));
+ }
+
+ send_command(OverlayCommand::SetEnabled(true));
+ send_command(OverlayCommand::SetSessionLabel(
+ "ga-r5-window-772012bf".into(),
+ ));
+ send_command(OverlayCommand::SnapTo {
+ x: 200.0,
+ y: 160.0,
+ heading_radians: None,
+ });
+ wait_for_shape(&conn, overlay, "cursor show")?;
+ // Let the arrival and the pill settle into their resting frame.
+ std::thread::sleep(Duration::from_millis(400));
+ conn.configure_window(
+ overlay,
+ &x11rb::protocol::xproto::ConfigureWindowAux::new()
+ .stack_mode(x11rb::protocol::xproto::StackMode::ABOVE),
+ )?;
+ conn.flush()?;
+ let visible_before = live_foreign_pixels(&conn, root)?;
+ anyhow::ensure!(
+ visible_before >= 50,
+ "the resting cursor and pill are not visible on screen ({visible_before} pixels)"
+ );
+
+ let (png, report) =
+ capture_excluding_overlays(&crate::overlay_capture::OverlayExcluder, |hidden| {
+ let png = crate::capture::screenshot_display_bytes()?;
+ Ok::<_, anyhow::Error>(crate::overlay_capture::verify_hidden_capture(png, hidden))
+ })?;
+ anyhow::ensure!(
+ report.status == AgentOverlayCaptureStatus::Excluded,
+ "desktop capture did not exclude the overlay: {report:?}"
+ );
+ let image = image::load_from_memory(&png)?.to_rgba8();
+ let in_capture = foreign_pixels(|x, y| {
+ let [r, g, b, _] = image.get_pixel(x as u32, y as u32).0;
+ [r, g, b]
+ });
+ eprintln!("visible_before={visible_before} in_capture={in_capture} report={report:?}");
+ anyhow::ensure!(
+ in_capture == 0,
+ "desktop capture still contains {in_capture} overlay pixels over the target"
+ );
+
+ wait_for_shape(&conn, overlay, "restore after capture")?;
+ let deadline = Instant::now() + Duration::from_secs(3);
+ loop {
+ let visible_after = live_foreign_pixels(&conn, root)?;
+ if visible_after >= 50 {
+ eprintln!("visible_after={visible_after}");
+ break;
+ }
+ anyhow::ensure!(
+ Instant::now() < deadline,
+ "the cursor did not come back after the capture ({visible_after} pixels)"
+ );
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ Ok(())
+ }
+
#[test]
#[ignore = "requires a live X11 server (run under xvfb-run)"]
fn x11_overlay_input_shape_stays_empty_and_clicks_pass_through_after_resize(
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay_capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay_capture.rs
new file mode 100644
index 0000000000..b01fa6cec3
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay_capture.rs
@@ -0,0 +1,512 @@
+//! Linux adapter of `cursor_overlay::capture_exclusion`.
+//!
+//! X11 has no per-window capture exclusion: a root `GetImage` returns whatever
+//! the server is showing, our override-redirect overlay included. The adapter
+//! therefore hides the overlay around the grab:
+//!
+//! 1. the capture thread raises a hold and wakes the overlay owner thread;
+//! 2. the owner thread empties the overlay's bounding shape, round-trips the
+//! server so the change is in effect, answers with what it knows about the
+//! pixels it last painted, and paints nothing until the hold is released;
+//! 3. the capture thread lets the uncovered windows repaint, grabs the root,
+//! and releases the hold; the owner thread repaints the current frame.
+//!
+//! Under a compositing manager the uncovered pixels come back from the
+//! windows' own buffers on the next composite. Without one the windows below
+//! must answer Expose, and a slow client can leave our last frame in the
+//! framebuffer. Those pixels are still exactly what the overlay uploaded, so
+//! they are found by comparison and replaced with the desktop the overlay
+//! recorded under them ([`scrub_residual_overlay`]) — the same save-under rule
+//! the overlay's own compositing uses.
+//!
+//! Wayland overlays (layer-shell surface, GNOME Shell helper) are drawn by the
+//! compositor into the same output the screencopy / portal capture reads, and
+//! the Driver has no way to leave them out. That limitation is reported as
+//! `not_excluded` instead of being papered over.
+
+use std::sync::{Condvar, Mutex};
+use std::time::Duration;
+
+use cursor_overlay::capture_exclusion::{ExclusionStart, OverlayCaptureExcluder};
+
+/// Method names reported in `agent_overlay_capture.method`.
+pub const X11_METHOD: &str = "x11_unshaped_overlay";
+pub const X11_COMPOSITED_METHOD: &str = "x11_unshaped_overlay_under_compositor";
+pub const X11_SAVE_UNDER_REPAIR: &str = "x11_save_under";
+
+/// How long the capture thread waits for the overlay owner to hide.
+const HOLD_ACK_TIMEOUT: Duration = Duration::from_millis(500);
+/// Time given to the compositor, or to the windows answering Expose, before
+/// the root is read. Whatever is still ours afterwards is scrubbed.
+const HIDE_SETTLE: Duration = Duration::from_millis(50);
+
+/// One rect the overlay painted: the desktop it recorded under the rect and
+/// the pixels it uploaded, both row-major BGRX (`width * height * 4`).
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ResidualPatch {
+ pub x: i32,
+ pub y: i32,
+ pub width: usize,
+ pub height: usize,
+ pub under: Vec<u8>,
+ pub uploaded: Vec<u8>,
+}
+
+/// What the overlay owner reported after hiding.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct HiddenOverlay {
+ /// Whether any overlay pixel was on screen before the hide.
+ pub was_shown: bool,
+ /// Whether a compositing manager blends the overlay (no residual check is
+ /// possible or needed: the compositor repaints from window buffers).
+ pub composited: bool,
+ /// Painted rects that may still hold our pixels, newest first.
+ pub residual: Vec<ResidualPatch>,
+}
+
+/// The owner thread's answer to a hold request.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum HoldAnswer {
+ Hidden(HiddenOverlay),
+ Failed(String),
+}
+
+#[derive(Default)]
+struct HoldState {
+ generation: u64,
+ requested: bool,
+ answer: Option<(u64, HoldAnswer)>,
+}
+
+/// Handshake between a capture thread and the overlay owner thread.
+pub struct CaptureHold {
+ state: Mutex<HoldState>,
+ changed: Condvar,
+}
+
+impl CaptureHold {
+ pub const fn new() -> Self {
+ Self {
+ state: Mutex::new(HoldState {
+ generation: 0,
+ requested: false,
+ answer: None,
+ }),
+ changed: Condvar::new(),
+ }
+ }
+
+ fn lock(&self) -> std::sync::MutexGuard<'_, HoldState> {
+ self.state
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ }
+
+ /// Capture side: ask the owner to hide. Returns the request generation.
+ pub fn request(&self) -> u64 {
+ let mut state = self.lock();
+ state.generation = state.generation.wrapping_add(1);
+ state.requested = true;
+ state.answer = None;
+ state.generation
+ }
+
+ /// Capture side: wait for the owner's answer to `generation`.
+ pub fn wait_answer(&self, generation: u64, timeout: Duration) -> Option<HoldAnswer> {
+ let state = self.lock();
+ let (mut state, _) = self
+ .changed
+ .wait_timeout_while(
+ state,
+ timeout,
+ |state| !matches!(&state.answer, Some((answered, _)) if *answered == generation),
+ )
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ match state.answer.take() {
+ Some((answered, answer)) if answered == generation => Some(answer),
+ other => {
+ state.answer = other;
+ None
+ }
+ }
+ }
+
+ /// Capture side: let the overlay paint again.
+ pub fn release(&self) {
+ let mut state = self.lock();
+ state.requested = false;
+ state.answer = None;
+ self.changed.notify_all();
+ }
+
+ /// Owner side: the request still waiting for an answer, if any.
+ pub fn unanswered_request(&self) -> Option<u64> {
+ let state = self.lock();
+ (state.requested && state.answer.is_none()).then_some(state.generation)
+ }
+
+ /// Owner side: whether painting is still held.
+ pub fn is_requested(&self) -> bool {
+ self.lock().requested
+ }
+
+ /// Owner side: answer `generation`. A stale answer (the capture already
+ /// gave up and released, or asked again) is dropped.
+ pub fn answer(&self, generation: u64, answer: HoldAnswer) {
+ let mut state = self.lock();
+ if state.requested && state.generation == generation {
+ state.answer = Some((generation, answer));
+ self.changed.notify_all();
+ }
+ }
+}
+
+impl Default for CaptureHold {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// The process-wide hold shared by desktop captures and the X11 overlay owner.
+pub static CAPTURE_HOLD: CaptureHold = CaptureHold::new();
+
+/// Replace overlay pixels still present in `rgba` (row-major RGBA of the whole
+/// root, `width * height * 4`) with the desktop recorded under them.
+///
+/// A patch only speaks for pixels it actually painted (`uploaded != under`).
+/// For those, the newest patch decides: if the captured pixel still equals
+/// what that patch uploaded, the window below never repainted and the pixel is
+/// replaced with `under`; otherwise the window repainted and the capture is
+/// the truth. Pixels a patch did not paint fall through to older patches.
+///
+/// Returns how many pixels were replaced.
+pub fn scrub_residual_overlay(
+ rgba: &mut [u8],
+ width: usize,
+ height: usize,
+ patches: &[ResidualPatch],
+) -> usize {
+ if rgba.len() != width.saturating_mul(height).saturating_mul(4) {
+ return 0;
+ }
+ let mut decided = vec![false; width * height];
+ let mut replaced = 0;
+ for patch in patches {
+ let expected = patch.width.saturating_mul(patch.height).saturating_mul(4);
+ if patch.under.len() != expected || patch.uploaded.len() != expected {
+ continue;
+ }
+ for row in 0..patch.height {
+ let Some(y) = offset(patch.y, row, height) else {
+ continue;
+ };
+ for column in 0..patch.width {
+ let Some(x) = offset(patch.x, column, width) else {
+ continue;
+ };
+ let source = (row * patch.width + column) * 4;
+ let uploaded = &patch.uploaded[source..source + 3];
+ let under = &patch.under[source..source + 3];
+ if uploaded == under {
+ continue;
+ }
+ let index = y * width + x;
+ if decided[index] {
+ continue;
+ }
+ decided[index] = true;
+ let target = index * 4;
+ let captured = &rgba[target..target + 3];
+ // BGRX patch vs RGBA capture.
+ if captured[0] == uploaded[2]
+ && captured[1] == uploaded[1]
+ && captured[2] == uploaded[0]
+ {
+ rgba[target] = under[2];
+ rgba[target + 1] = under[1];
+ rgba[target + 2] = under[0];
+ replaced += 1;
+ }
+ }
+ }
+ }
+ replaced
+}
+
+fn offset(origin: i32, delta: usize, limit: usize) -> Option<usize> {
+ let value = i64::from(origin) + i64::try_from(delta).ok()?;
+ usize::try_from(value).ok().filter(|value| *value < limit)
+}
+
+/// Excluder used by the Linux `get_desktop_state`.
+pub struct OverlayExcluder;
+
+impl OverlayCaptureExcluder for OverlayExcluder {
+ type Hidden = HiddenOverlay;
+
+ fn exclude(&self) -> ExclusionStart<HiddenOverlay> {
+ if let Some(reason) = crate::overlay::wayland_capture_limitation() {
+ return ExclusionStart::Unsupported { reason };
+ }
+ if !crate::overlay::x11_overlay_live() {
+ return ExclusionStart::NotPresent;
+ }
+ let generation = CAPTURE_HOLD.request();
+ crate::overlay::wake_x11_overlay();
+ match CAPTURE_HOLD.wait_answer(generation, HOLD_ACK_TIMEOUT) {
+ Some(HoldAnswer::Hidden(hidden)) => {
+ if hidden.was_shown {
+ std::thread::sleep(HIDE_SETTLE);
+ }
+ ExclusionStart::Excluded {
+ method: if hidden.composited {
+ X11_COMPOSITED_METHOD
+ } else {
+ X11_METHOD
+ },
+ hidden,
+ }
+ }
+ Some(HoldAnswer::Failed(error)) => {
+ release_hold();
+ ExclusionStart::Unsupported {
+ reason: format!("the X11 agent cursor overlay could not be hidden: {error}"),
+ }
+ }
+ None => {
+ release_hold();
+ ExclusionStart::Unsupported {
+ reason: format!(
+ "the X11 agent cursor overlay did not hide within {} ms",
+ HOLD_ACK_TIMEOUT.as_millis()
+ ),
+ }
+ }
+ }
+ }
+
+ fn restore(&self, _hidden: HiddenOverlay) {
+ release_hold();
+ }
+}
+
+fn release_hold() {
+ CAPTURE_HOLD.release();
+ crate::overlay::wake_x11_overlay();
+}
+
+/// Inspect a PNG captured while the overlay was hidden, scrubbing any pixels
+/// the overlay's last frame left behind. Returns the (possibly re-encoded) PNG
+/// and the verdict.
+#[cfg(target_os = "linux")]
+pub fn verify_hidden_capture(
+ png: Vec<u8>,
+ hidden: Option<&HiddenOverlay>,
+) -> (Vec<u8>, cursor_overlay::capture_exclusion::ResidualCheck) {
+ use cursor_overlay::capture_exclusion::ResidualCheck;
+ let Some(hidden) = hidden.filter(|hidden| !hidden.residual.is_empty()) else {
+ return (png, ResidualCheck::Clean);
+ };
+ let decoded = match image::load_from_memory(&png) {
+ Ok(decoded) => decoded.to_rgba8(),
+ Err(error) => {
+ return (
+ png,
+ ResidualCheck::Residual {
+ reason: format!(
+ "the desktop capture could not be checked for leftover overlay pixels: \
+ {error}"
+ ),
+ },
+ )
+ }
+ };
+ let (width, height) = decoded.dimensions();
+ let mut rgba = decoded.into_raw();
+ let replaced =
+ scrub_residual_overlay(&mut rgba, width as usize, height as usize, &hidden.residual);
+ if replaced == 0 {
+ return (png, ResidualCheck::Clean);
+ }
+ match cua_driver_core::image_utils::encode_rgba_to_png(&rgba, width, height) {
+ Ok(repaired) => (
+ repaired,
+ ResidualCheck::Repaired {
+ method: X11_SAVE_UNDER_REPAIR,
+ },
+ ),
+ Err(error) => (
+ png,
+ ResidualCheck::Residual {
+ reason: format!(
+ "{replaced} leftover overlay pixels were found but the repaired capture \
+ could not be encoded: {error}"
+ ),
+ },
+ ),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn bgrx(pixels: &[[u8; 3]]) -> Vec<u8> {
+ pixels
+ .iter()
+ .flat_map(|[r, g, b]| [*b, *g, *r, 0])
+ .collect()
+ }
+
+ fn rgba(pixels: &[[u8; 3]]) -> Vec<u8> {
+ pixels
+ .iter()
+ .flat_map(|[r, g, b]| [*r, *g, *b, 255])
+ .collect()
+ }
+
+ const DESK: [u8; 3] = [200, 200, 200];
+ const CURSOR: [u8; 3] = [120, 60, 220];
+ const REPAINTED: [u8; 3] = [10, 120, 10];
+
+ #[test]
+ fn leftover_overlay_pixels_are_replaced_with_the_recorded_desktop() {
+ // 3x1 screen; the patch painted the middle pixel.
+ let mut capture = rgba(&[DESK, CURSOR, DESK]);
+ let patch = ResidualPatch {
+ x: 0,
+ y: 0,
+ width: 3,
+ height: 1,
+ under: bgrx(&[DESK, DESK, DESK]),
+ uploaded: bgrx(&[DESK, CURSOR, DESK]),
+ };
+ assert_eq!(scrub_residual_overlay(&mut capture, 3, 1, &[patch]), 1);
+ assert_eq!(capture, rgba(&[DESK, DESK, DESK]));
+ }
+
+ #[test]
+ fn a_repainted_pixel_is_kept() {
+ let mut capture = rgba(&[REPAINTED]);
+ let patch = ResidualPatch {
+ x: 0,
+ y: 0,
+ width: 1,
+ height: 1,
+ under: bgrx(&[DESK]),
+ uploaded: bgrx(&[CURSOR]),
+ };
+ assert_eq!(scrub_residual_overlay(&mut capture, 1, 1, &[patch]), 0);
+ assert_eq!(capture, rgba(&[REPAINTED]));
+ }
+
+ #[test]
+ fn the_newest_painting_patch_decides_and_unpainted_pixels_fall_through() {
+ // Newest patch painted pixel 0 only; pixel 1 is a backdrop copy there
+ // (uploaded == under), so the older patch that painted pixel 1 speaks.
+ let newest = ResidualPatch {
+ x: 0,
+ y: 0,
+ width: 2,
+ height: 1,
+ under: bgrx(&[DESK, DESK]),
+ uploaded: bgrx(&[CURSOR, DESK]),
+ };
+ let older = ResidualPatch {
+ x: 0,
+ y: 0,
+ width: 2,
+ height: 1,
+ under: bgrx(&[DESK, DESK]),
+ uploaded: bgrx(&[REPAINTED, CURSOR]),
+ };
+ // Pixel 0 shows the older frame's paint, which the newest frame
+ // painted over: after the hide the window must have repainted it.
+ let mut capture = rgba(&[REPAINTED, CURSOR]);
+ assert_eq!(
+ scrub_residual_overlay(&mut capture, 2, 1, &[newest, older]),
+ 1
+ );
+ assert_eq!(capture, rgba(&[REPAINTED, DESK]));
+ }
+
+ #[test]
+ fn patches_are_clipped_to_the_screen() {
+ let mut capture = rgba(&[CURSOR, DESK]);
+ let patch = ResidualPatch {
+ x: -1,
+ y: 0,
+ width: 2,
+ height: 2,
+ under: bgrx(&[DESK, DESK, DESK, DESK]),
+ uploaded: bgrx(&[CURSOR, CURSOR, CURSOR, CURSOR]),
+ };
+ assert_eq!(scrub_residual_overlay(&mut capture, 2, 1, &[patch]), 1);
+ assert_eq!(capture, rgba(&[DESK, DESK]));
+ }
+
+ #[test]
+ fn malformed_inputs_change_nothing() {
+ let mut capture = rgba(&[CURSOR]);
+ let short = ResidualPatch {
+ x: 0,
+ y: 0,
+ width: 1,
+ height: 1,
+ under: vec![0; 3],
+ uploaded: bgrx(&[CURSOR]),
+ };
+ assert_eq!(scrub_residual_overlay(&mut capture, 1, 1, &[short]), 0);
+ assert_eq!(scrub_residual_overlay(&mut capture, 2, 2, &[]), 0);
+ assert_eq!(capture, rgba(&[CURSOR]));
+ }
+
+ #[test]
+ fn hold_answers_reach_only_the_current_request() {
+ let hold = CaptureHold::new();
+ assert_eq!(hold.unanswered_request(), None);
+ let first = hold.request();
+ assert_eq!(hold.unanswered_request(), Some(first));
+ assert!(hold.is_requested());
+ hold.answer(first, HoldAnswer::Hidden(HiddenOverlay::default()));
+ assert_eq!(hold.unanswered_request(), None);
+ assert_eq!(
+ hold.wait_answer(first, Duration::ZERO),
+ Some(HoldAnswer::Hidden(HiddenOverlay::default()))
+ );
+ hold.release();
+ assert!(!hold.is_requested());
+
+ // An answer that arrives after the capture gave up is dropped.
+ let second = hold.request();
+ hold.release();
+ hold.answer(second, HoldAnswer::Failed("late".into()));
+ assert_eq!(hold.wait_answer(second, Duration::ZERO), None);
+
+ // An answer for an older request never satisfies a newer one.
+ let third = hold.request();
+ hold.answer(second, HoldAnswer::Failed("stale".into()));
+ assert_eq!(hold.wait_answer(third, Duration::ZERO), None);
+ assert_eq!(hold.unanswered_request(), Some(third));
+ }
+
+ #[test]
+ fn hold_wait_wakes_when_the_owner_answers_from_another_thread() {
+ let hold = std::sync::Arc::new(CaptureHold::new());
+ let generation = hold.request();
+ let owner = {
+ let hold = std::sync::Arc::clone(&hold);
+ std::thread::spawn(move || {
+ while hold.unanswered_request().is_none() {
+ std::thread::yield_now();
+ }
+ hold.answer(generation, HoldAnswer::Failed("owner".into()));
+ })
+ };
+ assert_eq!(
+ hold.wait_answer(generation, Duration::from_secs(5)),
+ Some(HoldAnswer::Failed("owner".into()))
+ );
+ owner.join().unwrap();
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index d3bdddda73..29e1939b77 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -12045,7 +12045,20 @@ impl Tool for GetDesktopStateTool {
// Capture the full display at native size first. When the
// compositor consumes logical input coordinates, normalize the
// image below so screenshot pixels still land exactly.
- let native_png = crate::capture::screenshot_display_bytes()?;
+ //
+ // The agent reads this image, so the Driver's own cursor and
+ // session pill are hidden around the grab (or the limitation is
+ // reported) instead of being baked over the controls it reads.
+ let (native_png, overlay_capture) =
+ cursor_overlay::capture_exclusion::capture_excluding_overlays(
+ &crate::overlay_capture::OverlayExcluder,
+ |hidden| {
+ let png = crate::capture::screenshot_display_bytes()?;
+ Ok::<_, anyhow::Error>(crate::overlay_capture::verify_hidden_capture(
+ png, hidden,
+ ))
+ },
+ )?;
let (native_w, native_h) = crate::capture::png_dimensions_pub(&native_png)?;
// True screen size. On a pure-Wayland session (native backend
// opted in, no X11 DISPLAY) the capture above came from the
@@ -12109,6 +12122,7 @@ impl Tool for GetDesktopStateTool {
written,
windows,
capture_id,
+ overlay_capture,
))
})
.await;
@@ -12124,6 +12138,7 @@ impl Tool for GetDesktopStateTool {
written,
windows,
capture_id,
+ overlay_capture,
))) => {
let frame_scale = if shot_w > 0 {
f64::from(screen_w) / f64::from(shot_w)
@@ -12143,6 +12158,7 @@ impl Tool for GetDesktopStateTool {
"screenshot_mime_type": "image/png",
"windows": windows.iter().map(window_record_json).collect::<Vec<_>>(),
"capture_id": capture_id,
+ "agent_overlay_capture": overlay_capture,
});
if (frame_scale - 1.0).abs() > 0.001 {
// Capped: the uncapped capture is the action frame.
@@ -12153,7 +12169,7 @@ impl Tool for GetDesktopStateTool {
content.push(cua_driver_core::protocol::Content::image_png(b64));
}
let window_lines = desktop_window_lines(&windows, frame_scale);
- let frame_note = if (frame_scale - 1.0).abs() > 0.001 {
+ let mut frame_note = if (frame_scale - 1.0).abs() > 0.001 {
format!(
"; x/y for scope:\"desktop\" actions are pixels of THIS screenshot \
(mapped ×{frame_scale:.2} back to the screen automatically)"
@@ -12161,6 +12177,10 @@ impl Tool for GetDesktopStateTool {
} else {
String::new()
};
+ cursor_overlay::capture_exclusion::append_summary_note(
+ &mut frame_note,
+ &overlay_capture,
+ );
if let Some(path) = written {
structured["screenshot_file_path"] = json!(path);
content.push(cua_driver_core::protocol::Content::text(format!(
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
index c900542748..0c551da514 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
@@ -163,6 +163,7 @@ fn map_overlay_msg(msg: &OverlayMsg) -> Option<WlOverlayCmd> {
cmd: kc.cmd.clone(),
}),
OverlayMsg::Revive(key) => Some(WlOverlayCmd::Revive(key.clone())),
+ OverlayMsg::Wake => None,
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
index b0eea64447..32e1ca6ed8 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
@@ -22,8 +22,11 @@
//!
//! ## Display capture
//!
-//! `screencapture -x <file>` still captures the full main display (unchanged
-//! in this slice).
+//! `screencapture -x <file>` captures the full main display. The agent-facing
+//! desktop capture instead uses a ScreenCaptureKit display filter that leaves
+//! the Driver's cursor overlay window out
+//! ([`screenshot_display_bytes_excluding_overlay`]); `screencapture` remains
+//! its fallback and reports that it could not exclude the overlay.
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
use std::collections::HashMap;
@@ -521,18 +524,28 @@ fn build_window_capture_plan(
///
/// https://developer.apple.com/documentation/screencapturekit/scscreenshotmanager
fn capture_window_from_plan(window_id: u32, plan: &WindowCapturePlan) -> anyhow::Result<Vec<u8>> {
- use screencapturekit::screenshot_manager::{CGImageExt, SCScreenshotManager};
+ use screencapturekit::screenshot_manager::SCScreenshotManager;
let image = SCScreenshotManager::capture_image(&plan.filter, &plan.config).map_err(|e| {
anyhow::anyhow!("SCScreenshotManager::capture_image failed for window {window_id}: {e}")
})?;
+ cgimage_to_png(&image, &format!("window {window_id}"))
+}
+
+/// Encode a ScreenCaptureKit `CGImage` as PNG after checking its geometry.
+fn cgimage_to_png(
+ image: &screencapturekit::screenshot_manager::CGImage,
+ label: &str,
+) -> anyhow::Result<Vec<u8>> {
+ use screencapturekit::screenshot_manager::CGImageExt;
+
let w = checked_image_dim(image.width(), "CGImage width")?;
let h = checked_image_dim(image.height(), "CGImage height")?;
let rgba = image
.rgba_data()
- .map_err(|e| anyhow::anyhow!("CGImage::rgba_data failed for window {window_id}: {e}"))?;
+ .map_err(|e| anyhow::anyhow!("CGImage::rgba_data failed for {label}: {e}"))?;
let expected_len = (w as u64)
.checked_mul(h as u64)
@@ -540,7 +553,7 @@ fn capture_window_from_plan(window_id: u32, plan: &WindowCapturePlan) -> anyhow:
.ok_or_else(|| anyhow::anyhow!("RGBA byte length overflow for {w}x{h}"))?;
if rgba.len() as u64 != expected_len {
anyhow::bail!(
- "CGImage RGBA length {} != {w}*{h}*4 ({expected_len}) for window {window_id}",
+ "CGImage RGBA length {} != {w}*{h}*4 ({expected_len}) for {label}",
rgba.len()
);
}
@@ -774,6 +787,141 @@ pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {
Ok(bytes)
}
+/// Name reported in `agent_overlay_capture.method` for the macOS mechanism.
+pub const OVERLAY_EXCLUSION_METHOD: &str = "screencapturekit_excluding_windows";
+
+/// Display captures run on their own single ScreenCaptureKit worker so a
+/// window capture in flight never forces the agent's desktop capture onto the
+/// fallback that cannot exclude the overlay.
+const DISPLAY_CAPTURE_NATIVE_TIMEOUT: Duration = Duration::from_secs(5);
+
+fn display_capture_gate() -> &'static NativeCaptureGate {
+ static GATE: NativeCaptureGate = NativeCaptureGate::new();
+ &GATE
+}
+
+/// macOS adapter of `cursor_overlay::capture_exclusion`: nothing is hidden on
+/// screen; the capture itself leaves the overlay window out through a
+/// ScreenCaptureKit display filter, so the cursor never flickers for the user
+/// and external recorders keep seeing it.
+struct OverlayExcluder;
+
+impl cursor_overlay::capture_exclusion::OverlayCaptureExcluder for OverlayExcluder {
+ type Hidden = u32;
+
+ fn exclude(&self) -> cursor_overlay::capture_exclusion::ExclusionStart<u32> {
+ use cursor_overlay::capture_exclusion::ExclusionStart;
+ match crate::cursor::overlay::overlay_window_id() {
+ Some(window_id) if crate::cursor::overlay::overlay_may_show_pixels() => {
+ ExclusionStart::Excluded {
+ method: OVERLAY_EXCLUSION_METHOD,
+ hidden: window_id,
+ }
+ }
+ // No overlay window, or nothing painted in it: the plain display
+ // capture has no overlay pixels to leave out.
+ _ => ExclusionStart::NotPresent,
+ }
+ }
+
+ fn restore(&self, _window_id: u32) {}
+}
+
+/// Capture the main display for the agent with the Driver's cursor overlay
+/// window left out, and report how that went.
+///
+/// `screencapture` cannot leave a window out, so it is only used when the
+/// overlay shows nothing, or as a fallback that reports `not_excluded`.
+pub fn screenshot_display_bytes_excluding_overlay() -> anyhow::Result<(
+ Vec<u8>,
+ cursor_overlay::capture_exclusion::AgentOverlayCapture,
+)> {
+ use cursor_overlay::capture_exclusion::{capture_excluding_overlays, ResidualCheck};
+ capture_excluding_overlays(&OverlayExcluder, |overlay| {
+ let Some(&overlay_window_id) = overlay else {
+ return Ok((screenshot_display_bytes()?, ResidualCheck::Clean));
+ };
+ match run_native_capture_worker(
+ display_capture_gate(),
+ DISPLAY_CAPTURE_NATIVE_TIMEOUT,
+ move || screenshot_main_display_sck_excluding(overlay_window_id),
+ ) {
+ Ok(bytes) => Ok((bytes, ResidualCheck::Clean)),
+ Err(error) => {
+ tracing::warn!(
+ "ScreenCaptureKit display capture without the overlay failed: {error:#}"
+ );
+ let bytes = screenshot_display_bytes()?;
+ Ok((
+ bytes,
+ ResidualCheck::Residual {
+ reason: format!(
+ "ScreenCaptureKit could not capture the display without the \
+ overlay window ({error:#}); the screencapture fallback includes \
+ every on-screen window"
+ ),
+ },
+ ))
+ }
+ }
+ })
+}
+
+/// Main display at native pixel size through ScreenCaptureKit, with one
+/// window (the overlay) excluded. Requires macOS 14 for
+/// `SCScreenshotManager`.
+fn screenshot_main_display_sck_excluding(excluded_window_id: u32) -> anyhow::Result<Vec<u8>> {
+ use core_graphics::display::CGMainDisplayID;
+ use screencapturekit::prelude::{SCContentFilter, SCShareableContent, SCStreamConfiguration};
+ use screencapturekit::screenshot_manager::SCScreenshotManager;
+
+ let content = SCShareableContent::get()
+ .map_err(|e| anyhow::anyhow!("SCShareableContent::get failed: {e}"))?;
+ // SAFETY: CGMainDisplayID takes no arguments and only reads WindowServer state.
+ let main_display_id = unsafe { CGMainDisplayID() };
+ let display = content
+ .displays()
+ .into_iter()
+ .find(|display| display.display_id() == main_display_id)
+ .ok_or_else(|| {
+ anyhow::anyhow!("ScreenCaptureKit does not list the main display {main_display_id}")
+ })?;
+ let windows = content.windows();
+ let excluded = windows
+ .iter()
+ .find(|window| window.window_id() == excluded_window_id)
+ .ok_or_else(|| {
+ anyhow::anyhow!(
+ "ScreenCaptureKit does not list the overlay window {excluded_window_id}"
+ )
+ })?;
+
+ let mut filter = SCContentFilter::create()
+ .with_display(&display)
+ .with_excluding_windows(&[excluded])
+ .build();
+ // `screencapture` includes the menu bar; keep the same image.
+ filter.set_include_menu_bar(true);
+
+ let scale = f64::from(filter.point_pixel_scale());
+ let rect = filter.content_rect();
+ let (width_pts, height_pts) = if content_rect_usable(rect) {
+ (rect.size.width, rect.size.height)
+ } else {
+ let frame = display.frame();
+ (frame.size.width, frame.size.height)
+ };
+ let config = SCStreamConfiguration::new()
+ .with_width(rounded_pixel_dim(width_pts * scale, "width")?)
+ .with_height(rounded_pixel_dim(height_pts * scale, "height")?)
+ // `screencapture -x` leaves the hardware pointer out too.
+ .with_shows_cursor(false);
+
+ let image = SCScreenshotManager::capture_image(&filter, &config)
+ .map_err(|e| anyhow::anyhow!("SCScreenshotManager::capture_image failed: {e}"))?;
+ cgimage_to_png(&image, "main display")
+}
+
/// Capture the main display and return (base64-encoded PNG, width, height).
pub fn screenshot_display() -> anyhow::Result<(String, u32, u32)> {
let bytes = screenshot_display_bytes()?;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
index 6a32feed0e..8a4c547156 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/cursor/overlay.rs
@@ -31,7 +31,7 @@
use std::collections::HashMap;
use std::ffi::c_void;
-use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
@@ -89,6 +89,53 @@ pub(crate) fn is_overlay_window(window_id: u32) -> bool {
window_id != 0 && OVERLAY_WINDOW_ID.load(Ordering::Acquire) == window_id
}
+/// CGWindowID of the live overlay window, if one is on screen.
+pub(crate) fn overlay_window_id() -> Option<u32> {
+ match OVERLAY_WINDOW_ID.load(Ordering::Acquire) {
+ 0 => None,
+ window_id => Some(window_id),
+ }
+}
+
+/// Whether the last composed frame could hold visible pixels, and when that
+/// was last true. The layer update reaches WindowServer asynchronously, so a
+/// frame that just cleared is still treated as visible for a short grace.
+static OVERLAY_FRAME_PAINTS: AtomicBool = AtomicBool::new(false);
+static OVERLAY_LAST_PAINT: Mutex<Option<Instant>> = Mutex::new(None);
+const OVERLAY_CLEAR_GRACE: Duration = Duration::from_millis(250);
+
+fn note_overlay_frame(paints: bool) {
+ OVERLAY_FRAME_PAINTS.store(paints, Ordering::Release);
+ if paints {
+ if let Ok(mut last) = OVERLAY_LAST_PAINT.lock() {
+ *last = Some(Instant::now());
+ }
+ }
+}
+
+/// Whether overlay pixels may be on screen right now: a cursor, its session
+/// pill, a fade, or a focus rect. Desktop captures only need to exclude the
+/// overlay window when this is true.
+pub(crate) fn overlay_may_show_pixels() -> bool {
+ if OVERLAY_FRAME_PAINTS.load(Ordering::Acquire) {
+ return true;
+ }
+ OVERLAY_LAST_PAINT
+ .lock()
+ .ok()
+ .and_then(|last| *last)
+ .is_some_and(|last| last.elapsed() < OVERLAY_CLEAR_GRACE)
+}
+
+fn cursor_may_paint(state: &RenderState) -> bool {
+ state.focus_rect.is_some()
+ || (state.core.cfg.enabled
+ && state.core.visible
+ && state.core.idle_alpha > 0.0
+ && state.core.pos.0 > -50.0
+ && state.core.pos.1 > -50.0)
+}
+
/// Screen-global geometry kept beside the shared keyed render map
/// ([`cursor_overlay::RenderMap`], which owns the per-session lifecycle:
/// lazy creation, stable z-order, tombstones, revival, and the default guard).
@@ -799,6 +846,7 @@ fn render_loop(
let mut pm = tiny_skia::Pixmap::new(w.max(1), h.max(1))
.unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap());
let backing_scale_f32 = scale as f32;
+ note_overlay_frame(map.cursors.values().any(cursor_may_paint));
for (_k, rs) in &map.cursors {
let focus = rs.focus_rect.map(|rect| FocusRect {
rect,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
index 99cbe2a3a5..1bc64766b6 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
@@ -88,11 +88,21 @@ impl Tool for GetDesktopStateTool {
};
// Capture the FULL display at native size — no resize. Run the
- // blocking screencapture subprocess off the async runtime.
+ // blocking display capture off the async runtime.
let out_file = screenshot_out_file.clone();
let res = tokio::task::spawn_blocking(
- move || -> anyhow::Result<(Vec<u8>, Option<String>, u32, u32, (u32, u32))> {
- let png = crate::capture::screenshot_display_bytes()?;
+ move || -> anyhow::Result<(
+ Vec<u8>,
+ Option<String>,
+ u32,
+ u32,
+ (u32, u32),
+ cursor_overlay::capture_exclusion::AgentOverlayCapture,
+ )> {
+ // The agent reads this image, so the Driver's own cursor and
+ // session pill must not sit over the controls it is reading.
+ let (png, overlay_capture) =
+ crate::capture::screenshot_display_bytes_excluding_overlay()?;
let full = crate::capture::png_dimensions(&png)?;
// Opt-in cap; later desktop-scope pixels from the capped
// image are mapped back at dispatch and by its capture_id.
@@ -105,19 +115,20 @@ impl Tool for GetDesktopStateTool {
let (w, h) = crate::capture::png_dimensions(&png)?;
if let Some(ref path) = out_file {
std::fs::write(path, &png)?;
- Ok((png, Some(path.clone()), w, h, full))
+ Ok((png, Some(path.clone()), w, h, full, overlay_capture))
} else {
- Ok((png, None, w, h, full))
+ Ok((png, None, w, h, full, overlay_capture))
}
},
)
.await;
- let (png, file_path, screenshot_width, screenshot_height, full_size) = match res {
- Ok(Ok(v)) => v,
- Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),
- Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),
- };
+ let (png, file_path, screenshot_width, screenshot_height, full_size, overlay_capture) =
+ match res {
+ Ok(Ok(v)) => v,
+ Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),
+ Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),
+ };
let native_width = match u32::try_from(screen_width) {
Ok(width) => width,
@@ -142,10 +153,11 @@ impl Tool for GetDesktopStateTool {
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
content.push(Content::image_png(BASE64.encode(&png)));
}
- let summary = format!(
+ let mut summary = format!(
"desktop screenshot {screenshot_width}x{screenshot_height} px \
(screen {screen_width}x{screen_height} pts @ {scale_factor}x)"
);
+ cursor_overlay::capture_exclusion::append_summary_note(&mut summary, &overlay_capture);
content.push(Content::text(summary));
let mut structured = serde_json::json!({
@@ -158,6 +170,7 @@ impl Tool for GetDesktopStateTool {
"scale_factor": scale_factor,
"screenshot_mime_type": "image/png",
"capture_id": capture_id,
+ "agent_overlay_capture": overlay_capture,
});
if full_size != (screenshot_width, screenshot_height) {
structured["screenshot_original_width"] = serde_json::json!(full_size.0);
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs
index eb2e52c298..2941adb5ac 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs
@@ -742,6 +742,18 @@ pub fn screenshot_display_bytes() -> Result<Vec<u8>> {
}
}
+/// Capture the primary display for the agent, with the Driver's own cursor
+/// overlay kept out of the pixels, and report how that went.
+pub fn screenshot_display_bytes_excluding_overlay() -> Result<(
+ Vec<u8>,
+ cursor_overlay::capture_exclusion::AgentOverlayCapture,
+)> {
+ use cursor_overlay::capture_exclusion::{capture_excluding_overlays, ResidualCheck};
+ capture_excluding_overlays(&crate::overlay::CaptureExcluder, |_| {
+ Ok((screenshot_display_bytes()?, ResidualCheck::Clean))
+ })
+}
+
/// Capture primary display, returning (base64_png, width, height).
pub fn screenshot_display() -> Result<(String, u32, u32)> {
let png_bytes = screenshot_display_bytes()?;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs
index 800749fd22..b7dfcd7c8c 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs
@@ -596,7 +596,7 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMs
set_timer_resolution_raised(true);
// Store hwnd and rx globally for the wnd_proc callback.
- OVERLAY_HWND.store(hwnd.0 as isize, std::sync::atomic::Ordering::Relaxed);
+ OVERLAY_HWND.store(hwnd.0 as isize, std::sync::atomic::Ordering::Release);
*CMD_RX_WIN.lock().unwrap() = Some(rx);
LAST_ZTICK.store(0, std::sync::atomic::Ordering::Relaxed);
let _ = Z_ORDER.set(WinZOrderEnforcer {
@@ -1397,6 +1397,93 @@ impl ZOrderEnforcer for WinZOrderEnforcer {
}
}
+// ── Capture exclusion (Windows adapter of cursor_overlay::capture_exclusion) ─
+
+/// Name reported in `agent_overlay_capture.method` for the Windows mechanism.
+pub const CAPTURE_EXCLUSION_METHOD: &str = "win32_display_affinity_exclude_from_capture";
+
+/// Keeps the overlay out of the Driver's own desktop BitBlt.
+///
+/// `WDA_EXCLUDEFROMCAPTURE` removes the window from every capture path DWM
+/// serves — GDI `BitBlt` of the screen DC included — while it stays on the
+/// monitor. It is applied only for the duration of one Driver desktop capture:
+/// a permanent affinity would also hide the cursor from screen recorders and
+/// from the Driver's own recordings, which must keep showing it.
+///
+/// The affinity is Windows 10 2004+ only. Older builds reject it; the capture
+/// then runs anyway and reports `not_excluded` with the system error.
+pub struct CaptureExcluder;
+
+impl cursor_overlay::capture_exclusion::OverlayCaptureExcluder for CaptureExcluder {
+ type Hidden = isize;
+
+ fn exclude(&self) -> cursor_overlay::capture_exclusion::ExclusionStart<isize> {
+ use cursor_overlay::capture_exclusion::ExclusionStart;
+ let hwnd = OVERLAY_HWND.load(std::sync::atomic::Ordering::Acquire);
+ if hwnd == 0 {
+ return ExclusionStart::NotPresent;
+ }
+ match set_capture_excluded(hwnd, true) {
+ Ok(()) => ExclusionStart::Excluded {
+ method: CAPTURE_EXCLUSION_METHOD,
+ hidden: hwnd,
+ },
+ Err(reason) => ExclusionStart::Unsupported { reason },
+ }
+ }
+
+ fn restore(&self, hwnd: isize) {
+ if let Err(error) = set_capture_excluded(hwnd, false) {
+ tracing::warn!("Win32 overlay: could not restore capture visibility: {error}");
+ }
+ }
+}
+
+/// Apply or clear `WDA_EXCLUDEFROMCAPTURE` and wait for DWM to compose a frame
+/// with the new affinity, so the very next screen read already honors it.
+/// `SetWindowDisplayAffinity` accepts any window of the calling process, not
+/// only windows of the calling thread.
+#[cfg(target_os = "windows")]
+fn set_capture_excluded(hwnd_isize: isize, excluded: bool) -> Result<(), String> {
+ use windows::Win32::Foundation::HWND;
+ use windows::Win32::Graphics::Dwm::DwmFlush;
+ use windows::Win32::UI::WindowsAndMessaging::{
+ SetWindowDisplayAffinity, WDA_EXCLUDEFROMCAPTURE, WDA_NONE,
+ };
+ let hwnd = HWND(hwnd_isize as *mut _);
+ let affinity = if excluded {
+ WDA_EXCLUDEFROMCAPTURE
+ } else {
+ WDA_NONE
+ };
+ unsafe { SetWindowDisplayAffinity(hwnd, affinity) }.map_err(|error| {
+ format!(
+ "SetWindowDisplayAffinity({}) failed on the agent cursor overlay: {error}; \
+ excluding it from desktop captures requires Windows 10 version 2004 or later",
+ if excluded {
+ "WDA_EXCLUDEFROMCAPTURE"
+ } else {
+ "WDA_NONE"
+ }
+ )
+ })?;
+ if excluded {
+ // One flush can land on a composition pass that started before the
+ // affinity changed; the second is the first frame composed after it.
+ for _ in 0..2 {
+ if unsafe { DwmFlush() }.is_err() {
+ break;
+ }
+ }
+ }
+ Ok(())
+}
+
+#[cfg(not(target_os = "windows"))]
+fn set_capture_excluded(_hwnd_isize: isize, _excluded: bool) -> Result<(), String> {
+ Err("window display affinity is only available on Windows".into())
+}
+
// ── Headless unit tests for the Windows adapter ───────────────────────────
//
// The keyed lifecycle (ownership, removal, default guard, tombstone, revival,
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 7ca5f8a66e..483bd2cd98 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -7786,9 +7786,12 @@ impl Tool for GetDesktopStateTool {
// blocking GDI capture off the async runtime.
let out_file = screenshot_out_file.clone();
let res = tokio::task::spawn_blocking(
- move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32, Vec<u8>, (u32, u32))> {
+ move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32, Vec<u8>, (u32, u32), cursor_overlay::capture_exclusion::AgentOverlayCapture)> {
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
- let png = crate::capture::screenshot_display_bytes()?;
+ // The agent reads this image, so the Driver's own cursor and
+ // session pill must not sit over the controls it is reading.
+ let (png, overlay_capture) =
+ crate::capture::screenshot_display_bytes_excluding_overlay()?;
let full = crate::capture::png_dimensions_pub(&png)?;
// Opt-in cap; later desktop-scope pixels from the capped
// image are mapped back at dispatch and by its capture_id.
@@ -7801,15 +7804,23 @@ impl Tool for GetDesktopStateTool {
let (w, h) = crate::capture::png_dimensions_pub(&png)?;
if let Some(ref path) = out_file {
std::fs::write(path, &png)?;
- Ok((None, Some(path.clone()), w, h, png, full))
+ Ok((None, Some(path.clone()), w, h, png, full, overlay_capture))
} else {
- Ok((Some(BASE64.encode(&png)), None, w, h, png, full))
+ Ok((Some(BASE64.encode(&png)), None, w, h, png, full, overlay_capture))
}
},
)
.await;
- let (b64_opt, file_path, screenshot_width, screenshot_height, png, full_size) = match res {
+ let (
+ b64_opt,
+ file_path,
+ screenshot_width,
+ screenshot_height,
+ png,
+ full_size,
+ overlay_capture,
+ ) = match res {
Ok(Ok(v)) => v,
Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),
Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),
@@ -7819,10 +7830,11 @@ impl Tool for GetDesktopStateTool {
if let Some(b64) = b64_opt {
content.push(Content::image_png(b64));
}
- let summary = format!(
+ let mut summary = format!(
"desktop screenshot {screenshot_width}x{screenshot_height} px \
(screen {screen_width}x{screen_height} px)"
);
+ cursor_overlay::capture_exclusion::append_summary_note(&mut summary, &overlay_capture);
content.push(Content::text(summary));
let mut structured = json!({
@@ -7837,6 +7849,7 @@ impl Tool for GetDesktopStateTool {
if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 }
},
"screenshot_mime_type": "image/png",
+ "agent_overlay_capture": overlay_capture,
});
if full_size != (screenshot_width, screenshot_height) {
structured["screenshot_original_width"] = json!(full_size.0);