Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): never send a Windows foreground drag to the window covering the target

injaneity36b6750ffb05
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
index d3721bca2a..9206b20651 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
@@ -108,7 +108,7 @@ Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local
</Tab>
<Tab value="Windows">
-Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. No focus steal. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background, the OS move/resize loop needs real pointer input, so they return background_unavailable; re-issue those with delivery_mode:"foreground". Limits and fallbacks: `get_guide({topic:"drag"})`.
+Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. Background delivery does not steal focus; delivery_mode:"foreground" activates the target and refuses with foreground_unavailable, sending nothing, when another window still covers the start point. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background, the OS move/resize loop needs real pointer input, so they return background_unavailable; re-issue those with delivery_mode:"foreground". Limits and fallbacks: `get_guide({topic:"drag"})`.
**Windows parameters**
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
index ce915e57da..42b812e5d3 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
@@ -49,7 +49,10 @@ use cua_driver_testkit::observer::TargetWindow;
use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel};
use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolResponse};
use windows::Win32::Foundation::HWND;
-use windows::Win32::UI::WindowsAndMessaging::{IsIconic, ShowWindow, SW_MINIMIZE};
+use windows::Win32::UI::WindowsAndMessaging::{
+ GetWindowLongPtrW, IsIconic, SetWindowPos, ShowWindow, GWL_EXSTYLE, HWND_TOPMOST,
+ SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SW_MINIMIZE, WS_EX_TOPMOST,
+};
#[path = "support/foreground_text_oracle.rs"]
mod foreground_text_oracle;
@@ -1787,6 +1790,121 @@ fn harness_wpf_slider_drag_background_refusal() {
});
}
+#[test]
+#[ignore]
+fn harness_wpf_slider_drag_foreground_under_topmost_cover_refuses_without_leak() {
+ // A foreground drag is coordinate-routed SendInput, so Windows delivers it
+ // to whatever window owns the start point. Cover the harness with an
+ // always-on-top sentinel: activating the harness cannot bring it above
+ // that cover, so the drag must refuse with foreground_unavailable before
+ // the button goes down. Previously the z-order-only raise could fail and
+ // the whole stroke landed in the cover while the tool reported "Sent drag".
+ let mut case = native_foreground_case(
+ "wpf",
+ "slider_drag_topmost_cover",
+ Targeting::Px,
+ DriverRoute::WindowsSendInput,
+ )
+ .expecting_refusal(vec![RefusalCode::ForegroundUnavailable]);
+ case.oracles.push(OracleKind::NoLeakedInput);
+ case.oracles.sort();
+ case.oracles.dedup();
+ run_case(case, |pid, wid, driver| {
+ let before = snapshot(driver, pid, wid);
+ let before_value = fixture_state_line(before.text(), "slider_value=").to_owned();
+ let (x, y, width, height) =
+ pixel_frame(&before, "sld-value", window_bounds(driver, pid, wid));
+
+ let sentinel = ForegroundSentinel::launch(driver);
+ // The sentinel's setup goes through bring_to_front, which leaves it
+ // non-topmost. Pin it above every normal window so activation of the
+ // harness cannot uncover the slider, and prove the pin took effect.
+ let cover = HWND(sentinel.target().native_id as usize as *mut _);
+ unsafe {
+ SetWindowPos(
+ cover,
+ Some(HWND_TOPMOST),
+ 0,
+ 0,
+ 0,
+ 0,
+ SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE,
+ )
+ .expect("pin the sentinel above normal windows");
+ assert_ne!(
+ GetWindowLongPtrW(cover, GWL_EXSTYLE) as u32 & WS_EX_TOPMOST.0,
+ 0,
+ "the sentinel did not become topmost"
+ );
+ }
+ sentinel
+ .assert_background_posture(TargetWindow {
+ pid,
+ native_id: wid,
+ })
+ .expect("the sentinel must cover the harness before the drag");
+ driver.start_behavior_recording();
+ let response = driver.call(
+ "drag",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "from_x": x + width * 0.05, "from_y": y + height / 2.0,
+ "to_x": x + width * 0.90, "to_y": y + height / 2.0,
+ "duration_ms": 700, "steps": 40,
+ "delivery_mode": "foreground"
+ }),
+ );
+ // Activating the harness may take focus from the sentinel; that is part
+ // of a foreground attempt, so only the leaked-input oracle applies. A
+ // silent journal proves nothing unless the sentinel was still alive.
+ let (sentinel_passed, violations) = sentinel.observe();
+ assert!(
+ !violations
+ .iter()
+ .any(|violation| violation.contains("heartbeat stopped")),
+ "the sentinel stopped journaling, so its leak oracle is void: {violations:?}"
+ );
+ let leaked = violations
+ .iter()
+ .filter(|violation| violation.contains("received input events"))
+ .collect::<Vec<_>>();
+ assert!(
+ leaked.is_empty(),
+ "foreground drag leaked into the covering window: {leaked:?}; tool said: {}",
+ response.text()
+ );
+ assert!(
+ response.is_error(),
+ "foreground drag under a topmost cover reported delivery: {}",
+ response.text()
+ );
+ assert_eq!(
+ response.structured()["code"].as_str(),
+ Some("foreground_unavailable"),
+ "foreground drag under a topmost cover returned the wrong refusal: {}",
+ response.text()
+ );
+
+ std::thread::sleep(Duration::from_millis(200));
+ let after = snapshot(driver, pid, wid);
+ assert_eq!(
+ fixture_state_line(after.text(), "slider_value="),
+ before_value,
+ "refused foreground drag changed the slider value"
+ );
+ let mut passed = vec![OracleKind::FixtureState];
+ if sentinel_passed.contains(&OracleKind::NoLeakedInput) {
+ passed.push(OracleKind::NoLeakedInput);
+ }
+ Observation::refused(
+ RefusalCode::ForegroundUnavailable,
+ passed,
+ response.text(),
+ Evidence::default(),
+ )
+ });
+}
+
#[test]
#[ignore]
fn harness_wpf_slider_increase_large() {
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
index c97a244e3f..f7cbfea4f7 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
@@ -318,6 +318,7 @@ pub enum RefusalCode {
BrowserActionUnavailable,
ModifiedPointerUnavailable,
WindowMinimized,
+ ForegroundUnavailable,
}
impl RefusalCode {
@@ -344,6 +345,7 @@ impl RefusalCode {
"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),
"modified_pointer_unavailable" => Some(Self::ModifiedPointerUnavailable),
"window_minimized" => Some(Self::WindowMinimized),
+ "foreground_unavailable" => Some(Self::ForegroundUnavailable),
_ => None,
}
}
@@ -589,28 +591,37 @@ impl CaseSpec {
let desktop_modifier_refusal = self.delivery == Delivery::Foreground
&& self.scope == Scope::Desktop
&& allowed_codes == &[RefusalCode::ModifiedPointerUnavailable];
+ // A coordinate-routed foreground pointer action whose target cannot
+ // be brought over a covering window must refuse before dispatch.
+ // The covering window's journal proves nothing leaked into it.
+ let covered_target_refusal = self.delivery == Delivery::Foreground
+ && self.scope == Scope::Window
+ && self.driver_route == DriverRoute::WindowsSendInput
+ && allowed_codes == &[RefusalCode::ForegroundUnavailable];
if self.delivery != Delivery::Background
&& !exact_activation_refusal
&& !desktop_modifier_refusal
+ && !covered_target_refusal
{
return Err(format!(
- "{}: only background delivery, exact-window activation, or a desktop modified-pointer limitation may declare refusal",
+ "{}: only background delivery, exact-window activation, a covered foreground target, or a desktop modified-pointer limitation may declare refusal",
self.cell_id
));
}
if allowed_codes.is_empty() {
return Err(format!("{}: refusal has no allowed code", self.cell_id));
}
- let required_oracles: &[OracleKind] =
- if exact_activation_refusal || desktop_modifier_refusal {
- &[OracleKind::FixtureState]
- } else {
- &[
- OracleKind::Focus,
- OracleKind::ZOrder,
- OracleKind::NoLeakedInput,
- ]
- };
+ let required_oracles: &[OracleKind] = if covered_target_refusal {
+ &[OracleKind::FixtureState, OracleKind::NoLeakedInput]
+ } else if exact_activation_refusal || desktop_modifier_refusal {
+ &[OracleKind::FixtureState]
+ } else {
+ &[
+ OracleKind::Focus,
+ OracleKind::ZOrder,
+ OracleKind::NoLeakedInput,
+ ]
+ };
for required in required_oracles {
if !self.oracles.contains(required) {
return Err(format!(
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
index 6b75d8ac31..58156458c8 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
@@ -374,7 +374,8 @@ pub fn inject_click_screen(
/// rather than an occluder. Used to enforce the macOS-aligned rule that a
/// `background` actuation never raises: when this is false the injection bails
/// and the tool returns `background_unavailable` (escalate to foreground).
-unsafe fn target_visible_at_point(target: HWND, sx: i32, sy: i32) -> bool {
+/// The foreground SendInput drag uses the same check before its button-down.
+pub(crate) unsafe fn target_visible_at_point(target: HWND, sx: i32, sy: i32) -> bool {
let top = WindowFromPoint(POINT { x: sx, y: sy });
if top.0.is_null() {
return false;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
index cbde68d8db..7dcbcb6a09 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
@@ -20,9 +20,9 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{
};
use windows::Win32::UI::WindowsAndMessaging::{
ChildWindowFromPointEx, GetAncestor, GetClassLongPtrW, GetCursorPos, GetForegroundWindow,
- GetSystemMetrics, GetWindowLongPtrW, PostMessageW, SetCursorPos, SetWindowPos, CS_DBLCLKS,
- CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT, GA_ROOT, GCL_STYLE, GWL_EXSTYLE,
- HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN,
+ GetSystemMetrics, GetWindowLongPtrW, PostMessageW, SetCursorPos, SetWindowPos, WindowFromPoint,
+ CS_DBLCLKS, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT, GA_ROOT, GCL_STYLE,
+ GWL_EXSTYLE, HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN,
SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, WM_LBUTTONDBLCLK,
WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDBLCLK, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_MOUSEMOVE,
WM_RBUTTONDBLCLK, WM_RBUTTONDOWN, WM_RBUTTONUP, WS_EX_TOPMOST,
@@ -773,9 +773,9 @@ fn send_click_synthesized_mods_impl(
Ok(())
}
-/// Press-hold-move-release drag via `SendInput`. Companion to
-/// [`send_click_synthesized`] for the `drag` tool's `delivery_mode:"foreground"`
-/// path.
+/// Press-hold-move-release drag via `SendInput`, used by the `drag` tool's
+/// `scope:"desktop"` path, which intentionally acts on whatever is visible at
+/// the screen coordinates.
///
/// Why a SendInput drag is needed at all: the PostMessage drag path posts
/// `WM_LBUTTONDOWN` + `WM_MOUSEMOVE`s + `WM_LBUTTONUP` to the target HWND.
@@ -800,6 +800,69 @@ pub fn send_drag_synthesized(
duration_ms: u64,
steps: usize,
button: &str,
+) -> Result<()> {
+ send_drag_synthesized_impl(
+ target,
+ sx_from,
+ sy_from,
+ sx_to,
+ sy_to,
+ duration_ms,
+ steps,
+ button,
+ false,
+ )
+}
+
+/// SendInput drag for an explicit window-targeted foreground request.
+///
+/// The drag is coordinate-routed: Windows delivers it to whatever window is
+/// under the pointer. The z-order-only raise used by the desktop path can
+/// silently fail to expose the target, and the whole stroke then lands in the
+/// window covering it. Like [`send_click_synthesized_active_mods`], this path
+/// activates the target instead, and it hit-tests the start point before the
+/// button goes down. When Windows refuses the activation, or another window
+/// still owns the start point, it returns `foreground_unavailable` without
+/// sending any mouse input. The caller owns any later foreground restore.
+pub fn send_drag_synthesized_active(
+ target: u64,
+ sx_from: i32,
+ sy_from: i32,
+ sx_to: i32,
+ sy_to: i32,
+ duration_ms: u64,
+ steps: usize,
+ button: &str,
+) -> Result<()> {
+ send_drag_synthesized_impl(
+ target,
+ sx_from,
+ sy_from,
+ sx_to,
+ sy_to,
+ duration_ms,
+ steps,
+ button,
+ true,
+ )
+}
+
+/// How long the active drag waits for the activated target to own its start
+/// point. Activation reorders windows synchronously in the common case; the
+/// short poll covers targets whose activation is still settling.
+const DRAG_START_HIT_TEST_TIMEOUT: Duration = Duration::from_millis(150);
+
+#[allow(clippy::too_many_arguments)]
+fn send_drag_synthesized_impl(
+ target: u64,
+ sx_from: i32,
+ sy_from: i32,
+ sx_to: i32,
+ sy_to: i32,
+ duration_ms: u64,
+ steps: usize,
+ button: &str,
+ activate: bool,
) -> Result<()> {
let target = HWND(target as *mut _);
if target.0.is_null() {
@@ -853,24 +916,93 @@ pub fn send_drag_synthesized(
let prev_fg = GetForegroundWindow();
let mut prev_cursor = POINT::default();
let _ = GetCursorPos(&mut prev_cursor);
-
- // Lock-free z-order raise (no focus steal) so the coordinate-routed drag
- // lands on the target — same technique as send_click_synthesized.
- // SetForegroundWindow is lock-denied without UIAccess and isn't needed
- // for pointer input; NoActivateGuard keeps the press from activating it.
- let _noact = crate::input::NoActivateGuard::arm(target);
// Capture whether the target was ALREADY always-on-top so we only demote
// below if WE promoted it (else we'd strip a legitimate topmost window).
let was_topmost = (GetWindowLongPtrW(target, GWL_EXSTYLE) as u32) & WS_EX_TOPMOST.0 != 0;
- let _ = SetWindowPos(
- target,
- HWND_TOPMOST,
- 0,
- 0,
- 0,
- 0,
- SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
- );
+
+ // Undo the desktop path's z-order raise and put the user's cursor back.
+ // The active path leaves z-order alone; its caller restores foreground.
+ // The cursor stays put when the release is unconfirmed: moving a cursor
+ // whose button may still be held would continue the drag.
+ let restore = |move_cursor: bool| {
+ if !activate {
+ if !was_topmost {
+ let _ = SetWindowPos(
+ target,
+ HWND_NOTOPMOST,
+ 0,
+ 0,
+ 0,
+ 0,
+ SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
+ );
+ }
+ if !prev_fg.0.is_null() && prev_fg != target {
+ let _ = SetWindowPos(
+ prev_fg,
+ HWND_TOP,
+ 0,
+ 0,
+ 0,
+ 0,
+ SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
+ );
+ }
+ }
+ if move_cursor {
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ }
+ };
+
+ let noactivate = if activate {
+ if !crate::input::force_foreground_assisted(target).0 {
+ let actual = GetForegroundWindow();
+ bail!(
+ "foreground_unavailable: Windows did not activate exact target HWND {:?} \
+ (actual foreground HWND {:?}); no mouse input was sent",
+ target.0,
+ actual.0
+ );
+ }
+ None
+ } else {
+ // Lock-free z-order raise (no focus steal) so the coordinate-routed
+ // drag lands on the target — same technique as send_click_synthesized.
+ // SetForegroundWindow is lock-denied without UIAccess and isn't needed
+ // for pointer input; NoActivateGuard keeps the press from activating it.
+ let guard = crate::input::NoActivateGuard::arm(target);
+ let _ = SetWindowPos(
+ target,
+ HWND_TOPMOST,
+ 0,
+ 0,
+ 0,
+ 0,
+ SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
+ );
+ Some(guard)
+ };
+
+ // A coordinate-routed press goes to whichever window owns the start
+ // point. Refuse before the button goes down unless that is the target.
+ if activate {
+ let deadline = std::time::Instant::now() + DRAG_START_HIT_TEST_TIMEOUT;
+ while !crate::input::inject::target_visible_at_point(target, sx_from, sy_from) {
+ if std::time::Instant::now() >= deadline {
+ let owner = WindowFromPoint(POINT {
+ x: sx_from,
+ y: sy_from,
+ });
+ bail!(
+ "foreground_unavailable: the drag start ({sx_from},{sy_from}) is covered \
+ by HWND {:?}, not target HWND {:?}; no mouse input was sent",
+ GetAncestor(owner, GA_ROOT).0,
+ target.0
+ );
+ }
+ sleep(Duration::from_millis(15));
+ }
+ }
// 1. Move + press at the start of the drag.
let (nfx, nfy) = norm(sx_from, sy_from);
@@ -881,29 +1013,9 @@ pub fn send_drag_synthesized(
];
let sent = SendInput(&prelude, std::mem::size_of::<INPUT>() as i32);
if sent as usize != prelude.len() {
- if !was_topmost {
- let _ = SetWindowPos(
- target,
- HWND_NOTOPMOST,
- 0,
- 0,
- 0,
- 0,
- SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
- );
- }
- if !prev_fg.0.is_null() && prev_fg != target {
- let _ = SetWindowPos(
- prev_fg,
- HWND_TOP,
- 0,
- 0,
- 0,
- 0,
- SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
- );
- }
- let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ // Records insert in order, so a short prelude never pressed the button.
+ restore(true);
+ drop(noactivate);
bail!(
"SendInput drag-prelude inserted {sent}/{} events",
prelude.len()
@@ -914,51 +1026,49 @@ pub fn send_drag_synthesized(
// so both the visible cursor and the system input queue track the
// same path — WPF's drag-handler watches GetKeyState during each
// move event.
+ let mut moves_ok = true;
+ let mut cursor_at = (sx_from, sy_from);
for i in 1..=steps {
let t = i as f64 / steps as f64;
let x = sx_from + ((sx_to - sx_from) as f64 * t).round() as i32;
let y = sy_from + ((sy_to - sy_from) as f64 * t).round() as i32;
let (nx, ny) = norm(x, y);
let _ = SetCursorPos(x, y);
+ cursor_at = (x, y);
let mv = [make_input(nx, ny, MOUSEEVENTF_MOVE)];
- let _ = SendInput(&mv, std::mem::size_of::<INPUT>() as i32);
+ if SendInput(&mv, std::mem::size_of::<INPUT>() as i32) != 1 {
+ moves_ok = false;
+ break;
+ }
if step_delay_ms > 0 {
sleep(Duration::from_millis(step_delay_ms));
}
}
- // 3. Release at the end.
+ // 3. Release at the end. The button is held at this point, so always
+ // attempt the release. A second attempt is a best-effort recovery;
+ // when it lands, the drag completed normally.
let (ntx, nty) = norm(sx_to, sy_to);
let release = [make_input(ntx, nty, up_flag)];
- let _ = SendInput(&release, std::mem::size_of::<INPUT>() as i32);
+ let released = SendInput(&release, std::mem::size_of::<INPUT>() as i32) == 1
+ || SendInput(&release, std::mem::size_of::<INPUT>() as i32) == 1;
// Brief settle, then restore z-order (demote target, restack user's
// window — no activation) and the cursor.
sleep(Duration::from_millis(40));
- if !was_topmost {
- let _ = SetWindowPos(
- target,
- HWND_NOTOPMOST,
- 0,
- 0,
- 0,
- 0,
- SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
+ restore(released);
+ drop(noactivate);
+
+ if !released {
+ let (cx, cy) = cursor_at;
+ bail!(
+ "SendInput did not insert the drag's button release after a retry; the \
+ button may still be held at screen ({cx},{cy}), so the cursor was left there."
);
}
- if !prev_fg.0.is_null() && prev_fg != target {
- let _ = SetWindowPos(
- prev_fg,
- HWND_TOP,
- 0,
- 0,
- 0,
- 0,
- SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
- );
+ if !moves_ok {
+ bail!("SendInput inserted fewer drag moves than expected; the button was released.");
}
- let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
- drop(_noact);
}
Ok(())
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 0ecc1cb90b..896e77dd0e 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -7763,7 +7763,9 @@ impl Tool for DragTool {
name: "drag".into(),
description: "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. \
duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate \
- WM_MOUSEMOVE events along the path. No focus steal. After a zoom call, pass from_zoom=true. \
+ WM_MOUSEMOVE events along the path. Background delivery does not steal focus; delivery_mode:\"foreground\" \
+ activates the target and refuses with foreground_unavailable, sending nothing, when another window still \
+ covers the start point. After a zoom call, pass from_zoom=true. \
Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the \
window itself) cannot be delivered in the background — the OS move/resize loop needs real pointer \
input — so they return background_unavailable; re-issue those with delivery_mode:\"foreground\".".into(),
@@ -7993,8 +7995,8 @@ impl Tool for DragTool {
// Slider thumbs (and any framework that polls GetKeyState during
// its drag handler — PostMessage doesn't update per-thread input
// state, so those targets see a button-up world during the drag
- // and never start tracking). Subject to the same UIAccess
- // foreground-lock constraint as send_click_synthesized.
+ // and never start tracking). Like the foreground click, it activates
+ // the target and refuses rather than drag in a window that covers it.
if delivery == DeliveryMode::Foreground {
let btn_fg = button.clone();
pin_overlay_above(&cursor_key, hwnd);
@@ -8002,7 +8004,7 @@ impl Tool for DragTool {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
let send_result = tokio::task::spawn_blocking(move || {
- crate::input::mouse::send_drag_synthesized(
+ crate::input::mouse::send_drag_synthesized_active(
hwnd,
sx_from,
sy_from,
@@ -8033,7 +8035,21 @@ impl Tool for DragTool {
from screen ({sx_from},{sy_from}) → ({sx_to},{sy_to}) \
in {duration_ms}ms / {steps} steps (delivery_mode:foreground)."
)),
- Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Ok(Err(e)) => {
+ let message = e.to_string();
+ if message.starts_with("foreground_unavailable:") {
+ // Nothing was sent; give callers the structured code
+ // and its recovery hint rather than bare text.
+ ToolResult::error(message).with_structured(json!({
+ "code": "foreground_unavailable",
+ "effect": "refused",
+ "pid": raw_pid,
+ "window_id": hwnd,
+ }))
+ } else {
+ ToolResult::error(message)
+ }
+ }
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
index cb4614fc22..dc632e3e2c 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
@@ -764,7 +764,7 @@
"read_only": false
},
{
- "description": "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. No focus steal. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background — the OS move/resize loop needs real pointer input — so they return background_unavailable; re-issue those with delivery_mode:\"foreground\". Limits and fallbacks: `get_guide({topic:\"drag\"})`.",
+ "description": "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. Background delivery does not steal focus; delivery_mode:\"foreground\" activates the target and refuses with foreground_unavailable, sending nothing, when another window still covers the start point. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background — the OS move/resize loop needs real pointer input — so they return background_unavailable; re-issue those with delivery_mode:\"foreground\". Limits and fallbacks: `get_guide({topic:\"drag\"})`.",
"destructive": true,
"idempotent": false,
"input_schema": {