Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): honor modifiers during Windows drags (#5667)

Sida Chengda23319e6526
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
index 4b961f659e..39775c6ed7 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
@@ -109,7 +109,7 @@ Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local
</Tab>
<Tab value="Windows">
-Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. Background delivery does not steal focus; delivery_mode:"foreground" activates the target and refuses with foreground_unavailable, sending nothing, when another window still covers the start point. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background, the OS move/resize loop needs real pointer input, so they return background_unavailable; re-issue those with delivery_mode:"foreground". Limits and fallbacks: `get_guide({topic:"drag"})`.
+Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. Background delivery does not steal focus; delivery_mode:"foreground" activates the target and refuses with foreground_unavailable, sending nothing, when another window still covers the start point. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background, the OS move/resize loop needs real pointer input, so they return background_unavailable; re-issue those with delivery_mode:"foreground". `modifier` holds keys from press to release in delivery_mode:"foreground" and scope:"desktop" via SendInput. Background drags with modifiers return background_unavailable because posted messages and pen injection cannot carry live modifier-key state. Limits and fallbacks: `get_guide({topic:"drag"})`.
**Windows parameters**
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
index 693e08d63b..359aed0408 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs
@@ -1769,6 +1769,93 @@ fn harness_wpf_slider_drag() {
);
}
+#[test]
+#[ignore]
+fn harness_wpf_slider_drag_holds_modifier() {
+ // A foreground drag holds `modifier` keys through SendInput from press to
+ // release. The fixture records WPF's Keyboard.Modifiers (live GetKeyState)
+ // on each thumb move, and a plain F5 afterwards still matches its
+ // unmodified binding, so Ctrl was released rather than latched. The drag
+ // grabs the thumb itself, which at value 0 spans from the slider's left
+ // edge to the IncreaseLarge track button; a press on the track would only
+ // step the value.
+ run_foreground_case(
+ "slider_drag_modifier",
+ Targeting::Px,
+ DriverRoute::WindowsSendInput,
+ Vec::new(),
+ |pid, wid, driver| {
+ focus_harness(driver, pid, wid);
+ let pre = snapshot(driver, pid, wid);
+ assert!(
+ pre.text().contains("slider_value=0") && pre.text().contains("accel_fired=0"),
+ "fixture did not start at slider_value=0 and accel_fired=0"
+ );
+ let bounds = window_bounds(driver, pid, wid);
+ let (x, y, width, height) = pixel_frame(&pre, "sld-value", bounds);
+ let (track_x, ..) = pixel_frame(&pre, "IncreaseLarge", bounds);
+
+ let drag = driver.call(
+ "drag",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "from_x": (x + track_x) / 2.0, "from_y": y + height / 2.0,
+ "to_x": x + width * 0.75, "to_y": y + height / 2.0,
+ "duration_ms": 700, "steps": 40,
+ "modifier": ["ctrl"],
+ "delivery_mode": "foreground"
+ }),
+ );
+ assert!(
+ !drag.is_error(),
+ "modified foreground drag failed: {}",
+ drag.text()
+ );
+ let deadline = Instant::now() + Duration::from_secs(3);
+ loop {
+ let post = snapshot(driver, pid, wid);
+ let line = fixture_state_line(post.text(), "slider_value=");
+ // The suffix appears only once the thumb moves; a value past
+ // the first steps shows the thumb followed the pointer.
+ let value = line
+ .split("slider_value=")
+ .nth(1)
+ .and_then(|rest| rest.split(' ').next())
+ .and_then(|value| value.parse::<u32>().ok())
+ .unwrap_or(0);
+ if value >= 50 && line.contains("modifiers=Control") {
+ println!("drag with ctrl observed: {}", line.trim());
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "WPF did not observe a thumb drag with Ctrl: {line}; tool said: {}",
+ drag.text()
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
+
+ let key = driver.call(
+ "press_key",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "key": "f5",
+ "delivery_mode": "foreground"
+ }),
+ );
+ assert!(!key.is_error(), "post-drag plain F5 failed: {}", key.text());
+ std::thread::sleep(Duration::from_millis(300));
+ let post = snapshot(driver, pid, wid);
+ assert!(
+ post.text().contains("accel_fired=1"),
+ "plain F5 did not match after the modified drag; Ctrl may be latched: {}",
+ snapshot_lines_containing(post.text(), &["accel_fired"])
+ );
+ println!("✅ harness_wpf_slider_drag_holds_modifier: Ctrl held, then released");
+ Vec::new()
+ },
+ );
+}
+
#[test]
#[ignore]
fn harness_wpf_slider_drag_background_refusal() {
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
index d0cb70c375..3923d75f9a 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
@@ -829,10 +829,16 @@ fn cursor_still_at_injected_click(x: i32, y: i32, click_x: i32, click_y: i32) ->
/// input queue and DOES update GetKeyState, so a WPF Slider thumb actually
/// tracks the drag.
///
+/// `modifiers` (ctrl/shift/alt/win) are held through the system input queue
+/// from before the press until after the release, so targets that read live
+/// key state (WPF `Keyboard.Modifiers`, Excel's Ctrl-drag copy) see a
+/// modified drag.
+///
/// Same UIAccess constraints as [`send_click_synthesized`] — the
/// `SetForegroundWindow` swap is rejected from non-UIAccess processes
/// when foreground-lock is active; route through `cua-driver-uia.exe`
/// for reliable operation.
+#[allow(clippy::too_many_arguments)]
pub fn send_drag_synthesized(
target: u64,
sx_from: i32,
@@ -842,6 +848,7 @@ pub fn send_drag_synthesized(
duration_ms: u64,
steps: usize,
button: &str,
+ modifiers: &[&str],
) -> Result<()> {
send_drag_synthesized_impl(
target,
@@ -852,6 +859,7 @@ pub fn send_drag_synthesized(
duration_ms,
steps,
button,
+ modifiers,
false,
)
}
@@ -866,6 +874,8 @@ pub fn send_drag_synthesized(
/// button goes down. When Windows refuses the activation, or another window
/// still owns the start point, it returns `foreground_unavailable` without
/// sending any mouse input. The caller owns any later foreground restore.
+/// `modifiers` are held as in [`send_drag_synthesized`].
+#[allow(clippy::too_many_arguments)]
pub fn send_drag_synthesized_active(
target: u64,
sx_from: i32,
@@ -875,6 +885,7 @@ pub fn send_drag_synthesized_active(
duration_ms: u64,
steps: usize,
button: &str,
+ modifiers: &[&str],
) -> Result<()> {
send_drag_synthesized_impl(
target,
@@ -885,6 +896,7 @@ pub fn send_drag_synthesized_active(
duration_ms,
steps,
button,
+ modifiers,
true,
)
}
@@ -904,6 +916,7 @@ fn send_drag_synthesized_impl(
duration_ms: u64,
steps: usize,
button: &str,
+ modifiers: &[&str],
activate: bool,
) -> Result<()> {
let target = HWND(target as *mut _);
@@ -919,6 +932,7 @@ fn send_drag_synthesized_impl(
"middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP),
_ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),
};
+ let (mod_downs, mod_ups) = crate::input::keyboard::modifier_hold_inputs(modifiers);
let (vd_x, vd_y, vd_w, vd_h) = unsafe {
(
@@ -1046,6 +1060,30 @@ fn send_drag_synthesized_impl(
}
}
+ // Release the held modifiers, retrying once like the click path so a
+ // partially inserted release does not leave system key state held.
+ let release_modifiers = || {
+ let size = std::mem::size_of::<INPUT>() as i32;
+ mod_ups.is_empty()
+ || SendInput(&mod_ups, size) as usize == mod_ups.len()
+ || SendInput(&mod_ups, size) as usize == mod_ups.len()
+ };
+
+ // 0. Hold the modifiers before the press.
+ if !mod_downs.is_empty() {
+ let sent = SendInput(&mod_downs, std::mem::size_of::<INPUT>() as i32);
+ if sent as usize != mod_downs.len() {
+ let _ = release_modifiers();
+ restore(true);
+ drop(noactivate);
+ bail!(
+ "SendInput inserted {sent}/{} modifier key presses; no mouse input was sent",
+ mod_downs.len()
+ );
+ }
+ sleep(Duration::from_millis(5));
+ }
+
// 1. Move + press at the start of the drag.
let (nfx, nfy) = norm(sx_from, sy_from);
let _ = SetCursorPos(sx_from, sy_from);
@@ -1056,6 +1094,7 @@ fn send_drag_synthesized_impl(
let sent = SendInput(&prelude, std::mem::size_of::<INPUT>() as i32);
if sent as usize != prelude.len() {
// Records insert in order, so a short prelude never pressed the button.
+ let _ = release_modifiers();
restore(true);
drop(noactivate);
bail!(
@@ -1095,9 +1134,12 @@ fn send_drag_synthesized_impl(
let released = SendInput(&release, std::mem::size_of::<INPUT>() as i32) == 1
|| SendInput(&release, std::mem::size_of::<INPUT>() as i32) == 1;
- // Brief settle, then restore z-order (demote target, restack user's
- // window — no activation) and the cursor.
+ // Brief settle, so a target that samples live key state while handling
+ // the button release still sees the modifiers held. Then release them
+ // and restore z-order (demote target, restack user's window — no
+ // activation) and the cursor.
sleep(Duration::from_millis(40));
+ let modifiers_released = release_modifiers();
restore(released);
drop(noactivate);
@@ -1108,6 +1150,12 @@ fn send_drag_synthesized_impl(
button may still be held at screen ({cx},{cy}), so the cursor was left there."
);
}
+ if !modifiers_released {
+ bail!(
+ "SendInput did not insert the drag's modifier key releases after a retry; \
+ modifier keys may still be held."
+ );
+ }
if !moves_ok {
bail!("SendInput inserted fewer drag moves than expected; the button was released.");
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 9c13cb84c7..de6f039b56 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -7812,7 +7812,10 @@ impl Tool for DragTool {
covers the start point. After a zoom call, pass from_zoom=true. \
Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the \
window itself) cannot be delivered in the background — the OS move/resize loop needs real pointer \
- input — so they return background_unavailable; re-issue those with delivery_mode:\"foreground\".".into(),
+ input — so they return background_unavailable; re-issue those with delivery_mode:\"foreground\". \
+ `modifier` holds keys from press to release in delivery_mode:\"foreground\" and scope:\"desktop\" via \
+ SendInput. Background drags with modifiers return background_unavailable because posted messages and \
+ pen injection cannot carry live modifier-key state.".into(),
input_schema: json!({"type":"object","required":["from_x","from_y","to_x","to_y"],"properties":{
"session": cua_driver_core::tool_schema::session_schema(),
"scope":{"type":"string","enum":["window","desktop"],"description":"Use desktop with no pid/window_id for screen-absolute coordinates."},
@@ -7851,11 +7854,13 @@ impl Tool for DragTool {
let duration_ms = input.duration_ms.unwrap_or(500);
let steps = input.steps.unwrap_or(20) as usize;
let button = input.button.unwrap_or(ClickButton::Left).as_str();
+ let modifiers = input.modifier.unwrap_or_default();
let hwnd = match crate::input::mouse::foreground_window() {
Ok(hwnd) => hwnd,
Err(error) => return ToolResult::error(error.to_string()),
};
let native_drag = tokio::task::spawn_blocking(move || {
+ let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();
crate::input::mouse::send_drag_synthesized(
hwnd,
from_x,
@@ -7865,6 +7870,7 @@ impl Tool for DragTool {
duration_ms,
steps,
&button,
+ &modifier_refs,
)
});
let visual_drag = track_overlay_drag(
@@ -7895,6 +7901,21 @@ impl Tool for DragTool {
};
let pid = raw_pid as u32;
let delivery = DeliveryMode::from_args(&args);
+ let modifiers: Vec<String> = args.str_array("modifier");
+ if !modifiers.is_empty() && delivery != DeliveryMode::Foreground {
+ return ToolResult::error(
+ "drag modifiers require delivery_mode:\"foreground\" on Windows; posted \
+ messages and pen injection cannot carry live modifier-key state",
+ )
+ .with_structured(json!({
+ "code": "background_unavailable",
+ "effect": "refused",
+ "escalation": {
+ "recommended": "foreground",
+ "reason": "Windows modifier drags require SendInput so the target observes live modifier state"
+ }
+ }));
+ }
// Accepts numeric JSON as either float or integer — coerce both to f64.
let coerce = |key: &str| -> Option<f64> {
@@ -8048,6 +8069,7 @@ impl Tool for DragTool {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
let send_result = tokio::task::spawn_blocking(move || {
+ let modifier_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();
crate::input::mouse::send_drag_synthesized_active(
hwnd,
sx_from,
@@ -8057,6 +8079,7 @@ impl Tool for DragTool {
duration_ms,
steps,
&btn_fg,
+ &modifier_refs,
)
});
let visual_drag = track_overlay_drag(
@@ -10685,6 +10708,9 @@ mod set_window_frame_geometry_tests;
#[cfg(test)]
mod desktop_scope_tests;
+#[cfg(test)]
+mod drag_modifier_tests;
+
#[cfg(test)]
mod pid_window_target_tests;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_/drag_modifier_tests.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_/drag_modifier_tests.rs
new file mode 100644
index 0000000000..3f133444d9
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_/drag_modifier_tests.rs
@@ -0,0 +1,33 @@
+use super::DragTool;
+use cua_driver_core::tool::Tool;
+use serde_json::json;
+
+// A background drag posts mouse messages or injects pen input, neither of
+// which carries live modifier-key state. The refusal comes before any window
+// lookup or input, so an unknown pid never reaches the OS.
+#[tokio::test]
+async fn background_drag_with_modifiers_is_refused_before_input() {
+ let tool = DragTool {
+ state: super::ToolState::new(None),
+ };
+ for delivery in [json!(null), json!("background")] {
+ let mut args = json!({
+ "pid": 0x7fff_fff0_i64,
+ "from_x": 10, "from_y": 10, "to_x": 50, "to_y": 10,
+ "modifier": ["ctrl"]
+ });
+ if !delivery.is_null() {
+ args["delivery_mode"] = delivery;
+ }
+ let result = tool.invoke(args).await;
+ assert_eq!(
+ result.is_error,
+ Some(true),
+ "modified background drag was not refused"
+ );
+ let structured = result.structured_content.expect("structured refusal");
+ assert_eq!(structured["code"], "background_unavailable");
+ assert_eq!(structured["effect"], "refused");
+ assert_eq!(structured["escalation"]["recommended"], "foreground");
+ }
+}
diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs
index 0c90d19a1e..70c0b731e9 100644
--- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs
+++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs
@@ -273,7 +273,9 @@ public partial class MainWindow : Window
private void OnSliderChanged(object sender, RoutedPropertyChangedEventArgs<double> e)
{
- LblSliderValue.Text = $"slider_value={(int)e.NewValue}";
+ // Keyboard.Modifiers reads live GetKeyState, so a thumb drag with held
+ // keys reports them for each move.
+ LblSliderValue.Text = $"slider_value={(int)e.NewValue} modifiers={Keyboard.Modifiers}";
}
private void UpdateChkState()
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
index dc632e3e2c..705b9bf2a5 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
@@ -764,7 +764,7 @@
"read_only": false
},
{
- "description": "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. Background delivery does not steal focus; delivery_mode:\"foreground\" activates the target and refuses with foreground_unavailable, sending nothing, when another window still covers the start point. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background — the OS move/resize loop needs real pointer input — so they return background_unavailable; re-issue those with delivery_mode:\"foreground\". Limits and fallbacks: `get_guide({topic:\"drag\"})`.",
+ "description": "Press-drag-release gesture from (from_x, from_y) to (to_x, to_y) in window-local screenshot pixels. duration_ms (default 500) is the wall-clock budget; steps (default 20) interpolates intermediate WM_MOUSEMOVE events along the path. Background delivery does not steal focus; delivery_mode:\"foreground\" activates the target and refuses with foreground_unavailable, sending nothing, when another window still covers the start point. After a zoom call, pass from_zoom=true. Drags that start on a window caption / title bar or resize border (i.e. moving or resizing the window itself) cannot be delivered in the background — the OS move/resize loop needs real pointer input — so they return background_unavailable; re-issue those with delivery_mode:\"foreground\". `modifier` holds keys from press to release in delivery_mode:\"foreground\" and scope:\"desktop\" via SendInput. Background drags with modifiers return background_unavailable because posted messages and pen injection cannot carry live modifier-key state. Limits and fallbacks: `get_guide({topic:\"drag\"})`.",
"destructive": true,
"idempotent": false,
"input_schema": {