Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): harden Windows delivery and GUI validation

Francesco Bonacci3c5c4c13552e
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/doctor.rs b/libs/cua-driver/rust/crates/cua-driver/src/doctor.rs
index ff53f6ec6b..a0f0c58aba 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/doctor.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/doctor.rs
@@ -253,7 +253,8 @@ fn append_platform_probes(report: &mut Report) {
// attached WindowStation+Desktop, which silently breaks every
// window-driving tool. Surface the misconfiguration directly so users
// don't waste hours debugging tools that are working as designed.
- let in_session_0 = match diag::current_session_id() {
+ let desktop = diag::desktop_state();
+ let in_session_0 = match desktop.session_id {
Some(0) => {
report.push(
Probe::warn(
@@ -267,32 +268,50 @@ fn append_platform_probes(report: &mut Report) {
true
}
Some(sid) => {
- match diag::interactive_desktop_check() {
- Ok(true) => report.push(Probe::ok(
+ if desktop.has_foreground_window() {
+ report.push(Probe::ok(
"interactive session",
format!(
- "session {sid} has an attached interactive desktop (WinSta0 + foreground window)"
+ "session {sid} has an attached interactive desktop ({})",
+ desktop.summary()
),
- )),
- Ok(false) => report.push(
+ ));
+ } else if desktop.input_desktop_is_default() {
+ report.push(
Probe::warn(
"interactive session",
format!(
- "session {sid}: WinSta0 reachable but no foreground window — desktop may be locked or no app is in the foreground"
+ "session {sid}: Default input desktop is reachable but no window is foreground ({})",
+ desktop.summary()
),
+ )
+ .with_detail(
+ "GUI tests can seed foreground by launching their focus sentinel; unattended runners should still validate that the sentinel becomes foreground.",
),
- ),
- Err(e) => report.push(Probe::warn(
- "interactive session",
- format!("session {sid} desktop probe failed: {e}"),
- )),
+ );
+ } else {
+ report.push(
+ Probe::warn(
+ "interactive session",
+ format!(
+ "session {sid}: input desktop is not the user Default desktop ({})",
+ desktop.summary()
+ ),
+ )
+ .with_detail(
+ "this usually means the RDP/console session is locked or disconnected; reconnect, use tscon-to-console, or boot the disposable GUI VM with an unlocked console session.",
+ ),
+ );
}
false
}
None => {
report.push(Probe::warn(
"interactive session",
- "ProcessIdToSessionId failed — cannot determine session id",
+ format!(
+ "ProcessIdToSessionId failed — cannot determine session id ({})",
+ desktop.summary()
+ ),
));
false
}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs
index cd8d793c93..c2264a9590 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs
@@ -146,12 +146,14 @@ fn launch_driver_and_test_app() -> Option<(McpDriver, u32, u64)> {
eprintln!("test app not available — skipping");
return None;
};
- let Some(app_wid) = find_window_for_pid(&mut driver, app_pid as i64) else {
+ let Some((window_pid, app_wid)) = find_window_for_pid(&mut driver, app_pid as i64) else {
eprintln!("test app window not found — skipping");
return None;
};
- Some((driver, app_pid, app_wid))
+ // Electron may create the visible window in a Chromium child process.
+ // Actions must use the PID that owns the HWND, not the launcher parent.
+ Some((driver, window_pid, app_wid))
}
fn kill_process_tree_by_image(image: &str) {
@@ -250,7 +252,7 @@ fn launch_focus_monitor() -> Option<(Child, u64, u32)> {
}
/// Find the first on-screen window belonging to the given pid.
-fn find_window_for_pid(driver: &mut McpDriver, pid: i64) -> Option<u64> {
+fn find_window_for_pid(driver: &mut McpDriver, pid: i64) -> Option<(u32, u64)> {
let resp = driver.call(
"list_windows",
serde_json::json!({"pid": pid, "on_screen_only": true}),
@@ -258,7 +260,7 @@ fn find_window_for_pid(driver: &mut McpDriver, pid: i64) -> Option<u64> {
resp.structured()["windows"]
.as_array()?
.iter()
- .find_map(|w| w["window_id"].as_u64())
+ .find_map(|w| Some((w["pid"].as_u64()? as u32, w["window_id"].as_u64()?)))
}
fn window_ids(driver: &mut McpDriver) -> HashSet<u64> {
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
index a75016e4ef..ede8d31e8b 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
@@ -48,7 +48,9 @@ use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolR
fn harness_exe() -> PathBuf {
if let Ok(p) = std::env::var("HARNESS_WPF_EXE") {
let pb = PathBuf::from(p);
- if pb.exists() { return pb; }
+ if pb.exists() {
+ return pb;
+ }
}
harness_app("harness-wpf", "CuaTestHarness.Wpf.exe")
}
@@ -68,8 +70,11 @@ fn launch_harness(driver: &mut McpDriver) -> Option<u32> {
return None;
}
let app = spawn_in_job(
- Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()),
- ).ok()?;
+ Command::new(&exe)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null()),
+ )
+ .ok()?;
let pid = app.id();
driver.reaper().push(app);
// Short fixed settle for cold-start (window-creation + initial
@@ -87,11 +92,32 @@ fn launch_harness(driver: &mut McpDriver) -> Option<u32> {
// exist in the tree without depending on tree-walk order, and to read the
// harness's marker/status text.
fn snapshot(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolResponse {
- driver.call("get_window_state", serde_json::json!({
- "pid": pid as i64,
- "window_id": window_id,
- "capture_mode": "ax"
- }))
+ driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": window_id,
+ "capture_mode": "ax"
+ }),
+ )
+}
+
+fn snapshot_lines_containing(text: &str, needles: &[&str]) -> String {
+ let mut lines = Vec::new();
+ for line in text.lines() {
+ let lower = line.to_lowercase();
+ if needles
+ .iter()
+ .any(|needle| lower.contains(&needle.to_lowercase()))
+ {
+ lines.push(line.trim().to_owned());
+ }
+ }
+ if lines.is_empty() {
+ text.lines().take(80).collect::<Vec<_>>().join(" / ")
+ } else {
+ lines.join(" / ")
+ }
}
// ── tests ────────────────────────────────────────────────────────────────────
@@ -99,11 +125,16 @@ fn snapshot(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolResponse {
#[test]
#[ignore]
fn harness_wpf_smoke() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some(pid) = launch_harness(&mut driver) else { return };
+ let Some(mut driver) = McpDriver::spawn() else {
+ return;
+ };
+ let Some(pid) = launch_harness(&mut driver) else {
+ return;
+ };
println!("harness pid={}", pid);
- let (wid, title) = driver.find_window(pid as i64, "CuaTestHarness WPF")
+ let (wid, title) = driver
+ .find_window(pid as i64, "CuaTestHarness WPF")
.expect("main window not found via list_windows");
println!("main window: id={} title={:?}", wid, title);
@@ -112,23 +143,41 @@ fn harness_wpf_smoke() {
// Buttons appear with explicit id=<aid> tags in the UIA markdown.
for aid in [
- "btn-increment", "btn-reset",
+ "btn-increment",
+ "btn-reset",
"btn-open-msgbox",
- "btn-save", "btn-cancel", // regression guard for #1696
- "btn-open-owned", "btn-open-layered",
+ "btn-save",
+ "btn-cancel", // regression guard for #1696
+ "btn-open-owned",
+ "btn-open-layered",
"btn-exit",
] {
- assert!(ax::has_id(text, aid), "missing AutomationId {aid} in WPF UIA snapshot");
+ assert!(
+ ax::has_id(text, aid),
+ "missing AutomationId {aid} in WPF UIA snapshot"
+ );
}
// TextBlocks are reported as bare Text nodes (no UIA Invoke/Value pattern,
// no AutomationId in the rendered tree). Assert on their content instead.
- assert!(text.contains("HARNESS_TEXT_MARKER_v1"), "text_body marker not in snapshot");
- assert!(text.contains("counter=0"), "initial counter label not in snapshot");
- assert!(text.contains("accel_fired=0"), "initial accel label not in snapshot");
+ assert!(
+ text.contains("HARNESS_TEXT_MARKER_v1"),
+ "text_body marker not in snapshot"
+ );
+ assert!(
+ text.contains("counter=0"),
+ "initial counter label not in snapshot"
+ );
+ assert!(
+ text.contains("accel_fired=0"),
+ "initial accel label not in snapshot"
+ );
// HwndHost child should surface the native Win32 BUTTON as a UIA Button.
- assert!(text.contains("\"Native Win32 Child\""), "native HWND child button not in snapshot");
+ assert!(
+ text.contains("\"Native Win32 Child\""),
+ "native HWND child button not in snapshot"
+ );
println!("✅ harness_wpf_smoke: all expected scenarios present in UIA tree");
}
@@ -140,10 +189,13 @@ fn harness_wpf_smoke() {
/// Object). Returns whatever the closure returns. The closure receives the
/// harness pid, a pre-resolved main window_id, and the driver.
fn with_session<F, R>(f: F) -> Option<R>
-where F: FnOnce(u32, u64, &mut McpDriver) -> R {
+where
+ F: FnOnce(u32, u64, &mut McpDriver) -> R,
+{
let mut driver = McpDriver::spawn()?;
let pid = launch_harness(&mut driver)?;
- let (wid, _) = driver.find_window(pid as i64, "CuaTestHarness WPF")
+ let (wid, _) = driver
+ .find_window(pid as i64, "CuaTestHarness WPF")
.expect("main window");
Some(f(pid, wid, &mut driver))
}
@@ -151,21 +203,29 @@ where F: FnOnce(u32, u64, &mut McpDriver) -> R {
#[test]
#[ignore]
fn harness_wpf_counter_invoke() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some(pid) = launch_harness(&mut driver) else { return };
-
- let (wid, _) = driver.find_window(pid as i64, "CuaTestHarness WPF")
+ let Some(mut driver) = McpDriver::spawn() else {
+ return;
+ };
+ let Some(pid) = launch_harness(&mut driver) else {
+ return;
+ };
+
+ let (wid, _) = driver
+ .find_window(pid as i64, "CuaTestHarness WPF")
.expect("main window");
// Pre-snapshot so element_cache has indices we can address.
let pre = snapshot(&mut driver, pid, wid);
let idx = ax::element_index_by_id(pre.text(), "btn-increment")
.expect("btn-increment not in pre-snapshot");
- let click = driver.call("click", serde_json::json!({
- "pid": pid as i64,
- "window_id": wid,
- "element_index": idx
- }));
+ let click = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": wid,
+ "element_index": idx
+ }),
+ );
println!("click [{idx}] btn-increment: {}", click.text());
std::thread::sleep(Duration::from_millis(300));
@@ -185,8 +245,8 @@ fn harness_wpf_counter_invoke() {
fn harness_wpf_type_text() {
with_session(|pid, wid, driver| {
let snap = snapshot(driver, pid, wid);
- let idx = ax::element_index_by_id(snap.text(), "txt-input")
- .expect("txt-input not in snapshot");
+ let idx =
+ ax::element_index_by_id(snap.text(), "txt-input").expect("txt-input not in snapshot");
// WPF's TextBox needs *keyboard focus* for WM_CHAR delivery — and
// PostMessage(WM_LBUTTONDOWN) doesn't reliably transfer keyboard
@@ -194,40 +254,57 @@ fn harness_wpf_type_text() {
// real ones). Use dispatch:"foreground" → SendInput synthesizes
// an OS-level click that WPF treats identically to a user mouse,
// landing actual keyboard focus on the TextBox.
- let _ = driver.call("bring_to_front", serde_json::json!({
- "pid": pid as i64, "window_id": wid
- }));
+ let _ = driver.call(
+ "bring_to_front",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid
+ }),
+ );
std::thread::sleep(Duration::from_millis(300));
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx,
- "delivery_mode": "foreground"
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "delivery_mode": "foreground"
+ }),
+ );
std::thread::sleep(Duration::from_millis(400));
// SendInput's restore_foreground_polling_best_effort may yank
// foreground back from the harness window between click and
// type_text. Re-assert foreground so PostMessage WM_CHAR finds
// the TextBox with keyboard focus.
- let _ = driver.call("bring_to_front", serde_json::json!({
- "pid": pid as i64, "window_id": wid
- }));
+ let _ = driver.call(
+ "bring_to_front",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid
+ }),
+ );
std::thread::sleep(Duration::from_millis(300));
- let resp = driver.call("type_text", serde_json::json!({
- "pid": pid as i64, "text": "harness-typed"
- }));
+ let resp = driver.call(
+ "type_text",
+ serde_json::json!({
+ "pid": pid as i64,
+ "text": "harness-typed",
+ "delivery_mode": "foreground"
+ }),
+ );
println!("type_text: {}", resp.text());
std::thread::sleep(Duration::from_millis(700));
let post = snapshot(driver, pid, wid);
let text = post.text();
- let mirror_lines: Vec<&str> = text.lines()
+ let mirror_lines: Vec<&str> = text
+ .lines()
.filter(|l| l.contains("mirror=") || l.contains("txt-input"))
.collect();
- assert!(text.contains("mirror=harness-typed"),
+ assert!(
+ text.contains("mirror=harness-typed"),
"TextBox mirror did not reflect typed text. Mirror/input lines: {:?}",
- mirror_lines);
+ mirror_lines
+ );
println!("✅ harness_wpf_type_text: TextBox mirror advanced to 'harness-typed'");
});
}
@@ -239,19 +316,24 @@ fn harness_wpf_set_value() {
// write path via the `set_value` tool. No focus needed — purely UIA.
with_session(|pid, wid, driver| {
let snap = snapshot(driver, pid, wid);
- let idx = ax::element_index_by_id(snap.text(), "txt-input")
- .expect("txt-input not in snapshot");
- let _ = driver.call("set_value", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx,
- "value": "via-uia-setvalue"
- }));
+ let idx =
+ ax::element_index_by_id(snap.text(), "txt-input").expect("txt-input not in snapshot");
+ let _ = driver.call(
+ "set_value",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "value": "via-uia-setvalue"
+ }),
+ );
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
let text = post.text();
- assert!(text.contains("mirror=via-uia-setvalue"),
+ assert!(
+ text.contains("mirror=via-uia-setvalue"),
"set_value did not update TextBox. Excerpt: {}",
- text.chars().take(500).collect::<String>());
+ text.chars().take(500).collect::<String>()
+ );
println!("✅ harness_wpf_set_value: ValuePattern.SetValue wrote to TextBox");
});
}
@@ -264,9 +346,12 @@ fn harness_wpf_set_value() {
// background-delivery path (which the counter_invoke test already
// covers via UIA Invoke).
fn focus_harness(driver: &mut McpDriver, pid: u32, wid: u64) {
- let _ = driver.call("bring_to_front", serde_json::json!({
- "pid": pid as i64, "window_id": wid
- }));
+ let _ = driver.call(
+ "bring_to_front",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid
+ }),
+ );
std::thread::sleep(Duration::from_millis(300));
}
@@ -281,20 +366,27 @@ fn harness_wpf_right_click() {
// Same dispatch:foreground rationale as type_text — PostMessage
// WM_RBUTTONDOWN doesn't always reach WPF's MouseRightButtonDown
// routed-event chain (intermittent in batch runs).
- let resp = driver.call("right_click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx,
- "delivery_mode": "foreground"
- }));
+ let resp = driver.call(
+ "right_click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "delivery_mode": "foreground"
+ }),
+ );
println!("right_click: {}", resp.text());
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
let text = post.text();
- let action_lines: Vec<&str> = text.lines()
+ let action_lines: Vec<&str> = text
+ .lines()
.filter(|l| l.contains("last_action=") || l.contains("clicks="))
.collect();
- assert!(text.contains("last_action=right_click"),
- "right_click handler did not fire. Action/click lines: {:?}", action_lines);
+ assert!(
+ text.contains("last_action=right_click"),
+ "right_click handler did not fire. Action/click lines: {:?}",
+ action_lines
+ );
println!("✅ harness_wpf_right_click: last_action=right_click");
});
}
@@ -310,21 +402,28 @@ fn harness_wpf_double_click() {
// dispatch:foreground for the same reason as right_click —
// PostMessage WM_LBUTTONDOWN ×2 doesn't always reach WPF's
// MouseDoubleClick / ClickCount=2 path under test-batch load.
- let resp = driver.call("double_click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx,
- "delivery_mode": "foreground"
- }));
+ let resp = driver.call(
+ "double_click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "delivery_mode": "foreground"
+ }),
+ );
println!("double_click: {}", resp.text());
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
let text = post.text();
- let action_lines: Vec<&str> = text.lines()
+ let action_lines: Vec<&str> = text
+ .lines()
.filter(|l| l.contains("last_action=") || l.contains("clicks="))
.collect();
- assert!(text.contains("last_action=double_click"),
+ assert!(
+ text.contains("last_action=double_click"),
"double_click handler did not register a 2nd click. \
- Action/click lines: {:?}", action_lines);
+ Action/click lines: {:?}",
+ action_lines
+ );
println!("✅ harness_wpf_double_click: last_action=double_click");
});
}
@@ -340,17 +439,22 @@ fn harness_wpf_press_key_accelerator() {
// helper that isn't in our test config. F5 has no modifier and works
// on the PostMessage path.
with_session(|pid, wid, driver| {
- let resp = driver.call("press_key", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "key": "f5"
- }));
+ let resp = driver.call(
+ "press_key",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "key": "f5"
+ }),
+ );
println!("press_key f5: {}", resp.text());
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
let text = post.text();
- assert!(text.contains("accel_fired=1"),
+ assert!(
+ text.contains("accel_fired=1"),
"F5 KeyBinding did not fire. Snapshot excerpt: {}",
- text.chars().take(500).collect::<String>());
+ text.chars().take(500).collect::<String>()
+ );
println!("✅ harness_wpf_press_key_accelerator: accel_fired=1 (F5 via PostMessage)");
});
}
@@ -362,34 +466,57 @@ fn harness_wpf_scroll() {
// Pre-snapshot to populate the cache + read initial offset.
let pre = snapshot(driver, pid, wid);
let pre_text = pre.text();
- assert!(pre_text.contains("scroll_offset=0"),
+ assert!(
+ pre_text.contains("scroll_offset=0"),
"expected initial scroll_offset=0, got: {}",
- pre_text.lines().filter(|l| l.contains("scroll_offset")).collect::<Vec<_>>().join(" / "));
+ pre_text
+ .lines()
+ .filter(|l| l.contains("scroll_offset"))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
// Click into the ScrollViewer so it gets focus / its descendants
// become the WM_VSCROLL target.
let idx = ax::element_index_by_id(pre.text(), "scroll-tall")
.expect("scroll-tall not in snapshot");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": wid,
+ "element_index": idx,
+ "delivery_mode": "foreground"
+ }),
+ );
std::thread::sleep(Duration::from_millis(200));
- // Scroll down 5 lines.
- let resp = driver.call("scroll", serde_json::json!({
- "pid": pid as i64, "window_id": wid,
- "direction": "down", "by": "line", "amount": 5
- }));
+ // Scroll down 5 lines. Keep this on the default background rung: the
+ // WPF harness translates the driver's WM_VSCROLL messages into the
+ // ScrollViewer movement we assert below.
+ let resp = driver.call(
+ "scroll",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "direction": "down", "by": "line", "amount": 5,
+ }),
+ );
println!("scroll down: {}", resp.text());
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
let text = post.text();
- let advanced = text.lines().any(|l|
- l.contains("scroll_offset=") && !l.contains("scroll_offset=0\""));
- assert!(advanced, "scroll offset did not advance after WM_VSCROLL. Lines: {}",
- text.lines().filter(|l| l.contains("scroll_offset"))
- .collect::<Vec<_>>().join(" / "));
+ let advanced = text
+ .lines()
+ .any(|l| l.contains("scroll_offset=") && !l.contains("scroll_offset=0\""));
+ assert!(
+ advanced,
+ "scroll offset did not advance after WM_VSCROLL. Lines: {}",
+ text.lines()
+ .filter(|l| l.contains("scroll_offset"))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
println!("✅ harness_wpf_scroll: scroll_offset advanced past 0");
});
}
@@ -401,37 +528,58 @@ fn harness_wpf_modal_messagebox() {
let snap = snapshot(driver, pid, wid);
let idx = ax::element_index_by_id(snap.text(), "btn-open-msgbox")
.expect("btn-open-msgbox not in snapshot");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx
+ }),
+ );
std::thread::sleep(Duration::from_millis(600));
// List windows — the modal MessageBox should be a new top-level window
// owned by the same pid.
- let resp = driver.call("list_windows", serde_json::json!({
- "pid": pid as i64
- }));
- let windows = resp.structured()["windows"].as_array()
+ let resp = driver.call(
+ "list_windows",
+ serde_json::json!({
+ "pid": pid as i64
+ }),
+ );
+ let windows = resp.structured()["windows"]
+ .as_array()
.expect("windows array");
- let modal = windows.iter()
- .find(|w| w["title"].as_str().map(|t| t.contains("Harness MessageBox")).unwrap_or(false))
+ let modal = windows
+ .iter()
+ .find(|w| {
+ w["title"]
+ .as_str()
+ .map(|t| t.contains("Harness MessageBox"))
+ .unwrap_or(false)
+ })
.expect("Harness MessageBox modal window not found");
let modal_wid = modal["window_id"].as_u64().unwrap();
println!("modal window_id={}", modal_wid);
// Walk the modal's UIA tree — expect OK and Cancel buttons.
- let modal_snap = driver.call("get_window_state", serde_json::json!({
- "pid": pid as i64, "window_id": modal_wid, "capture_mode": "ax"
- }));
+ let modal_snap = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": modal_wid, "capture_mode": "ax"
+ }),
+ );
let modal_text = modal_snap.text();
- assert!(modal_text.contains("\"OK\""),
+ assert!(
+ modal_text.contains("\"OK\""),
"MessageBox UIA tree missing OK button. Tree: {}",
- modal_text.chars().take(800).collect::<String>());
- assert!(modal_text.contains("\"Cancel\""),
- "MessageBox UIA tree missing Cancel button");
+ modal_text.chars().take(800).collect::<String>()
+ );
+ assert!(
+ modal_text.contains("\"Cancel\""),
+ "MessageBox UIA tree missing Cancel button"
+ );
// Dismiss by clicking Cancel in the modal.
- let cancel_idx = modal_text.lines()
+ let cancel_idx = modal_text
+ .lines()
.find(|l| l.contains("\"Cancel\"") && l.contains('['))
.and_then(|l| {
let s = l.find('[')? + 1;
@@ -439,9 +587,12 @@ fn harness_wpf_modal_messagebox() {
l[s..e].trim().parse::<u64>().ok()
})
.expect("Cancel button element_index not parseable");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": modal_wid, "element_index": cancel_idx
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": modal_wid, "element_index": cancel_idx
+ }),
+ );
std::thread::sleep(Duration::from_millis(400));
println!("✅ harness_wpf_modal_messagebox: opened + parsed + dismissed");
});
@@ -454,27 +605,44 @@ fn harness_wpf_owned_popup() {
let snap = snapshot(driver, pid, wid);
let idx = ax::element_index_by_id(snap.text(), "btn-open-owned")
.expect("btn-open-owned not in snapshot");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx
+ }),
+ );
std::thread::sleep(Duration::from_millis(500));
- let resp = driver.call("list_windows", serde_json::json!({
- "pid": pid as i64
- }));
+ let resp = driver.call(
+ "list_windows",
+ serde_json::json!({
+ "pid": pid as i64
+ }),
+ );
let windows = resp.structured()["windows"].as_array().unwrap();
- let owned = windows.iter()
- .find(|w| w["title"].as_str().map(|t| t.contains("Harness Owned Popup")).unwrap_or(false))
+ let owned = windows
+ .iter()
+ .find(|w| {
+ w["title"]
+ .as_str()
+ .map(|t| t.contains("Harness Owned Popup"))
+ .unwrap_or(false)
+ })
.expect("Harness Owned Popup window not found in list_windows");
let owned_wid = owned["window_id"].as_u64().unwrap();
- let owned_snap = driver.call("get_window_state", serde_json::json!({
- "pid": pid as i64, "window_id": owned_wid, "capture_mode": "ax"
- }));
+ let owned_snap = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": owned_wid, "capture_mode": "ax"
+ }),
+ );
let owned_text = owned_snap.text();
- assert!(owned_text.contains("OWNED_POPUP_MARKER_v1"),
+ assert!(
+ owned_text.contains("OWNED_POPUP_MARKER_v1"),
"owned popup body marker missing. Tree: {}",
- owned_text.chars().take(600).collect::<String>());
+ owned_text.chars().take(600).collect::<String>()
+ );
println!("✅ harness_wpf_owned_popup: opened + parsed");
});
}
@@ -486,42 +654,72 @@ fn harness_wpf_layered_popup_capture() {
let snap = snapshot(driver, pid, wid);
let idx = ax::element_index_by_id(snap.text(), "btn-open-layered")
.expect("btn-open-layered not in snapshot");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx
+ }),
+ );
std::thread::sleep(Duration::from_millis(600));
- let resp = driver.call("list_windows", serde_json::json!({
- "pid": pid as i64
- }));
+ let resp = driver.call(
+ "list_windows",
+ serde_json::json!({
+ "pid": pid as i64
+ }),
+ );
let windows = resp.structured()["windows"].as_array().unwrap();
- let layered = windows.iter()
- .find(|w| w["title"].as_str().map(|t| t.contains("Harness Layered Popup")).unwrap_or(false))
+ let layered = windows
+ .iter()
+ .find(|w| {
+ w["title"]
+ .as_str()
+ .map(|t| t.contains("Harness Layered Popup"))
+ .unwrap_or(false)
+ })
.expect("Harness Layered Popup window not found");
let layered_wid = layered["window_id"].as_u64().unwrap();
// Capture-only path — assert the screenshot is not all-black, which
// is the failure mode for PrintWindow against layered windows
// without the WGC fallback.
- let cap = driver.call("get_window_state", serde_json::json!({
- "pid": pid as i64, "window_id": layered_wid, "capture_mode": "vision"
- }));
- let img_b64 = cap.raw["result"]["content"].as_array()
- .and_then(|arr| arr.iter().find_map(|c| {
- if c["type"] == "image" { c["data"].as_str() } else { None }
- }))
+ let cap = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": layered_wid, "capture_mode": "vision"
+ }),
+ );
+ let img_b64 = cap.raw["result"]["content"]
+ .as_array()
+ .and_then(|arr| {
+ arr.iter().find_map(|c| {
+ if c["type"] == "image" {
+ c["data"].as_str()
+ } else {
+ None
+ }
+ })
+ })
.expect("layered window capture returned no image");
// Decode the PNG and look for any non-black pixel.
let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, img_b64)
.expect("base64");
let img = image::load_from_memory(&bytes).expect("png decode");
let rgb = img.to_rgb8();
- let any_color = rgb.pixels().any(|p| p.0[0] > 12 || p.0[1] > 12 || p.0[2] > 12);
- assert!(any_color,
+ let any_color = rgb
+ .pixels()
+ .any(|p| p.0[0] > 12 || p.0[1] > 12 || p.0[2] > 12);
+ assert!(
+ any_color,
"layered window capture is all-black ({}x{}). PrintWindow likely needs WGC fallback.",
- rgb.width(), rgb.height());
- println!("✅ harness_wpf_layered_popup_capture: capture has non-black pixels ({}x{})",
- rgb.width(), rgb.height());
+ rgb.width(),
+ rgb.height()
+ );
+ println!(
+ "✅ harness_wpf_layered_popup_capture: capture has non-black pixels ({}x{})",
+ rgb.width(),
+ rgb.height()
+ );
});
}
@@ -545,36 +743,48 @@ fn harness_wpf_slider_drag() {
with_session(|pid, wid, driver| {
focus_harness(driver, pid, wid);
let pre = snapshot(driver, pid, wid);
- assert!(pre.text().contains("slider_value=0"),
- "initial slider_value=0 missing");
-
- let resp = driver.call("drag", serde_json::json!({
- "pid": pid as i64, "window_id": wid,
- // Window-local coords along the slider TRACK. The track row sits at
- // window-local y≈304 (verified on the VM: y=275 landed ~29px above
- // it, on empty GroupBox space, so the thumb never moved); the thumb
- // rests at the left (x≈44) at value=0. Dragging left→right advances
- // the value. (TODO: derive these from the `sld-value` element frame
- // in get_window_state for DPI/placement independence.)
- "from_x": 44.0, "from_y": 304.0,
- "to_x": 330.0, "to_y": 304.0,
- "duration_ms": 700, "steps": 40,
- "delivery_mode": "foreground"
- }));
+ assert!(
+ pre.text().contains("slider_value=0"),
+ "initial slider_value=0 missing"
+ );
+
+ let resp = driver.call(
+ "drag",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ // Window-local coords along the slider TRACK. The track row sits at
+ // window-local y≈304 (verified on the VM: y=275 landed ~29px above
+ // it, on empty GroupBox space, so the thumb never moved); the thumb
+ // rests at the left (x≈44) at value=0. Dragging left→right advances
+ // the value. (TODO: derive these from the `sld-value` element frame
+ // in get_window_state for DPI/placement independence.)
+ "from_x": 44.0, "from_y": 304.0,
+ "to_x": 330.0, "to_y": 304.0,
+ "duration_ms": 700, "steps": 40,
+ "delivery_mode": "foreground"
+ }),
+ );
let msg = resp.text();
println!("drag slider (foreground): {msg}");
- assert!(msg.starts_with("✅"),
- "drag tool returned non-success: {msg}");
+ assert!(
+ msg.starts_with("✅"),
+ "drag tool returned non-success: {msg}"
+ );
std::thread::sleep(Duration::from_millis(500));
let post = snapshot(driver, pid, wid);
let text = post.text();
- let advanced = text.lines().any(|l|
- l.contains("slider_value=") && !l.contains("slider_value=0\""));
- assert!(advanced,
+ let advanced = text
+ .lines()
+ .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\""));
+ assert!(
+ advanced,
"Slider value did not advance via SendInput drag. Lines: {}",
- text.lines().filter(|l| l.contains("slider_value"))
- .collect::<Vec<_>>().join(" / "));
+ text.lines()
+ .filter(|l| l.contains("slider_value"))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag");
});
}
@@ -591,19 +801,29 @@ fn harness_wpf_slider_increase_large() {
let idx = ax::element_index_by_id(snap.text(), "IncreaseLarge")
.expect("slider IncreaseLarge button not in snapshot");
for i in 0..3 {
- let resp = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx
- }));
+ let resp = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx
+ }),
+ );
println!("invoke IncreaseLarge #{i}: {}", resp.text());
std::thread::sleep(Duration::from_millis(150));
}
std::thread::sleep(Duration::from_millis(300));
let post = snapshot(driver, pid, wid);
let text = post.text();
- let advanced = text.lines().any(|l|
- l.contains("slider_value=") && !l.contains("slider_value=0\""));
- assert!(advanced, "slider IncreaseLarge invokes did not advance value. Lines: {}",
- text.lines().filter(|l| l.contains("slider_value")).collect::<Vec<_>>().join(" / "));
+ let advanced = text
+ .lines()
+ .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\""));
+ assert!(
+ advanced,
+ "slider IncreaseLarge invokes did not advance value. Lines: {}",
+ text.lines()
+ .filter(|l| l.contains("slider_value"))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
println!("✅ harness_wpf_slider_increase_large: advanced via UIA Invoke");
});
}
@@ -614,23 +834,31 @@ fn harness_wpf_checkbox_toggle() {
with_session(|pid, wid, driver| {
focus_harness(driver, pid, wid);
let snap = snapshot(driver, pid, wid);
- let idx = ax::element_index_by_id(snap.text(), "chk-agreed")
- .expect("chk-agreed missing");
+ let idx = ax::element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed missing");
// CheckBox exposes UIA TogglePattern (actions=[toggle]), not Invoke.
// cua-driver's click tool tries UIA Invoke first; for elements that
// don't support it the PostMessage fallback path runs. Use
// dispatch:"foreground" to land a SendInput click that WPF
// recognises as a real user click and processes through Toggle.
- let resp = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx,
- "delivery_mode": "foreground"
- }));
+ let resp = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "delivery_mode": "foreground"
+ }),
+ );
println!("click chk-agreed: {}", resp.text());
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
- assert!(post.text().contains("agreed=True"),
+ assert!(
+ post.text().contains("agreed=True"),
"checkbox didn't toggle: {}",
- post.text().lines().filter(|l| l.contains("agreed=")).collect::<Vec<_>>().join(" / "));
+ post.text()
+ .lines()
+ .filter(|l| l.contains("agreed="))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
println!("✅ harness_wpf_checkbox_toggle: agreed=True");
});
}
@@ -641,18 +869,22 @@ fn harness_wpf_radio_select() {
with_session(|pid, wid, driver| {
focus_harness(driver, pid, wid);
let snap = snapshot(driver, pid, wid);
- let idx = ax::element_index_by_id(snap.text(), "rdo-high")
- .expect("rdo-high missing");
+ let idx = ax::element_index_by_id(snap.text(), "rdo-high").expect("rdo-high missing");
// RadioButton exposes SelectionItem pattern (actions=[select]).
// Same dispatch:foreground rationale as the checkbox test.
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx,
- "delivery_mode": "foreground"
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "delivery_mode": "foreground"
+ }),
+ );
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
- assert!(post.text().contains("prio=High"),
- "radio didn't switch to High");
+ assert!(
+ post.text().contains("prio=High"),
+ "radio didn't switch to High"
+ );
println!("✅ harness_wpf_radio_select: prio=High");
});
}
@@ -663,31 +895,43 @@ fn harness_wpf_combo_select() {
with_session(|pid, wid, driver| {
focus_harness(driver, pid, wid);
let snap = snapshot(driver, pid, wid);
- let combo_idx = ax::element_index_by_id(snap.text(), "cbo-color")
- .expect("cbo-color missing");
+ let combo_idx =
+ ax::element_index_by_id(snap.text(), "cbo-color").expect("cbo-color missing");
// WPF ComboBox UIA peer surfaces ExpandCollapsePattern (actions=[expand])
// but not ValuePattern — set_value at the parent is a no-op. Standard
// recipe: invoke the combo to expand the dropdown, re-snapshot so the
// item AIDs land in the element cache, then click the target item.
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": combo_idx,
- "delivery_mode": "foreground"
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": combo_idx,
+ "delivery_mode": "foreground"
+ }),
+ );
std::thread::sleep(Duration::from_millis(500));
let snap2 = snapshot(driver, pid, wid);
let item_idx = ax::element_index_by_id(snap2.text(), "cbo-item-orange")
.expect("cbo-item-orange missing after expand");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": item_idx,
- "delivery_mode": "foreground"
- }));
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": item_idx,
+ "delivery_mode": "foreground"
+ }),
+ );
std::thread::sleep(Duration::from_millis(500));
let post = snapshot(driver, pid, wid);
- assert!(post.text().contains("color=orange"),
+ assert!(
+ post.text().contains("color=orange"),
"combo didn't switch to orange: {}",
- post.text().lines().filter(|l| l.contains("color=")).collect::<Vec<_>>().join(" / "));
+ post.text()
+ .lines()
+ .filter(|l| l.contains("color="))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
println!("✅ harness_wpf_combo_select: color=orange");
});
}
@@ -698,17 +942,25 @@ fn harness_wpf_listbox_select() {
with_session(|pid, wid, driver| {
focus_harness(driver, pid, wid);
let snap = snapshot(driver, pid, wid);
- let idx = ax::element_index_by_id(snap.text(), "lst-cherry")
- .expect("lst-cherry missing");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": idx,
- "delivery_mode": "foreground"
- }));
+ let idx = ax::element_index_by_id(snap.text(), "lst-cherry").expect("lst-cherry missing");
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "delivery_mode": "foreground"
+ }),
+ );
std::thread::sleep(Duration::from_millis(400));
let post = snapshot(driver, pid, wid);
- assert!(post.text().contains("selected=cherry"),
+ assert!(
+ post.text().contains("selected=cherry"),
"list didn't select cherry: {}",
- post.text().lines().filter(|l| l.contains("selected=")).collect::<Vec<_>>().join(" / "));
+ post.text()
+ .lines()
+ .filter(|l| l.contains("selected="))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
println!("✅ harness_wpf_listbox_select: selected=cherry");
});
}
@@ -721,28 +973,50 @@ fn harness_wpf_menu_invoke() {
// Expand File menu first (UIA expand pattern on MenuItem)
let snap = snapshot(driver, pid, wid);
let file_idx = ax::element_index_by_id(snap.text(), "menu-file")
- .expect("menu-file missing");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": file_idx,
- "delivery_mode": "foreground"
- }));
+ .or_else(|| ax::element_index_containing(snap.text(), "File"))
+ .unwrap_or_else(|| {
+ panic!(
+ "menu-file missing. Menu-related snapshot lines: {}",
+ snapshot_lines_containing(snap.text(), &["menu", "file", "new", "open"])
+ )
+ });
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": file_idx,
+ }),
+ );
std::thread::sleep(Duration::from_millis(400));
// Re-snapshot so menu-file-new is in the cache (it materialized
// when the menu expanded).
let snap2 = snapshot(driver, pid, wid);
let new_idx = ax::element_index_by_id(snap2.text(), "menu-file-new")
- .expect("menu-file-new missing after expand");
- let _ = driver.call("click", serde_json::json!({
- "pid": pid as i64, "window_id": wid, "element_index": new_idx,
- "delivery_mode": "foreground"
- }));
+ .or_else(|| ax::element_index_containing(snap2.text(), "New"))
+ .unwrap_or_else(|| {
+ panic!(
+ "menu-file-new missing after expand. Menu-related snapshot lines: {}",
+ snapshot_lines_containing(snap2.text(), &["menu", "file", "new", "open"])
+ )
+ });
+ let _ = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": new_idx,
+ }),
+ );
std::thread::sleep(Duration::from_millis(500));
let post = snapshot(driver, pid, wid);
- assert!(post.text().contains("menu_action=file_new"),
+ assert!(
+ post.text().contains("menu_action=file_new"),
"File>New didn't invoke: {}",
- post.text().lines().filter(|l| l.contains("menu_action=")).collect::<Vec<_>>().join(" / "));
+ post.text()
+ .lines()
+ .filter(|l| l.contains("menu_action="))
+ .collect::<Vec<_>>()
+ .join(" / ")
+ );
println!("✅ harness_wpf_menu_invoke: menu_action=file_new");
});
}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_input_e2e_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_input_e2e_test.rs
index 5fbdaa3269..006bde69a1 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_input_e2e_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_input_e2e_test.rs
@@ -102,6 +102,12 @@ fn read_count(p: &Path) -> u32 {
.and_then(|s| s.trim().parse::<u32>().ok())
.unwrap_or(0)
}
+fn read_focus_hwnd() -> u64 {
+ fs::read_to_string(focus_hwnd_file())
+ .ok()
+ .and_then(|s| s.trim().parse::<u64>().ok())
+ .unwrap_or(0)
+}
fn window_ids(driver: &mut McpDriver) -> HashSet<u64> {
let r = driver.call("list_windows", serde_json::json!({}));
@@ -111,27 +117,68 @@ fn window_ids(driver: &mut McpDriver) -> HashSet<u64> {
.unwrap_or_default()
}
-fn require_interactive_desktop(context: &str) -> bool {
- if platform_windows::diagnostics::current_session_id() == Some(0) {
- eprintln!(
- "{context}: running in Windows Session 0; skipping GUI modality e2e. \
- Re-run from an interactive logon session (RDP/console)."
+fn gui_required() -> bool {
+ std::env::var("CUA_REQUIRE_GUI")
+ .ok()
+ .map(|v| matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on"))
+ .unwrap_or(false)
+}
+
+fn skip_desktop(context: &str, reason: String) -> bool {
+ let msg = format!("{context}: {reason}; skipping GUI modality e2e");
+ if gui_required() {
+ panic!("{msg}");
+ }
+ eprintln!("{msg}");
+ false
+}
+
+fn require_seedable_desktop(context: &str) -> bool {
+ let state = platform_windows::diagnostics::desktop_state();
+ if state.session_id == Some(0) {
+ return skip_desktop(
+ context,
+ format!(
+ "running in Windows Session 0 ({}) - re-run from RDP/console/scheduled task in user session",
+ state.summary()
+ ),
);
- return false;
}
+ if !state.has_process_window_station {
+ return skip_desktop(
+ context,
+ format!("no attached process window station ({})", state.summary()),
+ );
+ }
+ if !state.input_desktop_is_default() {
+ return skip_desktop(
+ context,
+ format!(
+ "input desktop is not the user Default desktop ({})",
+ state.summary()
+ ),
+ );
+ }
+ true
+}
- match platform_windows::diagnostics::interactive_desktop_check() {
- Ok(true) => true,
- Ok(false) => {
- eprintln!("{context}: no foreground interactive desktop; skipping GUI modality e2e");
- false
+fn require_focus_monitor_foreground(context: &str, expected_hwnd: u64) -> bool {
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ let state = platform_windows::diagnostics::desktop_state();
+ if state.foreground_hwnd == Some(expected_hwnd as usize) {
+ return true;
}
- Err(e) => {
- eprintln!(
- "{context}: interactive desktop check failed ({e}); skipping GUI modality e2e"
+ if Instant::now() >= deadline {
+ return skip_desktop(
+ context,
+ format!(
+ "focus monitor did not become foreground (expected HWND 0x{expected_hwnd:x}; {})",
+ state.summary()
+ ),
);
- false
}
+ std::thread::sleep(Duration::from_millis(100));
}
}
@@ -148,7 +195,7 @@ struct E2eFixture {
/// belongs to a child pid) → sentinel (grabs foreground, pushing the app to
/// the background) → reset counters.
fn setup(target_exe: &Path, _title_hint: &str) -> Option<E2eFixture> {
- if !require_interactive_desktop("modality_input_e2e_test") {
+ if !require_seedable_desktop("modality_input_e2e_test") {
return None;
}
@@ -238,12 +285,7 @@ fn setup(target_exe: &Path, _title_hint: &str) -> Option<E2eFixture> {
}
let sdeadline = Instant::now() + Duration::from_secs(10);
loop {
- let ok = read_count(&focus_pid_file()) != 0
- && fs::read_to_string(focus_hwnd_file())
- .ok()
- .and_then(|s| s.trim().parse::<u64>().ok())
- .unwrap_or(0)
- != 0;
+ let ok = read_count(&focus_pid_file()) != 0 && read_focus_hwnd() != 0;
if ok {
break;
}
@@ -254,6 +296,9 @@ fn setup(target_exe: &Path, _title_hint: &str) -> Option<E2eFixture> {
std::thread::sleep(Duration::from_millis(100));
}
std::thread::sleep(Duration::from_millis(400));
+ if !require_focus_monitor_foreground("modality_input_e2e_test sentinel", read_focus_hwnd()) {
+ return None;
+ }
let _ = fs::write(loss_file(), "0");
let _ = fs::write(key_loss_file(), "0");
@@ -499,8 +544,9 @@ fn e2e_electron_background_type_text_no_z_raise() {
}
}
-/// Electron key-combo (Ctrl+A): the dropped-PostMessage keyboard path now uses
-/// cloaked focus + SendInput. Hard invariant: no z-raise.
+/// Electron key-combo (Ctrl+A): background Chromium key-combos are currently
+/// outside the safe delivery capability. The contract is an explicit refusal,
+/// with no z-raise or silent partial chord.
#[test]
#[ignore]
fn e2e_electron_background_keycombo_no_z_raise() {
@@ -523,10 +569,11 @@ fn e2e_electron_background_keycombo_no_z_raise() {
errored = r.is_error();
last_text = r.text().to_string();
});
- // Capability-first: the combo must be DELIVERED, not refused.
+ // W2: refuse honestly instead of claiming delivery through a route that
+ // cannot preserve the background/no-focus contract for Chromium.
assert!(
- !errored && !last_text.contains("background_unavailable"),
- "Ctrl+A must be delivered (capability over UX), got: {last_text:?}"
+ !errored && last_text.contains("Background delivery is not available"),
+ "Ctrl+A must return a structured background capability refusal, got: {last_text:?}"
);
- println!("electron Ctrl+A delivered: {last_text:?}");
+ println!("electron Ctrl+A refused honestly: {last_text:?}");
}
diff --git a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs
index 389ab809bf..bc1deb68c5 100644
--- a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs
+++ b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs
@@ -22,9 +22,9 @@ fn main() {
#[cfg(target_os = "windows")]
mod win {
- use std::sync::atomic::{AtomicU32, Ordering};
use std::ffi::OsStr;
use std::os::windows::ffi::OsStrExt;
+ use std::sync::atomic::{AtomicU32, Ordering};
use windows::Win32::Foundation::*;
use windows::Win32::Graphics::Gdi::*;
use windows::Win32::System::Threading::GetCurrentProcessId;
@@ -32,14 +32,22 @@ mod win {
// ── global loss counters ─────────────────────────────────────────────────
static ACTIVATE_LOSSES: AtomicU32 = AtomicU32::new(0);
- static ACTIVATE_GAINS: AtomicU32 = AtomicU32::new(0);
- static KEY_LOSSES: AtomicU32 = AtomicU32::new(0);
- static KEY_GAINS: AtomicU32 = AtomicU32::new(0);
+ static ACTIVATE_GAINS: AtomicU32 = AtomicU32::new(0);
+ static KEY_LOSSES: AtomicU32 = AtomicU32::new(0);
+ static KEY_GAINS: AtomicU32 = AtomicU32::new(0);
- fn loss_file() -> std::path::PathBuf { loss_path("focus_monitor_losses.txt") }
- fn gain_file() -> std::path::PathBuf { loss_path("focus_monitor_gains.txt") }
- fn key_loss_file() -> std::path::PathBuf { loss_path("focus_monitor_key_losses.txt") }
- fn key_gain_file() -> std::path::PathBuf { loss_path("focus_monitor_key_gains.txt") }
+ fn loss_file() -> std::path::PathBuf {
+ loss_path("focus_monitor_losses.txt")
+ }
+ fn gain_file() -> std::path::PathBuf {
+ loss_path("focus_monitor_gains.txt")
+ }
+ fn key_loss_file() -> std::path::PathBuf {
+ loss_path("focus_monitor_key_losses.txt")
+ }
+ fn key_gain_file() -> std::path::PathBuf {
+ loss_path("focus_monitor_key_gains.txt")
+ }
fn loss_path(name: &str) -> std::path::PathBuf {
let mut p = std::env::temp_dir();
@@ -52,11 +60,17 @@ mod win {
}
fn wide(s: &str) -> Vec<u16> {
- OsStr::new(s).encode_wide().chain(std::iter::once(0)).collect()
+ OsStr::new(s)
+ .encode_wide()
+ .chain(std::iter::once(0))
+ .collect()
}
unsafe extern "system" fn wnd_proc(
- hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM,
+ hwnd: HWND,
+ msg: u32,
+ wparam: WPARAM,
+ lparam: LPARAM,
) -> LRESULT {
match msg {
WM_ACTIVATE => {
@@ -90,8 +104,8 @@ mod win {
let text = wide(&format!(
"act: {act_l}L / {act_g}G key: {key_l}L / {key_g}G (should stay net 0)"
));
- TextOutW(hdc, 10, 10, &text);
- EndPaint(hwnd, &ps);
+ let _ = TextOutW(hdc, 10, 10, &text);
+ let _ = EndPaint(hwnd, &ps);
}
WM_DESTROY => {
PostQuitMessage(0);
@@ -106,8 +120,8 @@ mod win {
let class_name = wide("FocusMonitorWin");
let wc = WNDCLASSW {
- lpfnWndProc: Some(wnd_proc),
- hInstance: HINSTANCE(std::ptr::null_mut()),
+ lpfnWndProc: Some(wnd_proc),
+ hInstance: HINSTANCE(std::ptr::null_mut()),
lpszClassName: windows::core::PCWSTR(class_name.as_ptr()),
hbrBackground: HBRUSH(COLOR_WINDOW.0 as *mut _),
..Default::default()
@@ -120,14 +134,19 @@ mod win {
windows::core::PCWSTR(class_name.as_ptr()),
windows::core::PCWSTR(title.as_ptr()),
WS_OVERLAPPEDWINDOW,
- 100, 100, 600, 140,
- None, None,
+ 100,
+ 100,
+ 600,
+ 140,
+ None,
+ None,
HINSTANCE(std::ptr::null_mut()),
None,
- ).expect("CreateWindowExW failed");
+ )
+ .expect("CreateWindowExW failed");
- ShowWindow(hwnd, SW_SHOWNORMAL);
- UpdateWindow(hwnd).ok();
+ let _ = ShowWindow(hwnd, SW_SHOWNORMAL);
+ let _ = UpdateWindow(hwnd).ok();
// Write initial zeros so tests can read even before any event.
write_count(&loss_file(), 0);
@@ -139,9 +158,9 @@ mod win {
// when stdout is captured by the test runner in sandbox environments).
let pid = GetCurrentProcessId();
let hwnd_val = hwnd.0 as usize;
- let pid_file = std::env::temp_dir().join("focus_monitor_pid.txt");
+ let pid_file = std::env::temp_dir().join("focus_monitor_pid.txt");
let hwnd_file = std::env::temp_dir().join("focus_monitor_hwnd.txt");
- let _ = std::fs::write(&pid_file, pid.to_string());
+ let _ = std::fs::write(&pid_file, pid.to_string());
let _ = std::fs::write(&hwnd_file, hwnd_val.to_string());
// Also print to stdout as a secondary signal.
println!("FOCUS_PID={pid}");
@@ -152,7 +171,7 @@ mod win {
// Message loop.
let mut msg = MSG::default();
while GetMessageW(&mut msg, None, 0, 0).as_bool() {
- TranslateMessage(&msg);
+ let _ = TranslateMessage(&msg);
DispatchMessageW(&msg);
}
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/diagnostics.rs b/libs/cua-driver/rust/crates/platform-windows/src/diagnostics.rs
index 556a5d5373..3348409879 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/diagnostics.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/diagnostics.rs
@@ -19,7 +19,7 @@
//! the user's session) instead of debugging a non-bug.
#[cfg(target_os = "windows")]
-use windows::core::PCWSTR;
+use windows::Win32::Foundation::HANDLE;
#[cfg(target_os = "windows")]
use windows::Win32::System::Com::{
CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER,
@@ -29,15 +29,75 @@ use windows::Win32::System::Com::{
use windows::Win32::System::RemoteDesktop::ProcessIdToSessionId;
#[cfg(target_os = "windows")]
use windows::Win32::System::StationsAndDesktops::{
- CloseWindowStation, OpenWindowStationW,
+ CloseDesktop, GetProcessWindowStation, GetThreadDesktop, GetUserObjectInformationW,
+ OpenInputDesktop, DESKTOP_CONTROL_FLAGS, DESKTOP_READOBJECTS, HDESK, UOI_NAME,
};
#[cfg(target_os = "windows")]
-use windows::Win32::System::Threading::GetCurrentProcessId;
+use windows::Win32::System::Threading::{GetCurrentProcessId, GetCurrentThreadId};
#[cfg(target_os = "windows")]
use windows::Win32::UI::Accessibility::{CUIAutomation, IUIAutomation};
#[cfg(target_os = "windows")]
use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow;
+/// Snapshot of the desktop context visible to the calling process.
+#[derive(Debug, Clone)]
+pub struct DesktopState {
+ pub session_id: Option<u32>,
+ pub has_process_window_station: bool,
+ pub process_window_station_error: Option<String>,
+ pub thread_desktop_name: Option<String>,
+ pub thread_desktop_error: Option<String>,
+ pub input_desktop_name: Option<String>,
+ pub input_desktop_error: Option<String>,
+ pub foreground_hwnd: Option<usize>,
+}
+
+impl DesktopState {
+ /// True when the session input desktop is the normal user desktop.
+ pub fn input_desktop_is_default(&self) -> bool {
+ self.input_desktop_name
+ .as_deref()
+ .map(|name| name.eq_ignore_ascii_case("Default"))
+ .unwrap_or(false)
+ }
+
+ pub fn has_foreground_window(&self) -> bool {
+ self.foreground_hwnd.is_some()
+ }
+
+ pub fn summary(&self) -> String {
+ fn describe(name: &Option<String>, error: &Option<String>) -> String {
+ match (name, error) {
+ (Some(name), _) => name.clone(),
+ (None, Some(error)) => format!("error({error})"),
+ (None, None) => "unknown".to_owned(),
+ }
+ }
+
+ let session = self
+ .session_id
+ .map(|sid| sid.to_string())
+ .unwrap_or_else(|| "unknown".to_owned());
+ let window_station = if self.has_process_window_station {
+ "attached".to_owned()
+ } else {
+ describe(&None, &self.process_window_station_error)
+ };
+ let thread_desktop = describe(&self.thread_desktop_name, &self.thread_desktop_error);
+ let input_desktop = describe(&self.input_desktop_name, &self.input_desktop_error);
+ let foreground = self
+ .foreground_hwnd
+ .map(|hwnd| format!("0x{hwnd:x}"))
+ .unwrap_or_else(|| "0".to_owned());
+
+ format!(
+ "session={session}; window_station={window_station}; \
+ thread_desktop={thread_desktop}; input_desktop={input_desktop}; \
+ foreground_hwnd={foreground}"
+ )
+ }
+}
+
/// Session ID of the calling process via `ProcessIdToSessionId`.
///
/// Returns `None` only when the API call itself fails — a healthy process
@@ -55,31 +115,104 @@ pub fn current_session_id() -> Option<u32> {
}
}
+#[cfg(target_os = "windows")]
+unsafe fn desktop_name(handle: HDESK) -> Result<String, String> {
+ let mut buf = vec![0u16; 256];
+ let mut needed = 0u32;
+ GetUserObjectInformationW(
+ HANDLE(handle.0),
+ UOI_NAME,
+ Some(buf.as_mut_ptr() as *mut std::ffi::c_void),
+ (buf.len() * std::mem::size_of::<u16>()) as u32,
+ Some(&mut needed),
+ )
+ .map_err(|e| format!("{e}"))?;
+
+ let end = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());
+ Ok(String::from_utf16_lossy(&buf[..end]))
+}
+
+#[cfg(target_os = "windows")]
+pub fn desktop_state() -> DesktopState {
+ unsafe {
+ let session_id = current_session_id();
+ let (has_process_window_station, process_window_station_error) =
+ match GetProcessWindowStation() {
+ Ok(h) if !h.0.is_null() => (true, None),
+ Ok(_) => (
+ false,
+ Some("GetProcessWindowStation returned null".to_owned()),
+ ),
+ Err(e) => (false, Some(format!("GetProcessWindowStation: {e}"))),
+ };
+
+ let (thread_desktop_name, thread_desktop_error) =
+ match GetThreadDesktop(GetCurrentThreadId()) {
+ Ok(h) => match desktop_name(h) {
+ Ok(name) => (Some(name), None),
+ Err(e) => (None, Some(format!("GetThreadDesktop name: {e}"))),
+ },
+ Err(e) => (None, Some(format!("GetThreadDesktop: {e}"))),
+ };
+
+ let (input_desktop_name, input_desktop_error) = match OpenInputDesktop(
+ DESKTOP_CONTROL_FLAGS(0),
+ false,
+ DESKTOP_READOBJECTS,
+ ) {
+ Ok(h) => {
+ let name = desktop_name(h);
+ let _ = CloseDesktop(h);
+ match name {
+ Ok(name) => (Some(name), None),
+ Err(e) => (None, Some(format!("OpenInputDesktop name: {e}"))),
+ }
+ }
+ Err(e) => (None, Some(format!("OpenInputDesktop: {e}"))),
+ };
+
+ let fg = GetForegroundWindow();
+ let foreground_hwnd = (!fg.0.is_null()).then_some(fg.0 as usize);
+
+ DesktopState {
+ session_id,
+ has_process_window_station,
+ process_window_station_error,
+ thread_desktop_name,
+ thread_desktop_error,
+ input_desktop_name,
+ input_desktop_error,
+ foreground_hwnd,
+ }
+ }
+}
+
/// Whether the calling process has an attached interactive desktop.
///
/// Two-step check:
-/// 1. `OpenWindowStationW(L"WinSta0", ...)` — succeeds when the
-/// interactive window station exists and we can open a handle.
+/// 1. Session id is not `0` (services). SSH-launched Windows processes
+/// normally land in Session 0 and cannot drive the user's desktop.
/// 2. `GetForegroundWindow()` — returns a non-null HWND when a desktop
/// with a foreground window is actually attached (i.e. an
/// interactive logon session is active, not just a locked screen
/// with no foreground app).
///
-/// `Ok(true)` only when both succeed. `Ok(false)` when `OpenWindowStationW`
-/// succeeded but `GetForegroundWindow` returned null (locked desktop or
-/// transient state). `Err` when `OpenWindowStationW` failed outright.
+/// `Ok(true)` only when both succeed. `Ok(false)` when there is a non-service
+/// session with an attached window station but `GetForegroundWindow` returned
+/// null (locked desktop or transient state). `Err` when the process is in
+/// Session 0 or has no attached window station.
#[cfg(target_os = "windows")]
pub fn interactive_desktop_check() -> Result<bool, String> {
- unsafe {
- let name: Vec<u16> = "WinSta0\0".encode_utf16().collect();
- // 0 access mask = read-only probe; we never actually need to read
- // or modify the station, just confirm a handle is openable.
- let h = OpenWindowStationW(PCWSTR(name.as_ptr()), false, 0)
- .map_err(|e| format!("OpenWindowStationW(WinSta0): {e}"))?;
- let fg = GetForegroundWindow();
- let _ = CloseWindowStation(h);
- Ok(fg.0 != std::ptr::null_mut())
+ let state = desktop_state();
+ if state.session_id == Some(0) {
+ return Err("running in Windows Session 0".to_owned());
+ }
+ if !state.has_process_window_station {
+ return Err(state
+ .process_window_station_error
+ .unwrap_or_else(|| "GetProcessWindowStation returned null".to_owned()));
}
+ Ok(state.has_foreground_window())
}
/// Probe whether `CoCreateInstance(CUIAutomation)` succeeds — the same
@@ -135,6 +268,20 @@ pub fn current_session_id() -> Option<u32> {
None
}
+#[cfg(not(target_os = "windows"))]
+pub fn desktop_state() -> DesktopState {
+ DesktopState {
+ session_id: None,
+ has_process_window_station: false,
+ process_window_station_error: Some("not a Windows host".to_owned()),
+ thread_desktop_name: None,
+ thread_desktop_error: Some("not a Windows host".to_owned()),
+ input_desktop_name: None,
+ input_desktop_error: Some("not a Windows host".to_owned()),
+ foreground_hwnd: None,
+ }
+}
+
#[cfg(not(target_os = "windows"))]
pub fn interactive_desktop_check() -> Result<bool, String> {
Err("not a Windows host".to_owned())
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs
index bea65de2f4..89b11c86ec 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs
@@ -53,12 +53,12 @@ pub enum EventKind {
impl EventKind {
pub fn name(self) -> &'static str {
match self {
- Self::MouseClick => "mouse_click",
- Self::MouseMove => "mouse_move",
+ Self::MouseClick => "mouse_click",
+ Self::MouseMove => "mouse_move",
Self::MouseScroll => "mouse_scroll",
- Self::Keystroke => "keystroke",
- Self::KeyCombo => "key_combo",
- Self::TextInput => "text_input",
+ Self::Keystroke => "keystroke",
+ Self::KeyCombo => "key_combo",
+ Self::TextInput => "text_input",
}
}
}
@@ -99,7 +99,9 @@ impl DeliveryMode {
Self::parse(args.get("delivery_mode").and_then(|v| v.as_str()))
}
- pub fn is_foreground(self) -> bool { matches!(self, Self::Foreground) }
+ pub fn is_foreground(self) -> bool {
+ matches!(self, Self::Foreground)
+ }
}
/// JSON-schema fragment for the `delivery_mode` field. Include this in every
@@ -144,15 +146,15 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool {
return matches!(kind, MouseClick | MouseMove | MouseScroll | KeyCombo);
}
if is_wpf_target_window(hwnd) {
- // WPF ignores PostMessage mouse (its input manager drops mouse messages
- // unless the live system cursor is over the window — verified: posted
- // WM_MOUSE* raise no WPF events). It must be driven by coordinate-routed
- // system-queue input. We use a PERSISTENT synthetic touch digitizer
- // (see inject::TOUCH_DEV): WPF's stylus stack binds to the standing
- // device and consumes the contact as touch/stylus — promoting to mouse
- // internally, with NO OS cursor movement. WM_CHAR keystrokes still work,
- // so flag only the pointer-class events.
- return matches!(kind, MouseClick | MouseMove | MouseScroll);
+ // WPF ignores posted pointer messages (its input manager drops
+ // WM_MOUSE* unless the live system cursor is over the window). It must
+ // be driven by coordinate-routed system-queue input for clicks/moves.
+ //
+ // Do not classify WM_VSCROLL/WM_HSCROLL here: the scroll tool posts the
+ // scrollbar messages directly to the top-level HWND, and WPF hosts that
+ // explicitly handle those messages (including our harness hook) can
+ // consume them without a foreground swap.
+ return matches!(kind, MouseClick | MouseMove);
}
// NB: WinUI3 (`WinUIDesktopWin32WindowClass`) is deliberately NOT flagged
// here. It looks WPF-like, but its composition input-site does NOT consume
@@ -288,22 +290,61 @@ pub fn background_unavailable_error(
swap directly by setting delivery_mode:\"foreground\".",
kind.name()
);
- cua_driver_core::protocol::ToolResult::error(text)
- .with_structured(serde_json::json!({
- "code": "background_unavailable",
- "target_class": class,
- "event_kind": kind.name(),
- "suggestion":
- "Either call bring_to_front then retry with delivery_mode:\"foreground\", \
- or accept the foreground swap by setting delivery_mode:\"foreground\" directly.",
- // Windows analog of the macOS escalation signal: this surface drops
- // background input, so the deliberate next rung is foreground delivery.
- "escalation": {
- "recommended": "foreground",
- "reason": "background input is dropped by this surface — re-call \
- delivery_mode:\"foreground\" (or bring_to_front first).",
- },
- }))
+ cua_driver_core::protocol::ToolResult::error(text).with_structured(serde_json::json!({
+ "code": "background_unavailable",
+ "target_class": class,
+ "event_kind": kind.name(),
+ "suggestion":
+ "Either call bring_to_front then retry with delivery_mode:\"foreground\", \
+ or accept the foreground swap by setting delivery_mode:\"foreground\" directly.",
+ // Windows analog of the macOS escalation signal: this surface drops
+ // background input, so the deliberate next rung is foreground delivery.
+ "escalation": {
+ "recommended": "foreground",
+ "reason": "background input is dropped by this surface — re-call \
+ delivery_mode:\"foreground\" (or bring_to_front first).",
+ },
+ }))
+}
+
+/// Build a structured background refusal while preserving the concrete
+/// actuator failure. This is used after a background coordinate-injection
+/// fallback was attempted and failed, so callers can distinguish a true
+/// occlusion miss from a generic class-level refusal.
+pub fn background_unavailable_error_with_cause(
+ hwnd: u64,
+ kind: EventKind,
+ cause: impl Into<String>,
+) -> cua_driver_core::protocol::ToolResult {
+ let class = read_class_name(hwnd);
+ let cause = cause.into();
+ let cause_lower = cause.to_ascii_lowercase();
+ let code = if cause_lower.contains("occluded") {
+ "background_occluded"
+ } else if cause_lower.contains("uipi") || cause_lower.contains("higher-integrity") {
+ "background_uipi_blocked"
+ } else {
+ "background_unavailable"
+ };
+ let text = format!(
+ "Background delivery is not available for target window class \
+ '{class}' on this event kind ({}): {cause}",
+ kind.name()
+ );
+ cua_driver_core::protocol::ToolResult::error(text).with_structured(serde_json::json!({
+ "code": code,
+ "target_class": class,
+ "event_kind": kind.name(),
+ "cause": cause,
+ "suggestion":
+ "Either call bring_to_front then retry with delivery_mode:\"foreground\", \
+ or accept the foreground swap by setting delivery_mode:\"foreground\" directly.",
+ "escalation": {
+ "recommended": "foreground",
+ "reason": "background input could not be delivered by the coordinate actuator — \
+ re-call delivery_mode:\"foreground\" (or bring_to_front first).",
+ },
+ }))
}
#[cfg(test)]
@@ -313,10 +354,19 @@ mod tests {
#[test]
fn delivery_mode_parses_known_values() {
let j = |s: &str| serde_json::json!({"delivery_mode": s});
- assert_eq!(DeliveryMode::from_args(&j("background")), DeliveryMode::Background);
- assert_eq!(DeliveryMode::from_args(&j("foreground")), DeliveryMode::Foreground);
+ assert_eq!(
+ DeliveryMode::from_args(&j("background")),
+ DeliveryMode::Background
+ );
+ assert_eq!(
+ DeliveryMode::from_args(&j("foreground")),
+ DeliveryMode::Foreground
+ );
// Case-insensitive, matching macOS DeliveryMode::parse.
- assert_eq!(DeliveryMode::from_args(&j("Foreground")), DeliveryMode::Foreground);
+ assert_eq!(
+ DeliveryMode::from_args(&j("Foreground")),
+ DeliveryMode::Foreground
+ );
}
#[test]
@@ -324,7 +374,10 @@ mod tests {
// Missing field, garbage value, null, and the removed legacy "auto"
// all resolve to Background. This is the cua-driver no-foreground-by-
// default contract: an unrecognised value never silently fronts.
- assert_eq!(DeliveryMode::from_args(&serde_json::json!({})), DeliveryMode::Background);
+ assert_eq!(
+ DeliveryMode::from_args(&serde_json::json!({})),
+ DeliveryMode::Background
+ );
assert_eq!(
DeliveryMode::from_args(&serde_json::json!({"delivery_mode": "garbage"})),
DeliveryMode::Background
@@ -338,4 +391,44 @@ mod tests {
DeliveryMode::Background
);
}
+
+ #[test]
+ fn background_unavailable_with_cause_preserves_actuator_failure() {
+ let result = background_unavailable_error_with_cause(
+ 0,
+ EventKind::MouseClick,
+ "target point is occluded by another window",
+ );
+ let structured = result.structured_content.as_ref().expect("structured");
+ assert_eq!(result.is_error, Some(true));
+ assert_eq!(
+ structured["code"].as_str(),
+ Some("background_occluded")
+ );
+ assert_eq!(structured["event_kind"].as_str(), Some("mouse_click"));
+ assert!(
+ structured["cause"]
+ .as_str()
+ .unwrap_or_default()
+ .contains("occluded")
+ );
+
+ let text = match &result.content[0] {
+ cua_driver_core::protocol::Content::Text { text, .. } => text,
+ _ => panic!("expected text content"),
+ };
+ assert!(text.contains("occluded"), "result text lost cause: {text}");
+
+ let uipi = background_unavailable_error_with_cause(
+ 0,
+ EventKind::MouseClick,
+ "blocked by UIPI higher-integrity target",
+ );
+ assert_eq!(
+ uipi.structured_content
+ .as_ref()
+ .and_then(|v| v["code"].as_str()),
+ Some("background_uipi_blocked")
+ );
+ }
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs
index 68da4caadd..ffb23cb153 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs
@@ -19,20 +19,18 @@ use anyhow::{bail, Result};
use std::thread::sleep;
use std::time::Duration;
use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};
+use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};
+use windows::Win32::UI::Input::KeyboardAndMouse::GetFocus;
use windows::Win32::UI::Input::KeyboardAndMouse::{
- MapVirtualKeyW, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT,
- KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP,
- KEYEVENTF_SCANCODE, KEYEVENTF_UNICODE, MAPVK_VK_TO_VSC, VIRTUAL_KEY,
-};
-use windows::Win32::UI::WindowsAndMessaging::{
- GetForegroundWindow, SetForegroundWindow,
+ MapVirtualKeyW, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYBD_EVENT_FLAGS,
+ KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE, KEYEVENTF_UNICODE, MAPVK_VK_TO_VSC,
+ VIRTUAL_KEY,
};
use windows::Win32::UI::WindowsAndMessaging::{
- GetClassNameW, GetWindowThreadProcessId, IsChild,
- PostMessageW, WM_CHAR, WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, WM_SYSKEYUP,
+ GetClassNameW, GetWindowThreadProcessId, IsChild, PostMessageW, WM_CHAR, WM_KEYDOWN, WM_KEYUP,
+ WM_SYSKEYDOWN, WM_SYSKEYUP,
};
-use windows::Win32::UI::Input::KeyboardAndMouse::GetFocus;
-use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};
+use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, SetForegroundWindow};
// ── XAML / UWP host detection ────────────────────────────────────────────────
//
@@ -66,14 +64,18 @@ const XAML_HOST_EXES: &[&str] = &[
fn class_name(hwnd: HWND) -> Option<String> {
let mut buf = [0u16; 256];
let n = unsafe { GetClassNameW(hwnd, &mut buf) };
- if n <= 0 { None } else { Some(String::from_utf16_lossy(&buf[..n as usize])) }
+ if n <= 0 {
+ None
+ } else {
+ Some(String::from_utf16_lossy(&buf[..n as usize]))
+ }
}
fn owning_exe_basename(hwnd: HWND) -> Option<String> {
use windows::Win32::Foundation::CloseHandle;
use windows::Win32::System::Threading::{
- OpenProcess, QueryFullProcessImageNameW,
- PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION,
+ OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_FORMAT,
+ PROCESS_QUERY_LIMITED_INFORMATION,
};
let mut pid: u32 = 0;
@@ -85,8 +87,12 @@ fn owning_exe_basename(hwnd: HWND) -> Option<String> {
let mut buf = [0u16; 1024];
let mut len: u32 = buf.len() as u32;
let result = unsafe {
- QueryFullProcessImageNameW(handle, PROCESS_NAME_FORMAT(0),
- windows::core::PWSTR(buf.as_mut_ptr()), &mut len)
+ QueryFullProcessImageNameW(
+ handle,
+ PROCESS_NAME_FORMAT(0),
+ windows::core::PWSTR(buf.as_mut_ptr()),
+ &mut len,
+ )
};
let _ = unsafe { CloseHandle(handle) };
if result.is_err() || len == 0 {
@@ -134,10 +140,14 @@ const KEY_DELAY_MS: u64 = 4;
/// We detach immediately after — attaching for the duration of the post
/// would change input-state visibility for the duration.
fn focused_descendant(parent: HWND) -> Option<HWND> {
- if parent.0.is_null() { return None; }
+ if parent.0.is_null() {
+ return None;
+ }
let mut target_pid: u32 = 0;
let target_thread = unsafe { GetWindowThreadProcessId(parent, Some(&mut target_pid)) };
- if target_thread == 0 { return None; }
+ if target_thread == 0 {
+ return None;
+ }
let our_thread = unsafe { GetCurrentThreadId() };
let focused = if our_thread == target_thread {
@@ -148,8 +158,12 @@ fn focused_descendant(parent: HWND) -> Option<HWND> {
let _ = unsafe { AttachThreadInput(our_thread, target_thread, false) };
f
};
- if focused.0.is_null() { return None; }
- if focused == parent { return None; }
+ if focused.0.is_null() {
+ return None;
+ }
+ if focused == parent {
+ return None;
+ }
// Only retarget if focus is genuinely a descendant of `parent` — protects
// against accidentally posting to an unrelated window if the target is
// not the foreground app at the moment.
@@ -193,8 +207,13 @@ unsafe fn post_enter_keystroke(h: HWND) -> Result<()> {
let vk = windows::Win32::UI::Input::KeyboardAndMouse::VK_RETURN;
let scan = MapVirtualKeyW(vk.0 as u32, MAPVK_VK_TO_VSC);
let lp_down = 1u32 | (scan << 16);
- let lp_up = lp_down | (1u32 << 30) | (1u32 << 31);
- PostMessageW(h, WM_KEYDOWN, WPARAM(vk.0 as usize), LPARAM(lp_down as isize))?;
+ let lp_up = lp_down | (1u32 << 30) | (1u32 << 31);
+ PostMessageW(
+ h,
+ WM_KEYDOWN,
+ WPARAM(vk.0 as usize),
+ LPARAM(lp_down as isize),
+ )?;
// Hold time between KEYDOWN and KEYUP — matches `post_key`'s pattern and
// gives the target's message loop time to TranslateMessage the KEYDOWN
// (which synthesizes WM_CHAR(0x0D)) and DispatchMessage the paragraph
@@ -203,7 +222,7 @@ unsafe fn post_enter_keystroke(h: HWND) -> Result<()> {
// next character — visible in the "ABC\nDEF\nGHI" repro as "ABC / DEF /
// (gap) / HI".
sleep(Duration::from_millis(KEY_DELAY_MS));
- PostMessageW(h, WM_KEYUP, WPARAM(vk.0 as usize), LPARAM(lp_up as isize))?;
+ PostMessageW(h, WM_KEYUP, WPARAM(vk.0 as usize), LPARAM(lp_up as isize))?;
Ok(())
}
@@ -240,7 +259,9 @@ pub fn post_type_text_with_delay(hwnd: u64, text: &str, inter_char_ms: u64) -> R
prev_was_cr = false;
}
'\n' | '\r' => {
- unsafe { post_enter_keystroke(h)?; }
+ unsafe {
+ post_enter_keystroke(h)?;
+ }
prev_was_cr = ch == '\r';
// Extra settle after Enter — paragraph creation in rich
// editors (VCL Writer, Scintilla, RichEdit) is heavier than
@@ -253,7 +274,9 @@ pub fn post_type_text_with_delay(hwnd: u64, text: &str, inter_char_ms: u64) -> R
_ => {
prev_was_cr = false;
let code = ch as u32 as usize;
- unsafe { PostMessageW(h, WM_CHAR, WPARAM(code), LPARAM(1))?; }
+ unsafe {
+ PostMessageW(h, WM_CHAR, WPARAM(code), LPARAM(1))?;
+ }
sleep(Duration::from_millis(KEY_DELAY_MS + inter_char_ms));
}
}
@@ -267,6 +290,11 @@ pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {
anyhow::bail!(msg);
}
let hwnd_win = HWND(hwnd as *mut _);
+ // WebView2/Tauri keeps the editable renderer in a focused child HWND.
+ // Posting only to the top-level frame reports success but never reaches
+ // the renderer; mirror the WM_CHAR path and retarget to that child when
+ // the target thread exposes one.
+ let target = focused_descendant(hwnd_win).unwrap_or(hwnd_win);
let vk = key_name_to_vk(key)?;
let has_alt = modifiers.iter().any(|m| *m == "alt" || *m == "menu");
@@ -274,8 +302,12 @@ pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {
let repeat_lp = |scan: u32, extended: bool, key_up: bool| {
let mut lp: u32 = 1; // repeat count
lp |= scan << 16;
- if extended { lp |= 1 << 24; }
- if key_up { lp |= (1 << 30) | (1 << 31); }
+ if extended {
+ lp |= 1 << 24;
+ }
+ if key_up {
+ lp |= (1 << 30) | (1 << 31);
+ }
LPARAM(lp as isize)
};
@@ -285,25 +317,43 @@ pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> {
(WM_KEYDOWN, WM_KEYUP)
};
- let mod_vks: Vec<VIRTUAL_KEY> = modifiers.iter()
- .filter_map(|m| modifier_vk(m))
- .collect();
+ let mod_vks: Vec<VIRTUAL_KEY> = modifiers.iter().filter_map(|m| modifier_vk(m)).collect();
unsafe {
// Press modifiers.
for mvk in &mod_vks {
let ms = MapVirtualKeyW(mvk.0 as u32, MAPVK_VK_TO_VSC);
- PostMessageW(hwnd_win, down_msg, WPARAM(mvk.0 as usize), repeat_lp(ms, false, false))?;
+ PostMessageW(
+ target,
+ down_msg,
+ WPARAM(mvk.0 as usize),
+ repeat_lp(ms, false, false),
+ )?;
}
// Press key.
- PostMessageW(hwnd_win, down_msg, WPARAM(vk.0 as usize), repeat_lp(scan, is_extended(vk), false))?;
+ PostMessageW(
+ target,
+ down_msg,
+ WPARAM(vk.0 as usize),
+ repeat_lp(scan, is_extended(vk), false),
+ )?;
sleep(Duration::from_millis(KEY_DELAY_MS));
// Release key.
- PostMessageW(hwnd_win, up_msg, WPARAM(vk.0 as usize), repeat_lp(scan, is_extended(vk), true))?;
+ PostMessageW(
+ target,
+ up_msg,
+ WPARAM(vk.0 as usize),
+ repeat_lp(scan, is_extended(vk), true),
+ )?;
// Release modifiers (reverse order).
for mvk in mod_vks.iter().rev() {
let ms = MapVirtualKeyW(mvk.0 as u32, MAPVK_VK_TO_VSC);
- PostMessageW(hwnd_win, up_msg, WPARAM(mvk.0 as usize), repeat_lp(ms, false, true))?;
+ PostMessageW(
+ target,
+ up_msg,
+ WPARAM(mvk.0 as usize),
+ repeat_lp(ms, false, true),
+ )?;
}
}
Ok(())
@@ -344,10 +394,7 @@ pub fn send_key_synthesized(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<
bail!(msg);
}
let key_vk = key_name_to_vk(key)?;
- let mod_vks: Vec<VIRTUAL_KEY> = modifiers
- .iter()
- .filter_map(|m| modifier_vk(m))
- .collect();
+ let mod_vks: Vec<VIRTUAL_KEY> = modifiers.iter().filter_map(|m| modifier_vk(m)).collect();
// Build the INPUT sequence: modifiers down, key down, key up, modifiers up
// (reverse order). Each event sends the scancode + EXTENDEDKEY flag where
@@ -394,7 +441,8 @@ pub fn send_key_synthesized(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<
and route hotkey calls through it. Until then, the calling \
app must already be foreground for delivery_mode:\"foreground\" \
to be safe.",
- target.0, actual_fg.0
+ target.0,
+ actual_fg.0
);
}
@@ -500,7 +548,8 @@ pub fn send_text_synthesized(hwnd: u64, text: &str) -> Result<()> {
the cua-driver-uia worker (UIAccess-manifested PE) and route \
type_text through it. Until then, the calling app must already \
be foreground for delivery_mode:\"foreground\" to be safe.",
- target.0, actual_fg.0
+ target.0,
+ actual_fg.0
);
}
@@ -554,9 +603,15 @@ fn key_input(vk: VIRTUAL_KEY, up: bool) -> INPUT {
let mut flags: KEYBD_EVENT_FLAGS = KEYBD_EVENT_FLAGS(0);
// Scancode is more reliable than VK for some apps. EXTENDEDKEY flag
// makes arrow / nav / right-side modifier keys work correctly.
- if scan != 0 { flags |= KEYEVENTF_SCANCODE; }
- if is_extended(vk) { flags |= KEYEVENTF_EXTENDEDKEY; }
- if up { flags |= KEYEVENTF_KEYUP; }
+ if scan != 0 {
+ flags |= KEYEVENTF_SCANCODE;
+ }
+ if is_extended(vk) {
+ flags |= KEYEVENTF_EXTENDEDKEY;
+ }
+ if up {
+ flags |= KEYEVENTF_KEYUP;
+ }
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
@@ -597,10 +652,23 @@ pub fn modifier_hold_inputs(modifiers: &[&str]) -> (Vec<INPUT>, Vec<INPUT>) {
fn is_extended(vk: VIRTUAL_KEY) -> bool {
use windows::Win32::UI::Input::KeyboardAndMouse::*;
- matches!(vk,
- VK_DELETE | VK_INSERT | VK_HOME | VK_END | VK_PRIOR | VK_NEXT |
- VK_UP | VK_DOWN | VK_LEFT | VK_RIGHT |
- VK_RCONTROL | VK_RMENU | VK_RWIN | VK_NUMLOCK | VK_SNAPSHOT
+ matches!(
+ vk,
+ VK_DELETE
+ | VK_INSERT
+ | VK_HOME
+ | VK_END
+ | VK_PRIOR
+ | VK_NEXT
+ | VK_UP
+ | VK_DOWN
+ | VK_LEFT
+ | VK_RIGHT
+ | VK_RCONTROL
+ | VK_RMENU
+ | VK_RWIN
+ | VK_NUMLOCK
+ | VK_SNAPSHOT
)
}
@@ -622,9 +690,18 @@ fn key_name_to_vk(key: &str) -> Result<VIRTUAL_KEY> {
"down" => VK_DOWN,
"left" => VK_LEFT,
"right" => VK_RIGHT,
- "f1" => VK_F1, "f2" => VK_F2, "f3" => VK_F3, "f4" => VK_F4,
- "f5" => VK_F5, "f6" => VK_F6, "f7" => VK_F7, "f8" => VK_F8,
- "f9" => VK_F9, "f10" => VK_F10, "f11" => VK_F11, "f12" => VK_F12,
+ "f1" => VK_F1,
+ "f2" => VK_F2,
+ "f3" => VK_F3,
+ "f4" => VK_F4,
+ "f5" => VK_F5,
+ "f6" => VK_F6,
+ "f7" => VK_F7,
+ "f8" => VK_F8,
+ "f9" => VK_F9,
+ "f10" => VK_F10,
+ "f11" => VK_F11,
+ "f12" => VK_F12,
"ctrl" | "control" => VK_CONTROL,
"shift" => VK_SHIFT,
"alt" => VK_MENU,
@@ -633,12 +710,13 @@ fn key_name_to_vk(key: &str) -> Result<VIRTUAL_KEY> {
"numlock" => VK_NUMLOCK,
_ => {
// Single printable character.
- let ch = key.chars().next()
+ let ch = key
+ .chars()
+ .next()
.ok_or_else(|| anyhow::anyhow!("Empty key name"))?;
// VkKeyScanW returns VK in low byte.
- let vk_scan = unsafe {
- windows::Win32::UI::Input::KeyboardAndMouse::VkKeyScanW(ch as u16)
- };
+ let vk_scan =
+ unsafe { windows::Win32::UI::Input::KeyboardAndMouse::VkKeyScanW(ch as u16) };
if vk_scan == -1i16 as u16 as i16 {
bail!("Unknown key: {key}");
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index f1b9b84bd3..610ca60791 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -17,8 +17,15 @@ fn pin_overlay_above(key: &str, hwnd: u64) {
use windows::Win32::Foundation::HWND;
use windows::Win32::UI::WindowsAndMessaging::{GetAncestor, GA_ROOT};
let root = unsafe { GetAncestor(HWND(hwnd as *mut _), GA_ROOT) };
- let wid = if !root.0.is_null() { root.0 as u64 } else { hwnd };
- crate::overlay::send_command(key.to_owned(), cursor_overlay::OverlayCommand::PinAbove(wid));
+ let wid = if !root.0.is_null() {
+ root.0 as u64
+ } else {
+ hwnd
+ };
+ crate::overlay::send_command(
+ key.to_owned(),
+ cursor_overlay::OverlayCommand::PinAbove(wid),
+ );
}
/// Resolve the screen point a coordinate action should actuate at, scrolling
@@ -113,8 +120,10 @@ fn bitmap_to_screen(hwnd: u64, px: i32, py: i32) -> (i32, i32) {
std::mem::size_of::<RECT>() as u32,
);
if hr.is_ok() {
- return (dwm.left + DWM_CROP_INSET_PX + px,
- dwm.top + DWM_CROP_INSET_PX + py);
+ return (
+ dwm.left + DWM_CROP_INSET_PX + px,
+ dwm.top + DWM_CROP_INSET_PX + py,
+ );
}
// Fallback path — capture also keeps the full bitmap when DWM
// bounds aren't available, so the bitmap origin IS GetWindowRect
@@ -142,12 +151,19 @@ fn bitmap_to_screen(hwnd: u64, px: i32, py: i32) -> (i32, i32) {
/// (anonymous, no declared session) is cursor-less: every overlay op
/// short-circuits, so the action runs with no visible cursor.
async fn overlay_glide_to(key: &str, sx: f64, sy: f64) {
- if key.is_empty() { return; }
- if !crate::overlay::is_enabled(key) { return; }
+ if key.is_empty() {
+ return;
+ }
+ if !crate::overlay::is_enabled(key) {
+ return;
+ }
let pos = crate::overlay::current_position(key);
if pos.0 < 0.0 && pos.1 < 0.0 {
// Snap to target on first use; no animation to wait for.
- crate::overlay::send_command(key.to_owned(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });
+ crate::overlay::send_command(
+ key.to_owned(),
+ cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
+ );
return;
}
// Fixed-duration glide → decouple the click from the render thread. When
@@ -169,12 +185,18 @@ async fn overlay_glide_to(key: &str, sx: f64, sy: f64) {
end_heading_radians: std::f64::consts::FRAC_PI_4,
},
);
- tokio::time::sleep(std::time::Duration::from_millis(motion.glide_duration_ms as u64)).await;
+ tokio::time::sleep(std::time::Duration::from_millis(
+ motion.glide_duration_ms as u64,
+ ))
+ .await;
return;
}
crate::overlay::animate_cursor_to(key.to_owned(), sx, sy).await;
}
-use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};
+use cua_driver_core::{
+ protocol::ToolResult,
+ tool::{Tool, ToolDef, ToolRegistry},
+};
use serde_json::{json, Value};
use std::sync::{Arc, RwLock};
@@ -242,7 +264,13 @@ pub struct DriverConfig {
}
impl Default for DriverConfig {
- fn default() -> Self { Self { capture_mode: "ax".into(), capture_scope: "window".into(), max_image_dimension: 1568 } }
+ fn default() -> Self {
+ Self {
+ capture_mode: "ax".into(),
+ capture_scope: "window".into(),
+ max_image_dimension: 1568,
+ }
+ }
}
/// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to
@@ -254,14 +282,21 @@ impl Default for DriverConfig {
/// affects behavior — `get_window_state` always returns tree + screenshot.)
pub fn load_driver_config() -> DriverConfig {
let mut cfg = DriverConfig::default();
- if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) {
+ if let Some(v) =
+ pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned))
+ {
cfg.capture_mode = v;
}
- if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) {
+ if let Some(v) =
+ pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned))
+ {
cfg.capture_scope = v;
}
- if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) {
- if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; }
+ if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64())
+ {
+ if let Ok(v32) = u32::try_from(v) {
+ cfg.max_image_dimension = v32;
+ }
}
cfg
}
@@ -271,10 +306,20 @@ pub struct ResizeRegistry {
}
impl ResizeRegistry {
- pub fn new() -> Self { Self { ratios: std::sync::Mutex::new(Default::default()) } }
- pub fn set_ratio(&self, pid: u32, ratio: f64) { self.ratios.lock().unwrap().insert(pid, ratio); }
- pub fn clear_ratio(&self, pid: u32) { self.ratios.lock().unwrap().remove(&pid); }
- pub fn ratio(&self, pid: u32) -> Option<f64> { self.ratios.lock().unwrap().get(&pid).copied() }
+ pub fn new() -> Self {
+ Self {
+ ratios: std::sync::Mutex::new(Default::default()),
+ }
+ }
+ pub fn set_ratio(&self, pid: u32, ratio: f64) {
+ self.ratios.lock().unwrap().insert(pid, ratio);
+ }
+ pub fn clear_ratio(&self, pid: u32) {
+ self.ratios.lock().unwrap().remove(&pid);
+ }
+ pub fn ratio(&self, pid: u32) -> Option<f64> {
+ self.ratios.lock().unwrap().get(&pid).copied()
+ }
}
/// Per-process zoom context — stores padded crop origin and resize scale from
@@ -290,7 +335,10 @@ pub struct ZoomContext {
impl ZoomContext {
pub fn zoom_to_window(&self, px: f64, py: f64) -> (f64, f64) {
- (self.origin_x + px * self.scale_inv, self.origin_y + py * self.scale_inv)
+ (
+ self.origin_x + px * self.scale_inv,
+ self.origin_y + py * self.scale_inv,
+ )
}
}
@@ -299,9 +347,17 @@ pub struct ZoomRegistry {
}
impl ZoomRegistry {
- pub fn new() -> Self { Self { inner: std::sync::Mutex::new(Default::default()) } }
- pub fn set(&self, pid: u32, ctx: ZoomContext) { self.inner.lock().unwrap().insert(pid, ctx); }
- pub fn get(&self, pid: u32) -> Option<ZoomContext> { self.inner.lock().unwrap().get(&pid).copied() }
+ pub fn new() -> Self {
+ Self {
+ inner: std::sync::Mutex::new(Default::default()),
+ }
+ }
+ pub fn set(&self, pid: u32, ctx: ZoomContext) {
+ self.inner.lock().unwrap().insert(pid, ctx);
+ }
+ pub fn get(&self, pid: u32) -> Option<ZoomContext> {
+ self.inner.lock().unwrap().get(&pid).copied()
+ }
}
pub struct ToolState {
@@ -335,7 +391,8 @@ impl Tool for ListAppsTool {
fn def(&self) -> &ToolDef {
LIST_APPS_DEF.get_or_init(|| ToolDef {
name: "list_apps".into(),
- description: "List Windows apps — both currently running and installed-but-not-running — \
+ description:
+ "List Windows apps — both currently running and installed-but-not-running — \
with per-app state flags:\n\n\
- running: is a process for this app live? (pid is 0 when false)\n\
- active: is it the system-frontmost app? (implies running)\n\
@@ -358,9 +415,13 @@ impl Tool for ListAppsTool {
Use this for \"is X installed?\" as well as \"is X running?\". For per-window \
state — on-screen, minimized, window titles — call list_windows instead. \
For just opening an app — running or not — call launch_app({path: ...}) \
- directly; list_apps is not a prerequisite.".into(),
+ directly; list_apps is not a prerequisite."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
@@ -487,15 +548,21 @@ impl Tool for ListAppsTool {
out
}).await.unwrap_or_default();
- let running_count = apps.iter().filter(|a| a["running"].as_bool().unwrap_or(false)).count();
+ let running_count = apps
+ .iter()
+ .filter(|a| a["running"].as_bool().unwrap_or(false))
+ .count();
let total = apps.len();
let installed_only = total - running_count;
let mut lines = vec![format!(
"✅ Found {total} app(s): {running_count} running, {installed_only} installed-not-running."
)];
- for app in apps.iter().filter(|a| a["running"].as_bool().unwrap_or(false)) {
+ for app in apps
+ .iter()
+ .filter(|a| a["running"].as_bool().unwrap_or(false))
+ {
let name = app["name"].as_str().unwrap_or("?");
- let pid = app["pid"].as_u64().unwrap_or(0);
+ let pid = app["pid"].as_u64().unwrap_or(0);
lines.push(format!("- {name} (pid {pid})"));
}
// `apps` is the unified shape; `processes` retained for any pre-existing
@@ -524,8 +591,12 @@ fn disambiguate_installed_match(
installed: &[crate::win32::InstalledApp],
_proc_exe_basename: &str,
) -> Option<usize> {
- if candidates.is_empty() { return None; }
- if candidates.len() == 1 { return Some(candidates[0]); }
+ if candidates.is_empty() {
+ return None;
+ }
+ if candidates.len() == 1 {
+ return Some(candidates[0]);
+ }
candidates
.iter()
@@ -585,7 +656,9 @@ impl Tool for ListWindowsTool {
let map: std::collections::HashMap<u32, String> =
procs.into_iter().map(|p| (p.pid, p.name)).collect();
(wins, map)
- }).await.unwrap_or_default();
+ })
+ .await
+ .unwrap_or_default();
// Swift surfaces a warning when a pid filter matches nothing.
if let Some(fp) = filter_pid {
@@ -617,24 +690,35 @@ impl Tool for ListWindowsTool {
// Invert via `(len - 1 - i)` so the front-most window gets the
// largest z.
let n = windows.len();
- let records: Vec<serde_json::Value> = windows.iter().enumerate().map(|(i, w)| {
- let app_name = pid_to_name.get(&w.pid).cloned().unwrap_or_default();
- let z_index = (n.saturating_sub(1).saturating_sub(i)) as i64;
- json!({
- "window_id": w.hwnd,
- "pid": w.pid,
- "app_name": app_name,
- "title": w.title,
- "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height },
- "layer": 0,
- "z_index": z_index,
- "is_on_screen": true,
+ let records: Vec<serde_json::Value> = windows
+ .iter()
+ .enumerate()
+ .map(|(i, w)| {
+ let app_name = pid_to_name.get(&w.pid).cloned().unwrap_or_default();
+ let z_index = (n.saturating_sub(1).saturating_sub(i)) as i64;
+ json!({
+ "window_id": w.hwnd,
+ "pid": w.pid,
+ "app_name": app_name,
+ "title": w.title,
+ "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height },
+ "layer": 0,
+ "z_index": z_index,
+ "is_on_screen": true,
+ })
})
- }).collect();
+ .collect();
let total = records.len();
- let pid_count = windows.iter().map(|w| w.pid).collect::<std::collections::HashSet<_>>().len();
- let on_screen = records.iter().filter(|r| r["is_on_screen"].as_bool().unwrap_or(false)).count();
+ let pid_count = windows
+ .iter()
+ .map(|w| w.pid)
+ .collect::<std::collections::HashSet<_>>()
+ .len();
+ let on_screen = records
+ .iter()
+ .filter(|r| r["is_on_screen"].as_bool().unwrap_or(false))
+ .count();
// Swift text format 1:1 — append the SPI-unavailable note since
// Windows has no equivalent of macOS Spaces.
let header = format!(
@@ -643,21 +727,36 @@ impl Tool for ListWindowsTool {
);
let mut lines = vec![header];
for r in &records {
- let app = r["app_name"].as_str().unwrap_or("?");
- let pid = r["pid"].as_u64().unwrap_or(0);
- let tt = r["title"].as_str().unwrap_or("");
- let wid = r["window_id"].as_u64().unwrap_or(0);
- let title_disp = if tt.is_empty() { "(no title)".to_owned() } else { format!("\"{tt}\"") };
- let tag = if r["is_on_screen"].as_bool().unwrap_or(false) { "" } else { " [off-screen]" };
- lines.push(format!("- {app} (pid {pid}) {title_disp} [window_id: {wid}]{tag}"));
+ let app = r["app_name"].as_str().unwrap_or("?");
+ let pid = r["pid"].as_u64().unwrap_or(0);
+ let tt = r["title"].as_str().unwrap_or("");
+ let wid = r["window_id"].as_u64().unwrap_or(0);
+ let title_disp = if tt.is_empty() {
+ "(no title)".to_owned()
+ } else {
+ format!("\"{tt}\"")
+ };
+ let tag = if r["is_on_screen"].as_bool().unwrap_or(false) {
+ ""
+ } else {
+ " [off-screen]"
+ };
+ lines.push(format!(
+ "- {app} (pid {pid}) {title_disp} [window_id: {wid}]{tag}"
+ ));
}
// Legacy alias: keep the old flat fields under each record for any
// pre-existing Rust callers, but they read from the same source.
- let legacy_windows: Vec<serde_json::Value> = windows.iter().map(|w| json!({
- "window_id": w.hwnd, "pid": w.pid, "title": w.title,
- "x": w.x, "y": w.y, "width": w.width, "height": w.height,
- })).collect();
+ let legacy_windows: Vec<serde_json::Value> = windows
+ .iter()
+ .map(|w| {
+ json!({
+ "window_id": w.hwnd, "pid": w.pid, "title": w.title,
+ "x": w.x, "y": w.y, "width": w.width, "height": w.height,
+ })
+ })
+ .collect();
let structured = json!({
"windows": records,
"current_space_id": serde_json::Value::Null, // Windows has no Spaces
@@ -740,30 +839,37 @@ impl Tool for GetWindowStateTool {
// Swift error wording 1:1.
let pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(v) => v as u32,
- None => return ToolResult::error("Missing required integer field pid."),
- };
- let hwnd = match args.get("window_id").and_then(|v| v.as_u64()) {
- Some(v) => v,
- None => return ToolResult::error(
- "Missing required integer field window_id. Use `list_windows` to enumerate \
- the target app's windows, or read `launch_app`'s `windows` array."),
+ None => return ToolResult::error("Missing required integer field pid."),
};
+ let hwnd =
+ match args.get("window_id").and_then(|v| v.as_u64()) {
+ Some(v) => v,
+ None => return ToolResult::error(
+ "Missing required integer field window_id. Use `list_windows` to enumerate \
+ the target app's windows, or read `launch_app`'s `windows` array.",
+ ),
+ };
// Validate window belongs to pid — Swift's hard error.
- let windows_for_pid = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await.unwrap_or_default();
+ let windows_for_pid =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
if !windows_for_pid.iter().any(|w| w.hwnd == hwnd) {
// Check if the window exists under a different pid.
let all = tokio::task::spawn_blocking(|| crate::win32::list_windows(None))
- .await.unwrap_or_default();
+ .await
+ .unwrap_or_default();
if let Some(w) = all.iter().find(|w| w.hwnd == hwnd) {
return ToolResult::error(format!(
"window_id {hwnd} belongs to pid {}, not pid {pid}. Call \
`list_windows({{\"pid\": {pid}}})` to get this pid's own windows.",
- w.pid));
+ w.pid
+ ));
}
return ToolResult::error(format!(
"No window with window_id {hwnd} exists. Call `list_windows({{\"pid\": \
- {pid}}})` for candidates."));
+ {pid}}})` for candidates."
+ ));
}
let max_dim = {
let cfg = self.state.config.read().unwrap();
@@ -809,7 +915,12 @@ impl Tool for GetWindowStateTool {
let out_file = screenshot_out_file.clone();
let blocking = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let tree_result = if do_tree {
- Some(crate::uia::walk_tree_bounded(hwnd, q.as_deref(), max_elements, max_depth))
+ Some(crate::uia::walk_tree_bounded(
+ hwnd,
+ q.as_deref(),
+ max_elements,
+ max_depth,
+ ))
} else {
None
};
@@ -823,7 +934,9 @@ impl Tool for GetWindowStateTool {
let (screenshot, screenshot_err) = if do_shot {
match crate::capture::screenshot_window_bytes(hwnd) {
Ok(raw) => {
- let orig_w = crate::capture::png_dimensions_pub(&raw).map(|(w, _)| w).unwrap_or(0);
+ let orig_w = crate::capture::png_dimensions_pub(&raw)
+ .map(|(w, _)| w)
+ .unwrap_or(0);
let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;
let (w, h) = crate::capture::png_dimensions_pub(&png)?;
let original_w = if w < orig_w { Some(orig_w) } else { None };
@@ -846,20 +959,19 @@ impl Tool for GetWindowStateTool {
Ok((tree_result, screenshot, screenshot_err))
});
// Timeout: Chrome's UIA provider can block indefinitely on property reads.
- let result: Result<anyhow::Result<_>, _> = match tokio::time::timeout(
- std::time::Duration::from_secs(4), blocking).await
- {
- Ok(join_result) => join_result.map_err(|e| anyhow::anyhow!("task panic: {e}")),
- Err(_elapsed) => {
- // Surface the target's window class + an actionable hint
- // instead of just "UIA provider unresponsive". The class
- // points the caller at the right workaround (e.g. SALFRAME
- // → screenshot + pixel coords + delivery_mode:"foreground"; UWP
- // class → re-call with a depth-limited scan and act by pixel
- // off the screenshot if the tree stays unusable).
- let class = crate::input::delivery::read_class_name(hwnd);
- Err(anyhow::anyhow!(
- "get_window_state timed out after 4s (UIA provider unresponsive on \
+ let result: Result<anyhow::Result<_>, _> =
+ match tokio::time::timeout(std::time::Duration::from_secs(4), blocking).await {
+ Ok(join_result) => join_result.map_err(|e| anyhow::anyhow!("task panic: {e}")),
+ Err(_elapsed) => {
+ // Surface the target's window class + an actionable hint
+ // instead of just "UIA provider unresponsive". The class
+ // points the caller at the right workaround (e.g. SALFRAME
+ // → screenshot + pixel coords + delivery_mode:"foreground"; UWP
+ // class → re-call with a depth-limited scan and act by pixel
+ // off the screenshot if the tree stays unusable).
+ let class = crate::input::delivery::read_class_name(hwnd);
+ Err(anyhow::anyhow!(
+ "get_window_state timed out after 4s (UIA provider unresponsive on \
hwnd 0x{hwnd:x}, class '{class}'). Fallback options: \
(a) re-call this tool with a depth-limited scan \
(`max_elements` / `max_depth`) — if the tree stays unusable, act \
@@ -867,9 +979,9 @@ impl Tool for GetWindowStateTool {
(b) if the target is a transient VCL / message-box dialog, send \
`press_key` with `delivery_mode:\"foreground\"` (SendInput) to fire the \
default accelerator (Esc / Enter / Y / N) without needing the tree."
- ))
- }
- };
+ ))
+ }
+ };
let result = result.and_then(|r| r);
match result {
@@ -878,9 +990,15 @@ impl Tool for GetWindowStateTool {
let mut structured = json!({ "window_id": hwnd, "pid": pid });
if let Some(tr) = tree_opt {
- let count = tr.nodes.iter().filter(|n| n.element_index.is_some()).count();
+ let count = tr
+ .nodes
+ .iter()
+ .filter(|n| n.element_index.is_some())
+ .count();
let header = format!("window_id={hwnd} pid={pid} elements={count}\n\n");
- content.push(cua_driver_core::protocol::Content::text(header + &tr.tree_markdown));
+ content.push(cua_driver_core::protocol::Content::text(
+ header + &tr.tree_markdown,
+ ));
// Route the cache to the matching dispatch path: any
// node whose msaa_role is Some came from the MSAA
// walker, so the entire snapshot must Drop via
@@ -899,8 +1017,11 @@ impl Tool for GetWindowStateTool {
// stores u32 — truncate (HWND fits in 32-bit on
// every supported edition; the upper 32 bits are
// zero in user-space).
- let snapshot_id = cua_driver_core::element_token::global()
- .register_snapshot(pid as i32, hwnd as u32, count);
+ let snapshot_id = cua_driver_core::element_token::global().register_snapshot(
+ pid as i32,
+ hwnd as u32,
+ count,
+ );
// Structured `elements` array — preferred consumption
// path. Shape matches the cross-platform spec:
@@ -955,11 +1076,10 @@ impl Tool for GetWindowStateTool {
.collect();
structured["elements"] = json!(elements);
// Surface 6: snapshot id mirror for debug correlation.
- structured["snapshot_id"] = json!(
- cua_driver_core::element_token::token_for(snapshot_id, 0)
+ structured["snapshot_id"] =
+ json!(cua_driver_core::element_token::token_for(snapshot_id, 0)
.trim_end_matches(":0")
- .to_string()
- );
+ .to_string());
structured["_note"] = json!(
"Prefer `elements` — `tree_markdown` will continue to work \
but new fields will only be added to the structured side. \
@@ -995,7 +1115,9 @@ impl Tool for GetWindowStateTool {
if let Some((b64_opt, file_path, w, h, orig_w)) = screenshot_opt {
if let Some(ow) = orig_w {
- if w > 0 { state.resize_registry.set_ratio(pid, ow as f64 / w as f64); }
+ if w > 0 {
+ state.resize_registry.set_ratio(pid, ow as f64 / w as f64);
+ }
} else {
state.resize_registry.clear_ratio(pid);
}
@@ -1022,13 +1144,17 @@ impl Tool for GetWindowStateTool {
// (so MCP clients with structured-only parsing can detect
// and act on it). Without this the caller saw an empty
// response with no clue why and burned turns retrying.
- content.push(cua_driver_core::protocol::Content::text(
- format!("screenshot unavailable: {err}")
- ));
+ content.push(cua_driver_core::protocol::Content::text(format!(
+ "screenshot unavailable: {err}"
+ )));
structured["screenshot_error"] = json!(err);
}
- ToolResult { content, is_error: None, structured_content: Some(structured) }
+ ToolResult {
+ content,
+ is_error: None,
+ structured_content: Some(structured),
+ }
}
Err(e) => ToolResult::error(format!("Error: {e}")),
}
@@ -1049,7 +1175,9 @@ impl Tool for GetWindowStateTool {
/// `--profile-directory="Profile 2"` survives.
fn split_launchable_target(s: &str) -> (String, String) {
let trimmed = s.trim();
- if trimmed.is_empty() { return (String::new(), String::new()); }
+ if trimmed.is_empty() {
+ return (String::new(), String::new());
+ }
// AUMID / shell launch tokens go through unchanged.
if trimmed.starts_with("shell:") || trimmed.contains('!') {
return (trimmed.to_owned(), String::new());
@@ -1089,7 +1217,11 @@ fn split_launchable_target(s: &str) -> (String, String) {
/// matched via `split_launchable_target`).
fn is_chromium_browser_target(target: &str) -> bool {
let (file, _) = split_launchable_target(target);
- let candidate = if file.is_empty() { target } else { file.as_str() };
+ let candidate = if file.is_empty() {
+ target
+ } else {
+ file.as_str()
+ };
let lower = candidate.to_ascii_lowercase();
let basename = lower
.rsplit_once('\\')
@@ -1151,15 +1283,15 @@ fn inject_chromium_anti_throttling_flags(extra_args: &mut Vec<String>) {
{
extra_args[idx] = format!("{},{OCCLUSION_FEATURE}", extra_args[idx]);
} else {
- extra_args.insert(
- 0,
- format!("--disable-features={OCCLUSION_FEATURE}"),
- );
+ extra_args.insert(0, format!("--disable-features={OCCLUSION_FEATURE}"));
}
}
// Boolean toggles — insert at front only when not already present.
- for flag in &["--disable-backgrounding-occluded-windows", "--disable-renderer-backgrounding"] {
+ for flag in &[
+ "--disable-backgrounding-occluded-windows",
+ "--disable-renderer-backgrounding",
+ ] {
if !extra_args.iter().any(|a| a == flag) {
extra_args.insert(0, (*flag).to_string());
}
@@ -1236,10 +1368,7 @@ fn should_restore_foreground_after_launch(shape: LaunchTargetShape) -> bool {
/// than `HWND` directly) because `HWND` wraps `*mut c_void` which is
/// `!Send` and would prevent this future from being scheduled on the
/// multi-threaded tokio runtime.
-async fn restore_foreground_polling_best_effort(
- prior_foreground_addr: usize,
- spawned_pid: u32,
-) {
+async fn restore_foreground_polling_best_effort(prior_foreground_addr: usize, spawned_pid: u32) {
use windows::Win32::Foundation::{CloseHandle, HWND};
use windows::Win32::System::Threading::{
OpenProcess, WaitForInputIdle, PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE,
@@ -1427,15 +1556,31 @@ impl Tool for LaunchAppTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- let launch_path_opt = args.get("launch_path").and_then(|v| v.as_str()).map(str::to_owned);
+ let launch_path_opt = args
+ .get("launch_path")
+ .and_then(|v| v.as_str())
+ .map(str::to_owned);
let path_opt = args.get("path").and_then(|v| v.as_str()).map(str::to_owned);
let name_opt = args.get("name").and_then(|v| v.as_str()).map(str::to_owned);
- let bundle_id_opt = args.get("bundle_id").and_then(|v| v.as_str()).map(str::to_owned);
- let aumid_opt = args.get("aumid").and_then(|v| v.as_str()).map(str::to_owned);
- let urls: Vec<String> = args.get("urls").and_then(|v| v.as_array())
- .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())
+ let bundle_id_opt = args
+ .get("bundle_id")
+ .and_then(|v| v.as_str())
+ .map(str::to_owned);
+ let aumid_opt = args
+ .get("aumid")
+ .and_then(|v| v.as_str())
+ .map(str::to_owned);
+ let urls: Vec<String> = args
+ .get("urls")
+ .and_then(|v| v.as_array())
+ .map(|a| {
+ a.iter()
+ .filter_map(|v| v.as_str().map(str::to_owned))
+ .collect()
+ })
.unwrap_or_default();
- let start_minimized = args.get("start_minimized")
+ let start_minimized = args
+ .get("start_minimized")
.and_then(|v| v.as_bool())
.unwrap_or(false);
// `nShow` for the ShellExecuteEx-based launches. SW_SHOWNOACTIVATE
@@ -1460,16 +1605,15 @@ impl Tool for LaunchAppTool {
// process with a different name owns the actual window. Captured
// here (before the launch) so the new soffice.bin pid won't be
// in the snapshot.
- let pre_launch_pids: std::sync::Arc<std::collections::HashSet<u32>> =
- if start_minimized {
- let pids: std::collections::HashSet<u32> = crate::win32::list_processes()
- .into_iter()
- .map(|p| p.pid)
- .collect();
- std::sync::Arc::new(pids)
- } else {
- std::sync::Arc::new(std::collections::HashSet::new())
- };
+ let pre_launch_pids: std::sync::Arc<std::collections::HashSet<u32>> = if start_minimized {
+ let pids: std::collections::HashSet<u32> = crate::win32::list_processes()
+ .into_iter()
+ .map(|p| p.pid)
+ .collect();
+ std::sync::Arc::new(pids)
+ } else {
+ std::sync::Arc::new(std::collections::HashSet::new())
+ };
// Capture the foreground window BEFORE any launch path runs so the
// post-spawn polling restore (below) has a target HWND to flip back
@@ -1485,31 +1629,37 @@ impl Tool for LaunchAppTool {
// Stored as `usize` (not `HWND`) so this `invoke` future stays
// `Send` — `HWND` wraps `*mut c_void` and would taint the future
// for the multi-threaded tokio runtime if held across `.await`.
- let foreground_before_addr: usize = unsafe {
- windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
- };
+ let foreground_before_addr: usize =
+ unsafe { windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize };
// Pure-function decision on whether to fire the post-launch
// restore. Shape mirrors the resolved params — see
// `should_restore_foreground_after_launch` for the rule.
let launch_shape = LaunchTargetShape {
- has_aumid: aumid_opt.is_some(),
- has_bundle_id: bundle_id_opt.is_some(),
- has_name: name_opt.is_some(),
- has_path: path_opt.is_some(),
+ has_aumid: aumid_opt.is_some(),
+ has_bundle_id: bundle_id_opt.is_some(),
+ has_name: name_opt.is_some(),
+ has_path: path_opt.is_some(),
has_launch_path: launch_path_opt.is_some(),
- has_urls: !urls.is_empty(),
+ has_urls: !urls.is_empty(),
};
let restore_foreground = should_restore_foreground_after_launch(launch_shape);
- let mut extra_args: Vec<String> = args.get("additional_arguments").and_then(|v| v.as_array())
- .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())
+ let mut extra_args: Vec<String> = args
+ .get("additional_arguments")
+ .and_then(|v| v.as_array())
+ .map(|a| {
+ a.iter()
+ .filter_map(|v| v.as_str().map(str::to_owned))
+ .collect()
+ })
.unwrap_or_default();
// Resolve target — launch_path > path > aumid > name > bundle_id
// (alias of name on Windows). launch_path is highest precedence for
// round-trip with list_apps; the value is handed to ShellExecuteEx
// unchanged (UWP routing below is bypassed when launch_path is set).
- let target = launch_path_opt.clone()
+ let target = launch_path_opt
+ .clone()
.or(path_opt.clone())
.or(aumid_opt.clone())
.or(name_opt.clone())
@@ -1570,7 +1720,10 @@ impl Tool for LaunchAppTool {
None
} else if let Some(a) = aumid_opt.clone() {
Some(a)
- } else if let Some(b) = bundle_id_opt.clone().filter(|s| crate::launch_uwp::is_aumid(s)) {
+ } else if let Some(b) = bundle_id_opt
+ .clone()
+ .filter(|s| crate::launch_uwp::is_aumid(s))
+ {
Some(b)
} else if let Some(n) = name_opt.clone() {
// Run the (cached) AppsFolder lookup on a blocking thread to
@@ -1633,9 +1786,11 @@ impl Tool for LaunchAppTool {
.await;
match activation {
Ok(Ok(p)) => p,
- Ok(Err(e)) => return ToolResult::error(format!(
- "Failed to activate packaged app {aumid:?}: {e}"
- )),
+ Ok(Err(e)) => {
+ return ToolResult::error(format!(
+ "Failed to activate packaged app {aumid:?}: {e}"
+ ))
+ }
Err(e) => return ToolResult::error(format!("Task error: {e}")),
}
} else {
@@ -1656,18 +1811,18 @@ impl Tool for LaunchAppTool {
// backstop for any *other* blocking broker dialog (SmartScreen, an
// elevation/consent surface) so a bad target can't hang the daemon.
let launch = tokio::task::spawn_blocking(move || -> anyhow::Result<u32> {
+ use windows::core::PCWSTR;
+ use windows::Win32::Foundation::CloseHandle;
+ use windows::Win32::System::Threading::GetProcessId;
use windows::Win32::UI::Shell::{
- ShellExecuteExW, SHELLEXECUTEINFOW, SEE_MASK_NOCLOSEPROCESS,
- SEE_MASK_FLAG_NO_UI,
+ ShellExecuteExW, SEE_MASK_FLAG_NO_UI, SEE_MASK_NOCLOSEPROCESS,
+ SHELLEXECUTEINFOW,
};
- use windows::Win32::System::Threading::GetProcessId;
- use windows::Win32::Foundation::CloseHandle;
- use windows::core::PCWSTR;
fn to_wide(s: &str) -> Vec<u16> {
s.encode_utf16().chain(std::iter::once(0)).collect()
}
- let op_w = to_wide("open");
+ let op_w = to_wide("open");
let file_w = to_wide(if let Some(t) = target_for_shell.as_deref() {
t
} else {
@@ -1676,24 +1831,30 @@ impl Tool for LaunchAppTool {
let args_w = to_wide(&extra_for_shell);
let mut info = SHELLEXECUTEINFOW {
- cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32,
- fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI,
- lpVerb: PCWSTR(op_w.as_ptr()),
- lpFile: PCWSTR(file_w.as_ptr()),
+ cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32,
+ fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI,
+ lpVerb: PCWSTR(op_w.as_ptr()),
+ lpFile: PCWSTR(file_w.as_ptr()),
lpParameters: if extra_for_shell.is_empty() {
PCWSTR::null()
} else {
PCWSTR(args_w.as_ptr())
},
- nShow: n_show_for_shell,
+ nShow: n_show_for_shell,
..Default::default()
};
- unsafe { ShellExecuteExW(&mut info)?; }
+ unsafe {
+ ShellExecuteExW(&mut info)?;
+ }
let pid = if !info.hProcess.is_invalid() {
let p = unsafe { GetProcessId(info.hProcess) };
- unsafe { let _ = CloseHandle(info.hProcess); }
+ unsafe {
+ let _ = CloseHandle(info.hProcess);
+ }
p
- } else { 0 };
+ } else {
+ 0
+ };
// Open any additional URLs in the default browser (no focus
// steal, no pid capture for these — Swift's NSWorkspace flow
@@ -1702,13 +1863,15 @@ impl Tool for LaunchAppTool {
let file = to_wide(url);
let mut url_info = SHELLEXECUTEINFOW {
cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32,
- fMask: SEE_MASK_FLAG_NO_UI,
+ fMask: SEE_MASK_FLAG_NO_UI,
lpVerb: PCWSTR(op_w.as_ptr()),
lpFile: PCWSTR(file.as_ptr()),
- nShow: n_show_for_shell,
+ nShow: n_show_for_shell,
..Default::default()
};
- unsafe { let _ = ShellExecuteExW(&mut url_info); }
+ unsafe {
+ let _ = ShellExecuteExW(&mut url_info);
+ }
}
Ok(pid)
@@ -1719,23 +1882,23 @@ impl Tool for LaunchAppTool {
// blocking task (a spawn_blocking thread can't be cancelled — it
// unblocks if/when the modal is dismissed) and return an error so
// the daemon stays responsive instead of wedging on the request.
- let result = tokio::time::timeout(
- std::time::Duration::from_secs(15),
- launch,
- ).await;
+ let result = tokio::time::timeout(std::time::Duration::from_secs(15), launch).await;
match result {
- Ok(Ok(Ok(p))) => p,
+ Ok(Ok(Ok(p))) => p,
Ok(Ok(Err(e))) => return ToolResult::error(format!("Failed to launch: {e}")),
- Ok(Err(e)) => return ToolResult::error(format!("Task error: {e}")),
- Err(_elapsed) => return ToolResult::error(format!(
- "Launch of {:?} timed out after 15s — the target likely has no \
+ Ok(Err(e)) => return ToolResult::error(format!("Task error: {e}")),
+ Err(_elapsed) => {
+ return ToolResult::error(format!(
+ "Launch of {:?} timed out after 15s — the target likely has no \
registered handler and a blocking shell dialog appeared on the \
session desktop; aborted to keep the daemon responsive.",
- target_file_opt.as_deref()
- .or_else(|| urls.first().map(|s| s.as_str()))
- .unwrap_or("")
- )),
+ target_file_opt
+ .as_deref()
+ .or_else(|| urls.first().map(|s| s.as_str()))
+ .unwrap_or("")
+ ))
+ }
}
};
@@ -1770,8 +1933,8 @@ impl Tool for LaunchAppTool {
let urls_clone = urls.clone();
let n_show_for_urls = n_show;
let _ = tokio::task::spawn_blocking(move || {
- use windows::Win32::UI::Shell::{ShellExecuteExW, SHELLEXECUTEINFOW};
use windows::core::PCWSTR;
+ use windows::Win32::UI::Shell::{ShellExecuteExW, SHELLEXECUTEINFOW};
fn to_wide(s: &str) -> Vec<u16> {
s.encode_utf16().chain(std::iter::once(0)).collect()
}
@@ -1782,10 +1945,12 @@ impl Tool for LaunchAppTool {
cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32,
lpVerb: PCWSTR(op_w.as_ptr()),
lpFile: PCWSTR(file.as_ptr()),
- nShow: n_show_for_urls,
+ nShow: n_show_for_urls,
..Default::default()
};
- unsafe { let _ = ShellExecuteExW(&mut url_info); }
+ unsafe {
+ let _ = ShellExecuteExW(&mut url_info);
+ }
}
})
.await;
@@ -1820,11 +1985,10 @@ impl Tool for LaunchAppTool {
// frame can lag the activation by a few hundred ms.
if aumid_for_uwp.is_some() {
for _ in 0..10 {
- let host = tokio::task::spawn_blocking(move || {
- crate::win32::resolve_uwp_host_window(pid)
- })
- .await
- .unwrap_or(None);
+ let host =
+ tokio::task::spawn_blocking(move || crate::win32::resolve_uwp_host_window(pid))
+ .await
+ .unwrap_or(None);
if let Some(w) = host {
windows_json = vec![json!({
"window_id": w.hwnd, "title": w.title,
@@ -1848,7 +2012,8 @@ impl Tool for LaunchAppTool {
break;
}
let wins = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
- .await.unwrap_or_default();
+ .await
+ .unwrap_or_default();
if !wins.is_empty() {
windows_json = wins.iter().map(|w| json!({
"window_id": w.hwnd, "title": w.title,
@@ -1926,7 +2091,9 @@ impl Tool for LaunchAppTool {
// For slow launchers, keep re-scanning descendants — the
// wrapper may not have spawned its child yet. Cap total
// wait at ~12s (60 × 200ms) for the slow path.
- if !is_slow_launcher || total_attempts > 60 { break; }
+ if !is_slow_launcher || total_attempts > 60 {
+ break;
+ }
// Give the wrapper a moment to spawn before re-scanning.
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
total_attempts += 3; // count the 500ms wait as 3 attempts
@@ -1972,7 +2139,8 @@ impl Tool for LaunchAppTool {
if start_minimized {
// First, minimize anything already resolved (covers the common
// single-process path where windows_json was populated).
- let immediate_hwnds: Vec<u64> = windows_json.iter()
+ let immediate_hwnds: Vec<u64> = windows_json
+ .iter()
.filter_map(|w| w["window_id"].as_u64())
.collect();
// Capture pid + basename for the post-launch polling task to
@@ -1992,9 +2160,12 @@ impl Tool for LaunchAppTool {
use windows::Win32::Foundation::HWND;
use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_MINIMIZE};
for h in immediate_hwnds {
- unsafe { let _ = ShowWindow(HWND(h as *mut _), SW_MINIMIZE); }
+ unsafe {
+ let _ = ShowWindow(HWND(h as *mut _), SW_MINIMIZE);
+ }
}
- }).await;
+ })
+ .await;
// Detached polling for launcher-stub late-window cases.
// Strategy: every 200 ms for 5 s, find pids that
// (a) weren't in the pre-launch snapshot, AND
@@ -2056,11 +2227,9 @@ impl Tool for LaunchAppTool {
hit_count_total += 1;
let hwnd_iso = w.hwnd as usize;
let _ = tokio::task::spawn_blocking(move || unsafe {
- let _ = ShowWindow(
- HWND(hwnd_iso as *mut _),
- SW_MINIMIZE,
- );
- }).await;
+ let _ = ShowWindow(HWND(hwnd_iso as *mut _), SW_MINIMIZE);
+ })
+ .await;
}
}
}
@@ -2085,11 +2254,17 @@ impl Tool for LaunchAppTool {
summary.push_str("\n\nWindows:");
for w in &windows_json {
let title = w["title"].as_str().unwrap_or("");
- let title_disp = if title.is_empty() { "(no title)".to_owned() } else { format!("\"{title}\"") };
+ let title_disp = if title.is_empty() {
+ "(no title)".to_owned()
+ } else {
+ format!("\"{title}\"")
+ };
let wid = w["window_id"].as_u64().unwrap_or(0);
summary.push_str(&format!("\n- {title_disp} [window_id: {wid}]"));
}
- summary.push_str(&format!("\n→ Call get_window_state(pid: {pid}, window_id) to inspect."));
+ summary.push_str(&format!(
+ "\n→ Call get_window_state(pid: {pid}, window_id) to inspect."
+ ));
}
// `bundle_id` is the AUMID when we went through the packaged-app
// path (so the caller can round-trip the same value to relaunch),
@@ -2192,9 +2367,9 @@ impl Tool for ClickTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- use cua_driver_core::tool_args::ArgsExt;
- use crate::input::delivery::{DeliveryMode, EventKind, background_unavailable_error};
+ use crate::input::delivery::{DeliveryMode, EventKind};
use crate::uia::cache::SnapshotKind;
+ use cua_driver_core::tool_args::ArgsExt;
let cursor_key = resolve_cursor_key(&args);
// ── Window-less screen-absolute branch (capture_scope="desktop") ──────
@@ -2213,7 +2388,7 @@ impl Tool for ClickTool {
"click: x,y given with no pid/window_id, but capture_scope is \
\"window\". Screen-absolute clicks require desktop scope. Call \
set_config with capture_scope=desktop (and use get_desktop_state \
- to pick coordinates), or pass a pid/window_id."
+ to pick coordinates), or pass a pid/window_id.",
)
.with_structured(json!({
"code": "desktop_scope_disabled",
@@ -2228,7 +2403,11 @@ impl Tool for ClickTool {
"click: unknown button \"{button_raw}\" — expected one of left, right, middle."
));
}
- let button = if button_raw.is_empty() { "left".to_string() } else { button_raw };
+ let button = if button_raw.is_empty() {
+ "left".to_string()
+ } else {
+ button_raw
+ };
let count = args.u64_or("count", 1) as usize;
let modifiers: Vec<String> = args.str_array("modifier");
let sx = args.f64_or("x", 0.0) as i32;
@@ -2236,9 +2415,13 @@ impl Tool for ClickTool {
// Animate the agent cursor to the screen point, then click.
overlay_glide_to(&cursor_key, sx as f64, sy as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: sx as f64, y: sy as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx as f64,
+ y: sy as f64,
+ },
+ );
// Resolve the HWND that owns this screen pixel and click it via
// send_click_synthesized — it does the foreground-swap + UIPI checks
@@ -2255,12 +2438,19 @@ impl Tool for ClickTool {
}
let hwnd_u = target.0 as u64;
let mod_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect();
- crate::input::send_click_synthesized_mods(hwnd_u, sx, sy, count, &button, &mod_refs)?;
+ crate::input::send_click_synthesized_mods(
+ hwnd_u, sx, sy, count, &button, &mod_refs,
+ )?;
Ok(hwnd_u)
- }).await;
+ })
+ .await;
return match send_result {
Ok(Ok(hwnd_u)) => {
- let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" };
+ let click_word = match count {
+ 2 => "double-click",
+ 3 => "triple-click",
+ _ => "click",
+ };
ToolResult::text(format!(
"✅ Sent {click_word} via SendInput at screen ({sx},{sy}) on HWND 0x{hwnd_u:x} (desktop scope)."
))
@@ -2272,7 +2462,10 @@ impl Tool for ClickTool {
};
}
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Surface 6: element_token / element_index precedence resolution.
// Windows uses u64 HWND but the token registry stores u32; truncate
// through the same path get_window_state used when registering.
@@ -2287,13 +2480,15 @@ impl Tool for ClickTool {
Err(e) => return e,
};
let elem_idx = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
+ .map(|v| v as u64)
+ .or_else(|| args.opt_u64("window_id")),
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
let x = args.opt_f64("x");
@@ -2307,7 +2502,11 @@ impl Tool for ClickTool {
"click: unknown button \"{button_raw}\" — expected one of left, right, middle."
));
}
- let button = if button_raw.is_empty() { "left".to_string() } else { button_raw };
+ let button = if button_raw.is_empty() {
+ "left".to_string()
+ } else {
+ button_raw
+ };
let count = args.u64_or("count", 1) as usize;
// macOS click `modifier` surface: held keys (cmd/shift/option/ctrl). Only
// the SendInput rung (delivery_mode:"foreground" pixel tap, and the
@@ -2335,16 +2534,26 @@ impl Tool for ClickTool {
// half) instead of the center (press half). Defaults to first
// action in the element's `actions=[...]` list, which preserves
// existing semantics for UIA elements.
- let action_req = args.get("action").and_then(|v| v.as_str()).map(|s| s.to_string());
+ let action_req = args
+ .get("action")
+ .and_then(|v| v.as_str())
+ .map(|s| s.to_string());
// Resolve HWND: explicit, or auto from pid.
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -2361,8 +2570,10 @@ impl Tool for ClickTool {
// `accDoDefaultAction` here because LO's MSAA impl applies
// the change asynchronously and returns S_OK either way,
// so the visible behavior is the same as a center click.
- if let Some((SnapshotKind::Msaa, role)) =
- self.state.element_cache.get_element_kind_and_role(pid, hwnd, idx)
+ if let Some((SnapshotKind::Msaa, role)) = self
+ .state
+ .element_cache
+ .get_element_kind_and_role(pid, hwnd, idx)
{
const ROLE_BUTTONDROPDOWN: i32 = 0x38;
const ROLE_BUTTONMENU: i32 = 0x39;
@@ -2371,7 +2582,12 @@ impl Tool for ClickTool {
let want_expand = action_req.as_deref() == Some("expand");
let is_dropdown_role = matches!(
role,
- Some(ROLE_BUTTONDROPDOWN | ROLE_BUTTONMENU | ROLE_BUTTONDROPDOWNGRID | ROLE_SPLITBUTTON)
+ Some(
+ ROLE_BUTTONDROPDOWN
+ | ROLE_BUTTONMENU
+ | ROLE_BUTTONDROPDOWNGRID
+ | ROLE_SPLITBUTTON
+ )
);
let (tx, ty) = if want_expand && is_dropdown_role {
match self.state.element_cache.get_element_rect(pid, hwnd, idx) {
@@ -2399,31 +2615,40 @@ impl Tool for ClickTool {
} else {
match self.state.element_cache.get_element_center(pid, hwnd, idx) {
Some(v) => v,
- None => return ToolResult::error(format!(
- "MSAA element [{idx}] not in cache for hwnd={hwnd}. \
+ None => {
+ return ToolResult::error(format!(
+ "MSAA element [{idx}] not in cache for hwnd={hwnd}. \
Call get_window_state first."
- )),
+ ))
+ }
}
};
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, tx as f64, ty as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: tx as f64, y: ty as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: tx as f64,
+ y: ty as f64,
+ },
+ );
let btn_fg = button.clone();
let prev_fg_addr = unsafe {
windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
};
let send_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized(hwnd, tx, ty, count, &btn_fg)
- }).await;
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
let half = if want_expand { "dropdown" } else { "press" };
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Performed SendInput click on MSAA [{idx}] {half} half at ({tx},{ty})."
))
- .with_structured(json!({ "path": "msaa", "verified": false, "effect": "unverifiable" })),
+ .with_structured(
+ json!({ "path": "msaa", "verified": false, "effect": "unverifiable" }),
+ ),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -2432,7 +2657,11 @@ impl Tool for ClickTool {
// UIA path: get cached center (no COM call needed — captured at walk time).
let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {
Some(v) => v,
- None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")),
+ None => {
+ return ToolResult::error(format!(
+ "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first."
+ ))
+ }
};
// NB: the off-screen guard is applied per-delivery-path below (the
// foreground SendInput tap and the background coordinate injection),
@@ -2444,9 +2673,13 @@ impl Tool for ClickTool {
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
// Step 3: click pulse + actual click.
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: cx as f64, y: cy as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: cx as f64,
+ y: cy as f64,
+ },
+ );
let btn = button.clone();
// delivery_mode:"foreground" — skip UIA Invoke and use SendInput at the
@@ -2459,7 +2692,13 @@ impl Tool for ClickTool {
// element is off-screen, scroll it into view and re-resolve so the
// tap doesn't land on the taskbar, else keep the clean failure.
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache, pid, hwnd, idx, cx, cy, "a foreground click",
+ &self.state.element_cache,
+ pid,
+ hwnd,
+ idx,
+ cx,
+ cy,
+ "a foreground click",
) {
Ok(p) => p,
Err(msg) => return ToolResult::error(msg),
@@ -2470,7 +2709,8 @@ impl Tool for ClickTool {
};
let send_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized(hwnd, cx, cy, count, &btn_fg)
- }).await;
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
@@ -2485,8 +2725,12 @@ impl Tool for ClickTool {
// left falls through to the UIA Invoke path below (already drives
// WinUI3). Double-left lands via a double UIA Invoke; right/middle
// can't both land and hold the contract → structured error.
- if delivery == DeliveryMode::Background && (count > 1 || btn == "right" || btn == "middle") {
- if let Some(r) = winui3_background_gesture(&self.state, pid, hwnd, Some(idx), count, &btn).await {
+ if delivery == DeliveryMode::Background
+ && (count > 1 || btn == "right" || btn == "middle")
+ {
+ if let Some(r) =
+ winui3_background_gesture(&self.state, pid, hwnd, Some(idx), count, &btn).await
+ {
return r;
}
}
@@ -2616,7 +2860,7 @@ impl Tool for ClickTool {
Ok(()) => return Ok(format!(
"✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)."
)),
- Err(_) => anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__"),
+ Err(e) => anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__{e}"),
}
}
crate::input::post_click_screen(hwnd, cx, cy, count, &btn)?;
@@ -2633,22 +2877,37 @@ impl Tool for ClickTool {
// confirms via screenshot. (The would_be_silently_dropped surfaces
// are diverted to `background_unavailable` below before reaching
// here, so a success here always means a real dispatch.)
- Ok(Ok(msg)) => ToolResult::text(msg)
- .with_structured(json!({ "path": "ax", "verified": false, "effect": "unverifiable" })),
+ Ok(Ok(msg)) => ToolResult::text(msg).with_structured(
+ json!({ "path": "ax", "verified": false, "effect": "unverifiable" }),
+ ),
Ok(Err(e)) if e.to_string().contains("__CUA_BG_UNAVAILABLE_CLICK__") => {
- background_unavailable_error(hwnd, EventKind::MouseClick)
+ let cause = e.to_string().replace("__CUA_BG_UNAVAILABLE_CLICK__", "");
+ crate::input::delivery::background_unavailable_error_with_cause(
+ hwnd,
+ EventKind::MouseClick,
+ cause,
+ )
}
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else if let (Some(mut px), Some(mut py)) = (x, y) {
- let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);
+ let from_zoom = args
+ .get("from_zoom")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
if from_zoom {
match self.state.zoom_registry.get(pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(px, py); px = wx; py = wy; }
- None => return ToolResult::error(
- format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")
- ),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(px, py);
+ px = wx;
+ py = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
px *= ratio;
@@ -2660,7 +2919,10 @@ impl Tool for ClickTool {
let (sx, sy) = (sx_i as f64, sy_i as f64);
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, sx, sy).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
+ );
let btn = button.clone();
// Vision-mode (x, y) dispatch is **layered**, mirroring the
// trope-cua reference impl
@@ -2703,12 +2965,19 @@ impl Tool for ClickTool {
let mods_owned = modifiers.clone();
let send_result = tokio::task::spawn_blocking(move || {
let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
- crate::input::send_click_synthesized_mods(hwnd, sx as i32, sy as i32, count, &btn, &mod_refs)
- }).await;
+ crate::input::send_click_synthesized_mods(
+ hwnd, sx as i32, sy as i32, count, &btn, &mod_refs,
+ )
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
return match send_result {
Ok(Ok(())) => {
- let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" };
+ let click_word = match count {
+ 2 => "double-click",
+ 3 => "triple-click",
+ _ => "click",
+ };
ToolResult::text(format!(
"✅ Sent {click_word} via SendInput to pid {pid} at ({sx},{sy}) (delivery_mode:foreground)."
))
@@ -2723,8 +2992,12 @@ impl Tool for ClickTool {
// path, no element_index → no cached element to UIA-Invoke): these
// can't both land and hold the contract on WinUI3 (posted input is
// ignored, the pen injector steals foreground) → structured error.
- if delivery == DeliveryMode::Background && (count > 1 || btn == "right" || btn == "middle") {
- if let Some(r) = winui3_background_gesture(&self.state, pid, hwnd, None, count, &btn).await {
+ if delivery == DeliveryMode::Background
+ && (count > 1 || btn == "right" || btn == "middle")
+ {
+ if let Some(r) =
+ winui3_background_gesture(&self.state, pid, hwnd, None, count, &btn).await
+ {
return r;
}
}
@@ -2732,7 +3005,9 @@ impl Tool for ClickTool {
if use_uia {
let invoked = tokio::task::spawn_blocking(move || {
crate::uia::windows_enum::try_invoke_in_window_at_point(
- hwnd as isize, sx as i32, sy as i32,
+ hwnd as isize,
+ sx as i32,
+ sy as i32,
)
})
.await
@@ -2741,7 +3016,9 @@ impl Tool for ClickTool {
return ToolResult::text(format!(
"✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."
))
- .with_structured(json!({ "path": "ax", "verified": false, "effect": "unverifiable" }));
+ .with_structured(
+ json!({ "path": "ax", "verified": false, "effect": "unverifiable" }),
+ );
}
}
@@ -2769,13 +3046,21 @@ impl Tool for ClickTool {
.await;
return match inj {
Ok(Ok(())) => {
- let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" };
+ let click_word = match count {
+ 2 => "double-click",
+ 3 => "triple-click",
+ _ => "click",
+ };
ToolResult::text(format!(
"✅ Injected {click_word} to pid {pid} at ({sx},{sy}) (background, no foreground swap)."
))
.with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" }))
}
- Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick),
+ Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause(
+ hwnd,
+ EventKind::MouseClick,
+ e.to_string(),
+ ),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -2794,7 +3079,8 @@ impl Tool for ClickTool {
// post_click_screen so we don't double-ClientToScreen.
let result = tokio::task::spawn_blocking(move || {
crate::input::post_click_screen(hwnd, sx_i, sy_i, count, &btn)
- }).await;
+ })
+ .await;
match result {
Ok(Ok(())) => {
// Match Swift text format: "✅ Posted click/double-click/triple-click to pid X."
@@ -2804,7 +3090,9 @@ impl Tool for ClickTool {
_ => "click",
};
ToolResult::text(format!("✅ Posted {click_word} to pid {pid}."))
- .with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" }))
+ .with_structured(
+ json!({ "path": "pixel", "verified": false, "effect": "unverifiable" }),
+ )
}
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -2840,11 +3128,23 @@ async fn focus_by_pixel(
"pid": pid, "x": x, "y": y,
"delivery_mode": if foreground { "foreground" } else { "background" },
});
- if let Some(wid) = window_id { click_args["window_id"] = json!(wid); }
- if let Some(s) = session { click_args["session"] = json!(s); }
- if let Some(s) = session_id { click_args["_session_id"] = json!(s); }
- if from_zoom { click_args["from_zoom"] = json!(true); }
- let focus = ClickTool { state: state.clone() }.invoke(click_args).await;
+ if let Some(wid) = window_id {
+ click_args["window_id"] = json!(wid);
+ }
+ if let Some(s) = session {
+ click_args["session"] = json!(s);
+ }
+ if let Some(s) = session_id {
+ click_args["_session_id"] = json!(s);
+ }
+ if from_zoom {
+ click_args["from_zoom"] = json!(true);
+ }
+ let focus = ClickTool {
+ state: state.clone(),
+ }
+ .invoke(click_args)
+ .await;
if focus.is_error == Some(true) {
return Err(ToolResult::error(format!(
"focus pixel-click at ({x:.0},{y:.0}) failed."
@@ -2909,11 +3209,17 @@ impl Tool for TypeTextTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
+ use crate::input::delivery::{DeliveryMode, EventKind};
use cua_driver_core::tool_args::ArgsExt;
- use crate::input::delivery::{DeliveryMode, EventKind, background_unavailable_error};
- let raw_pid = match args.require_i64("pid") { Ok(v) => v, Err(e) => return e };
+ let raw_pid = match args.require_i64("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let pid = raw_pid as u32;
- let text_raw = match args.require_str("text") { Ok(v) => v, Err(e) => return e };
+ let text_raw = match args.require_str("text") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let cursor_key = resolve_cursor_key(&args);
// Strip trailing agent-protocol closing tags before delivery —
// catches the case where an LLM hallucinated its own tool-
@@ -2934,13 +3240,15 @@ impl Tool for TypeTextTool {
Err(e) => return e,
};
let elem_idx: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index as u64),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index as u64)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
+ .map(|v| v as u64)
+ .or_else(|| args.opt_u64("window_id")),
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
let delivery = DeliveryMode::from_args(&args);
@@ -2957,14 +3265,26 @@ impl Tool for TypeTextTool {
if let (Some(cx), Some(cy)) = (px, py) {
if elem_idx.is_some() {
return ToolResult::error(
- "Pass either element_index (ax) or x,y (px) to type_text, not both."
+ "Pass either element_index (ax) or x,y (px) to type_text, not both.",
);
}
- let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);
+ let from_zoom = args
+ .get("from_zoom")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
if let Err(e) = focus_by_pixel(
- &self.state, pid, hwnd_opt, cx, cy, delivery.is_foreground(),
- args.opt_str("session"), args.opt_str("_session_id"), from_zoom,
- ).await {
+ &self.state,
+ pid,
+ hwnd_opt,
+ cx,
+ cy,
+ delivery.is_foreground(),
+ args.opt_str("session"),
+ args.opt_str("_session_id"),
+ from_zoom,
+ )
+ .await
+ {
return e;
}
// elem_idx stays None → the type path below writes to the now-
@@ -2976,7 +3296,8 @@ impl Tool for TypeTextTool {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
is scoped per (pid, window_id). Pass the same window_id you used in \
- `get_window_state`.");
+ `get_window_state`.",
+ );
}
// Same no-raise guard as click: a XAML/WPF ValuePattern.SetValue handler
// calls UIElement.Focus()→SetForegroundWindow; WS_EX_NOACTIVATE on the
@@ -2993,10 +3314,17 @@ impl Tool for TypeTextTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -3009,7 +3337,10 @@ impl Tool for TypeTextTool {
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::TextInput)
{
- return background_unavailable_error(hwnd, EventKind::TextInput);
+ return crate::input::delivery::background_unavailable_error(
+ hwnd,
+ EventKind::TextInput,
+ );
}
// delivery_mode:"foreground" — explicit SendInput Unicode with a brief
// foreground swap (send_text_synthesized), symmetric with press_key's
@@ -3022,7 +3353,8 @@ impl Tool for TypeTextTool {
let text_fg = text.clone();
let r = tokio::task::spawn_blocking(move || {
crate::input::send_text_synthesized(hwnd, &text_fg)
- }).await;
+ })
+ .await;
return match r {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Typed {text_len} char(s) on pid {raw_pid} via SendInput (delivery_mode:foreground)."
@@ -3050,11 +3382,19 @@ impl Tool for TypeTextTool {
// Only when an element_index is supplied (we have its cached center);
// the focused-element path has no resolvable position to point at.
if let Some(idx) = elem_idx {
- if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx as usize) {
+ if let Some((cx, cy)) =
+ self.state
+ .element_cache
+ .get_element_center(pid, hwnd, idx as usize)
+ {
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: cx as f64, y: cy as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: cx as f64,
+ y: cy as f64,
+ },
+ );
}
}
@@ -3066,7 +3406,10 @@ impl Tool for TypeTextTool {
// contract forbids in background. Surface background_unavailable; the
// agent escalates to delivery_mode:"foreground".
if crate::terminal::is_terminal_hwnd(hwnd) {
- return background_unavailable_error(hwnd, EventKind::TextInput);
+ return crate::input::delivery::background_unavailable_error(
+ hwnd,
+ EventKind::TextInput,
+ );
}
// CUA-543 routing: PostMessage WM_CHAR doesn't reach modern
@@ -3092,13 +3435,17 @@ impl Tool for TypeTextTool {
// get_window_state snapshot-replace on the same (pid, hwnd)
// can't Release it to zero while this SetValue is in flight.
// The guard is held for the whole closure.
- let Some(element_guard) = state.element_cache.get_element_retained(pid, hwnd, idx) else { return false; };
+ let Some(element_guard) = state.element_cache.get_element_retained(pid, hwnd, idx)
+ else {
+ return false;
+ };
let ptr = element_guard.as_ptr();
+ use windows::core::{Interface, BSTR};
use windows::Win32::UI::Accessibility::{
IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
};
- use windows::core::{Interface, BSTR};
- let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
+ let elem: IUIAutomationElement =
+ unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
let ok = (|| -> anyhow::Result<()> {
let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) }?;
let vp: IUIAutomationValuePattern = pattern.cast()?;
@@ -3108,15 +3455,17 @@ impl Tool for TypeTextTool {
// does NOT stop; the EnableWindow shield does (disabled
// top-level can't be foregrounded) while the a11y-channel
// SetValue still lands. Same fix as the UIA Invoke path.
- crate::uia::fg_bypass::run_with_uwp_bypass(
- hwnd as isize,
- || unsafe { vp.SetValue(&BSTR::from(text_for_uia.as_str())) },
- )?;
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
+ vp.SetValue(&BSTR::from(text_for_uia.as_str()))
+ })?;
Ok(())
- })().is_ok();
+ })()
+ .is_ok();
std::mem::forget(elem);
ok
- }).await.unwrap_or(false);
+ })
+ .await
+ .unwrap_or(false);
if set_ok {
// Read the value back by element handle — focus-independent
// (works regardless of which window is foreground), proving the
@@ -3128,9 +3477,11 @@ impl Tool for TypeTextTool {
match read_cached_element_value(&state_rb, pid, hwnd, idx) {
Some(v) if v.contains(text_rb.as_str()) => "confirmed",
Some(_) => "unchanged",
- None => "unreadable",
+ None => "unreadable",
}
- }).await.unwrap_or("unreadable");
+ })
+ .await
+ .unwrap_or("unreadable");
// SURFACE-AWARE VERIFICATION (mirrors macOS type_text). On
// Electron/Chromium web inputs the UIA layer accepts a
// `ValuePattern.SetValue` and echoes it straight back through
@@ -3143,21 +3494,26 @@ impl Tool for TypeTextTool {
// types (WPF / WinForms / UWP / XAML) pay nothing. The signal is
// the window class — Chromium and every Electron host expose
// `Chrome_WidgetWin_*` (see `is_chromium_target_window`).
- let ax_echo_surface = verify == "confirmed"
- && crate::input::is_chromium_target_window(hwnd);
+ let ax_echo_surface =
+ verify == "confirmed" && crate::input::is_chromium_target_window(hwnd);
let verified = verify == "confirmed" && !ax_echo_surface;
let (mark, note) = if verified {
("✅ Wrote", String::new())
} else if ax_echo_surface {
- ("📨 Sent (unverified)",
- " — Electron/web surface: the UIA layer accepts and echoes the \
+ (
+ "📨 Sent (unverified)",
+ " — Electron/web surface: the UIA layer accepts and echoes the \
write but the renderer may not have observed it, so the driver \
cannot confirm via UIA. Verify via the screenshot; if it didn't \
land, re-type with the px form (pass x,y to pixel-focus the \
- field).".to_string())
+ field)."
+ .to_string(),
+ )
} else {
- ("📨 Sent (unverified)",
- " — driver could not confirm; verify via screenshot.".to_string())
+ (
+ "📨 Sent (unverified)",
+ " — driver could not confirm; verify via screenshot.".to_string(),
+ )
};
return ToolResult::text(format!(
"{mark} {text_len} char(s) on pid {raw_pid} via UIA ValuePattern \
@@ -3209,7 +3565,10 @@ impl Tool for TypeTextTool {
// never raises); otherwise surface background_unavailable so the agent
// escalates to delivery_mode:"foreground".
if elem_idx.is_none() && crate::input::delivery::is_wpf_target_window(hwnd) {
- return background_unavailable_error(hwnd, EventKind::TextInput);
+ return crate::input::delivery::background_unavailable_error(
+ hwnd,
+ EventKind::TextInput,
+ );
}
// 3. Legacy Win32 / GDI / Chromium-IME — PostMessage WM_CHAR, no focus steal.
@@ -3233,14 +3592,15 @@ impl Tool for TypeTextTool {
// focused" with no element_index.
let read = |idx: Option<usize>| match idx {
Some(i) => read_cached_element_value(&state_rb, verify_pid, hwnd, i),
- None => read_focused_value_uia(verify_pid),
+ None => read_focused_value_uia(verify_pid),
};
let before = read(verify_idx);
let post_res = crate::input::post_type_text(hwnd, &text_for_post);
std::thread::sleep(std::time::Duration::from_millis(40));
let after = read(verify_idx);
(post_res, before, after)
- }).await;
+ })
+ .await;
match result {
Ok((Ok(()), before, after)) => {
// Three-way verdict. Crucially, "couldn't read the field" is NOT
@@ -3280,8 +3640,8 @@ impl Tool for TypeTextTool {
},
})),
_ => ToolResult::text(format!(
- "✅ Typed {text_len} char(s) on pid {raw_pid} via PostMessage \
- ({_delay_ms}ms delay; delivered, not verified — could not read \
+ "📨 Sent {text_len} char(s) to pid {raw_pid} via PostMessage \
+ ({_delay_ms}ms delay; not verified — could not read \
the focused field back, e.g. the target isn't foreground or \
exposes no ValuePattern). If it didn't land, retry with \
delivery_mode:\"foreground\"."
@@ -3300,7 +3660,7 @@ impl Tool for TypeTextTool {
}
}
Ok((Err(e), _, _)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
}
@@ -3314,27 +3674,24 @@ impl Tool for TypeTextTool {
/// `None` if the index isn't cached or the element exposes no readable text
/// pattern. Mirrors the cache-retain + `mem::forget` discipline of the
/// ValuePattern.SetValue path so the cached COM ref isn't released.
-fn read_cached_element_value(
- state: &ToolState,
- pid: u32,
- hwnd: u64,
- idx: usize,
-) -> Option<String> {
+fn read_cached_element_value(state: &ToolState, pid: u32, hwnd: u64, idx: usize) -> Option<String> {
+ use windows::core::Interface;
use windows::Win32::UI::Accessibility::{
IUIAutomationElement, IUIAutomationTextPattern, IUIAutomationValuePattern,
UIA_TextPatternId, UIA_ValuePatternId,
};
- use windows::core::Interface;
let guard = state.element_cache.get_element_retained(pid, hwnd, idx)?;
let ptr = guard.as_ptr();
let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
let val = unsafe {
- elem.GetCurrentPattern(UIA_ValuePatternId).ok()
+ elem.GetCurrentPattern(UIA_ValuePatternId)
+ .ok()
.and_then(|p| p.cast::<IUIAutomationValuePattern>().ok())
.and_then(|vp| vp.CurrentValue().ok())
.map(|b| b.to_string())
.or_else(|| {
- elem.GetCurrentPattern(UIA_TextPatternId).ok()
+ elem.GetCurrentPattern(UIA_TextPatternId)
+ .ok()
.and_then(|p| p.cast::<IUIAutomationTextPattern>().ok())
.and_then(|tp| tp.DocumentRange().ok())
.and_then(|r| r.GetText(-1).ok())
@@ -3357,20 +3714,19 @@ fn read_cached_element_value(
/// "could not confirm" rather than failure. Mirrors the focused-element read in
/// the `debug_window_info` tool.
fn read_focused_value_uia(expect_pid: u32) -> Option<String> {
+ use windows::core::Interface;
use windows::Win32::System::Com::{
- CoCreateInstance, CoInitializeEx, CoUninitialize,
- CLSCTX_INPROC_SERVER, COINIT_APARTMENTTHREADED,
+ CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER,
+ COINIT_APARTMENTTHREADED,
};
use windows::Win32::UI::Accessibility::{
CUIAutomation, IUIAutomation, IUIAutomationValuePattern, UIA_ValuePatternId,
};
- use windows::core::Interface;
let coinit_ok = unsafe { CoInitializeEx(None, COINIT_APARTMENTTHREADED).is_ok() };
let value = (|| -> Option<String> {
- let uia: IUIAutomation = unsafe {
- CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER)
- }.ok()?;
+ let uia: IUIAutomation =
+ unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) }.ok()?;
let focused = unsafe { uia.GetFocusedElement() }.ok()?;
// Scope to the target: system focus may be on a different app when the
// target is backgrounded; reading that element would give a bogus verdict.
@@ -3382,7 +3738,9 @@ fn read_focused_value_uia(expect_pid: u32) -> Option<String> {
Some(unsafe { vp.CurrentValue() }.ok()?.to_string())
})();
if coinit_ok {
- unsafe { CoUninitialize(); }
+ unsafe {
+ CoUninitialize();
+ }
}
value
}
@@ -3410,15 +3768,15 @@ impl Tool for PressKeyTool {
end, pageup, pagedown, f1-f12, plus any letter or digit. Optional `modifiers` \
array takes ctrl/shift/alt/win. For true combinations (ctrl+c), `hotkey` is a \
cleaner surface.\n\n\
- Note: `element_index` is accepted for cross-platform parity with macOS but \
- currently no-op on Windows (no UIA SetFocus path wired up yet).".into(),
+ `element_index` focuses the cached UIA element before sending the key; the \
+ top-level window remains backgrounded when delivery_mode is background.".into(),
input_schema: json!({
"type":"object","required":["pid","key"],"properties":{
"session": cua_driver_core::tool_schema::session_schema(),
"pid":{"type":"integer","description":"Target process ID."},
"key":{"type":"string","description":"Key name (return, tab, escape, up, down, left, right, space, delete, home, end, pageup, pagedown, f1-f12, letter, digit)."},
"modifiers":{"type":"array","items":{"type":"string"},"description":"Optional modifier names held while the key is pressed (ctrl/shift/alt/win)."},
- "element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},
+ "element_index":{"type":"integer","description":"Optional element_index from the last get_window_state; focuses that UIA element before the key is delivered."},
"element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
"x":{"type":"number","description":"Window-local screenshot-pixel X — the element px action form: pixel-click there to focus, then send the key. Use when the key must go to a Chromium/Electron surface the UIA path can't focus. Pass with y, no element_index."},
"y":{"type":"number","description":"Window-local screenshot-pixel Y (see x)."},
@@ -3431,11 +3789,17 @@ impl Tool for PressKeyTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
+ use crate::input::delivery::{DeliveryMode, EventKind};
use cua_driver_core::tool_args::ArgsExt;
- use crate::input::delivery::{DeliveryMode, EventKind, background_unavailable_error};
- let raw_pid = match args.require_i64("pid") { Ok(v) => v, Err(e) => return e };
+ let raw_pid = match args.require_i64("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let pid = raw_pid as u32;
- let key = match args.require_str("key") { Ok(v) => v, Err(e) => return e };
+ let key = match args.require_str("key") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let mods: Vec<String> = args.str_array("modifiers");
// Surface 6: element_token / element_index precedence resolution.
let resolved = match cua_driver_core::element_token::resolve_element_args(
@@ -3449,13 +3813,15 @@ impl Tool for PressKeyTool {
Err(e) => return e,
};
let elem_idx: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index as u64),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index as u64)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
+ .map(|v| v as u64)
+ .or_else(|| args.opt_u64("window_id")),
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
let delivery = DeliveryMode::from_args(&args);
@@ -3465,7 +3831,8 @@ impl Tool for PressKeyTool {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
is scoped per (pid, window_id). Pass the same window_id you used in \
- `get_window_state`.");
+ `get_window_state`.",
+ );
}
// px form: pixel-click to focus, then the key goes to the focused element.
@@ -3479,37 +3846,86 @@ impl Tool for PressKeyTool {
if let (Some(cx), Some(cy)) = (px, py) {
if elem_idx.is_some() {
return ToolResult::error(
- "Pass either element_index (ax) or x,y (px) to press_key, not both."
+ "Pass either element_index (ax) or x,y (px) to press_key, not both.",
);
}
- let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);
+ let from_zoom = args
+ .get("from_zoom")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
if let Err(e) = focus_by_pixel(
- &self.state, pid, hwnd_opt, cx, cy, delivery.is_foreground(),
- args.opt_str("session"), args.opt_str("_session_id"), from_zoom,
- ).await {
+ &self.state,
+ pid,
+ hwnd_opt,
+ cx,
+ cy,
+ delivery.is_foreground(),
+ args.opt_str("session"),
+ args.opt_str("_session_id"),
+ from_zoom,
+ )
+ .await
+ {
return e;
}
true
- } else { false }
+ } else {
+ false
+ }
};
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
+
+ // W1: an element-addressed key needs the control's actual focus
+ // target, not merely its owning top-level HWND. Keep background
+ // delivery non-activating while UIA establishes child focus.
+ let _noact = if elem_idx.is_some() && delivery == DeliveryMode::Background {
+ Some(crate::input::NoActivateGuard::arm(
+ windows::Win32::Foundation::HWND(hwnd as *mut _),
+ ))
+ } else {
+ None
+ };
+ if let Some(idx) = elem_idx {
+ let state = self.state.clone();
+ let focused = tokio::task::spawn_blocking(move || {
+ state.element_cache.focus_element(pid, hwnd, idx as usize)
+ })
+ .await;
+ match focused {
+ Ok(Ok(())) => {}
+ Ok(Err(e)) => return ToolResult::error(e.to_string()),
+ Err(e) => return ToolResult::error(format!("UIA focus task failed: {e}")),
+ }
+ }
let key_display = key.clone();
// Background mode: plain keystrokes (no modifiers) go through Chromium
// and GTK fine — would_be_silently_dropped returns false for the
// Keystroke variant by design. KeyCombo (modifiers) on Chromium IS
// dropped, so check that when modifiers are present.
- let event_kind = if mods.is_empty() { EventKind::Keystroke } else { EventKind::KeyCombo };
- if !px_focus && delivery == DeliveryMode::Background
+ let event_kind = if mods.is_empty() {
+ EventKind::Keystroke
+ } else {
+ EventKind::KeyCombo
+ };
+ if !px_focus
+ && delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, event_kind)
{
// macOS-aligned contract: a `background` actuation never fronts. This
@@ -3518,7 +3934,7 @@ impl Tool for PressKeyTool {
// combos) and the only way to land it is a foreground/focus grab —
// which background must not do. Surface background_unavailable so the
// agent escalates to delivery_mode:"foreground" (which may front).
- return background_unavailable_error(hwnd, event_kind);
+ return crate::input::delivery::background_unavailable_error(hwnd, event_kind);
}
// Foreground: send_key_synthesized takes the SetForegroundWindow path.
// Skipped when px-focus already fronted/clicked the target — the key then
@@ -3527,7 +3943,8 @@ impl Tool for PressKeyTool {
let send_result = tokio::task::spawn_blocking(move || {
let m: Vec<&str> = mods.iter().map(String::as_str).collect();
crate::input::send_key_synthesized(hwnd, &key, &m)
- }).await;
+ })
+ .await;
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Sent {key_display} via SendInput on pid {raw_pid} (delivery_mode:foreground)."
@@ -3539,12 +3956,27 @@ impl Tool for PressKeyTool {
let result = tokio::task::spawn_blocking(move || {
let m: Vec<&str> = mods.iter().map(String::as_str).collect();
crate::input::post_key(hwnd, &key, &m)
- }).await;
+ })
+ .await;
match result {
- // Match Swift's text format 1:1: `"✅ Pressed KEY on pid X."`.
- Ok(Ok(())) => ToolResult::text(format!("✅ Pressed {key_display} on pid {raw_pid}.")),
+ Ok(Ok(())) => ToolResult::text(format!(
+ "📨 Sent {key_display} to pid {raw_pid} via PostMessage (not verified). \
+ If it didn't land, retry with delivery_mode:\"foreground\"."
+ ))
+ .with_structured(serde_json::json!({
+ "path": "key_events",
+ "key": key_display,
+ "verified": false,
+ "verify": "unreadable",
+ "effect": "unverifiable",
+ "escalation": {
+ "recommended": "foreground",
+ "reason": "background key delivery could not be confirmed — re-call \
+ with delivery_mode:\"foreground\" if a screenshot shows the key didn't land."
+ },
+ })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
}
@@ -3552,8 +3984,10 @@ impl Tool for PressKeyTool {
// ── hotkey ────────────────────────────────────────────────────────────────────
fn is_modifier(k: &str) -> bool {
- matches!(k.to_lowercase().as_str(),
- "ctrl" | "control" | "shift" | "alt" | "win" | "windows" | "cmd" | "command")
+ matches!(
+ k.to_lowercase().as_str(),
+ "ctrl" | "control" | "shift" | "alt" | "win" | "windows" | "cmd" | "command"
+ )
}
pub struct HotkeyTool {
@@ -3614,10 +4048,13 @@ impl Tool for HotkeyTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
+ use crate::input::delivery::{DeliveryMode, EventKind};
use cua_driver_core::tool_args::ArgsExt;
- use crate::input::delivery::{DeliveryMode, EventKind, background_unavailable_error};
let hwnd_opt = args.opt_u64("window_id");
- let raw_pid = match args.require_i64("pid") { Ok(v) => v, Err(e) => return e };
+ let raw_pid = match args.require_i64("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let pid = raw_pid as u32;
let delivery = DeliveryMode::from_args(&args);
@@ -3626,9 +4063,12 @@ impl Tool for HotkeyTool {
let (key, mods, full_keys) = if let Some(arr) = args.get("keys") {
let raw = match arr.as_array() {
Some(r) => r,
- None => return ToolResult::error("Missing required array field keys."),
+ None => return ToolResult::error("Missing required array field keys."),
};
- let keys: Vec<String> = raw.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect();
+ let keys: Vec<String> = raw
+ .iter()
+ .filter_map(|v| v.as_str().map(str::to_owned))
+ .collect();
if keys.len() != raw.len() || keys.is_empty() {
return ToolResult::error("keys must be a non-empty array of strings.");
}
@@ -3639,10 +4079,17 @@ impl Tool for HotkeyTool {
}
(non_mods.last().unwrap().clone(), modifiers, keys)
} else if let Some(k) = args.get("key").and_then(|v| v.as_str()) {
- let m: Vec<String> = args.get("modifiers").and_then(|v| v.as_array())
- .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect())
+ let m: Vec<String> = args
+ .get("modifiers")
+ .and_then(|v| v.as_array())
+ .map(|a| {
+ a.iter()
+ .filter_map(|v| v.as_str().map(str::to_owned))
+ .collect()
+ })
.unwrap_or_default();
- let mut full = m.clone(); full.push(k.to_owned());
+ let mut full = m.clone();
+ full.push(k.to_owned());
(k.to_owned(), m, full)
} else {
return ToolResult::error("Missing required array field keys.");
@@ -3653,10 +4100,17 @@ impl Tool for HotkeyTool {
Some(h) => h,
None => {
let pid2 = pid;
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid2))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid2)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -3667,23 +4121,36 @@ impl Tool for HotkeyTool {
let key_display = full_keys.join("+");
// px form: pixel-click to focus, then the combo acts on the focused field
- // (e.g. Ctrl+V into a Chromium input). After it, deliver the combo via the
- // plain background PostMessage path (the focus-click already fronted if
- // foreground), so skip the XAML-accelerator walk, the background_unavailable
- // check, and the foreground SendInput swap below. Mirrors macOS hotkey.
+ // (e.g. Ctrl+V into a Chromium input). A background pixel focus does not
+ // make Chromium's modifier PostMessage path reliable, so it still goes
+ // through the capability guard below; foreground remains explicit.
let px_focus = {
let px = args.get("x").and_then(|v| v.as_f64());
let py = args.get("y").and_then(|v| v.as_f64());
if let (Some(cx), Some(cy)) = (px, py) {
- let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);
+ let from_zoom = args
+ .get("from_zoom")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
if let Err(e) = focus_by_pixel(
- &self.state, pid, hwnd_opt, cx, cy, delivery.is_foreground(),
- args.opt_str("session"), args.opt_str("_session_id"), from_zoom,
- ).await {
+ &self.state,
+ pid,
+ hwnd_opt,
+ cx,
+ cy,
+ delivery.is_foreground(),
+ args.opt_str("session"),
+ args.opt_str("_session_id"),
+ from_zoom,
+ )
+ .await
+ {
return e;
}
true
- } else { false }
+ } else {
+ false
+ }
};
if !px_focus && crate::input::is_xaml_host_hwnd(hwnd) {
@@ -3701,21 +4168,17 @@ impl Tool for HotkeyTool {
&accelerator_combo,
)
});
- let result = match tokio::time::timeout(
- std::time::Duration::from_secs(4),
- blocking,
- )
- .await
- {
- Ok(join_result) => join_result,
- Err(_) => {
- return ToolResult::error(format!(
- "hotkey timed out after 4s while scanning UIA accelerators on pid \
+ let result =
+ match tokio::time::timeout(std::time::Duration::from_secs(4), blocking).await {
+ Ok(join_result) => join_result,
+ Err(_) => {
+ return ToolResult::error(format!(
+ "hotkey timed out after 4s while scanning UIA accelerators on pid \
{raw_pid} (hwnd {hwnd}). A UIA provider in this app is likely \
unresponsive."
- ));
- }
- };
+ ));
+ }
+ };
return match result {
Ok(Ok((true, _))) => ToolResult::text(format!(
"✅ Pressed {key_display_for_result} on pid {raw_pid} via UIA \
@@ -3771,16 +4234,20 @@ impl Tool for HotkeyTool {
let has_modifiers = !mods.is_empty();
// delivery_mode:"background" — refuse if PostMessage of the key combo
// would be silently dropped on this target (Chromium with modifiers).
- let event_kind = if has_modifiers { EventKind::KeyCombo } else { EventKind::Keystroke };
- if !px_focus && delivery == DeliveryMode::Background
- && crate::input::delivery::would_be_silently_dropped(hwnd, event_kind)
+ let event_kind = if has_modifiers {
+ EventKind::KeyCombo
+ } else {
+ EventKind::Keystroke
+ };
+ if delivery == DeliveryMode::Background
+ && (px_focus || crate::input::delivery::would_be_silently_dropped(hwnd, event_kind))
{
// macOS-aligned: a `background` hotkey never fronts. This combo would
// be silently dropped (VCL/classic-Win32 TranslateAccelerator, or
// Chromium key-combos) and only a foreground/focus grab lands it —
// which background must not do. Surface background_unavailable; the
// agent escalates to delivery_mode:"foreground".
- return background_unavailable_error(hwnd, event_kind);
+ return crate::input::delivery::background_unavailable_error(hwnd, event_kind);
}
// delivery_mode:"foreground" — explicit SendInput swap (the path that
// unblocks TranslateAccelerator-style apps). A Background call that
@@ -3789,10 +4256,11 @@ impl Tool for HotkeyTool {
// on PostMessage and the no-foreground contract holds.
// px-focus delivers the combo via PostMessage to the now-focused field, so
// it never takes the SendInput foreground swap.
- let use_send_input = !px_focus && match delivery {
- DeliveryMode::Foreground => true,
- DeliveryMode::Background => false,
- };
+ let use_send_input = !px_focus
+ && match delivery {
+ DeliveryMode::Foreground => true,
+ DeliveryMode::Background => false,
+ };
let result = tokio::task::spawn_blocking(move || {
let m: Vec<&str> = mods.iter().map(String::as_str).collect();
if use_send_input {
@@ -3800,15 +4268,20 @@ impl Tool for HotkeyTool {
} else {
crate::input::post_key(hwnd, &key, &m)
}
- }).await;
- let path = if use_send_input { "SendInput" } else { "PostMessage" };
+ })
+ .await;
+ let path = if use_send_input {
+ "SendInput"
+ } else {
+ "PostMessage"
+ };
match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Pressed {key_display} on pid {raw_pid} via {path} \
(Win32 target)."
)),
Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
}
@@ -3860,12 +4333,14 @@ impl Tool for SetValueTool {
let cursor_key = resolve_cursor_key(&args);
let raw_pid = args.get("pid").and_then(|v| v.as_i64());
if raw_pid.is_none() {
- return ToolResult::error("Missing required integer fields pid, window_id, and element_index.");
+ return ToolResult::error(
+ "Missing required integer fields pid, window_id, and element_index.",
+ );
}
let pid = raw_pid.unwrap() as u32;
let value = match args.get("value").and_then(|v| v.as_str()) {
Some(v) => v.to_owned(),
- None => return ToolResult::error("Missing required string field value."),
+ None => return ToolResult::error("Missing required string field value."),
};
// Surface 6: element_token / element_index precedence resolution.
let resolved = match cua_driver_core::element_token::resolve_element_args(
@@ -3880,16 +4355,23 @@ impl Tool for SetValueTool {
};
let (hwnd, idx) = match resolved {
cua_driver_core::element_token::ResolvedElement::Element {
- window_id: Some(wid), element_index, ..
+ window_id: Some(wid),
+ element_index,
+ ..
} => (wid as u64, element_index),
cua_driver_core::element_token::ResolvedElement::Element {
window_id: None, ..
- } => return ToolResult::error(
- "set_value requires window_id when element_index is used \
- (omit only when supplying element_token, which carries it)."
- ),
- cua_driver_core::element_token::ResolvedElement::None =>
- return ToolResult::error("Missing required integer fields pid, window_id, and element_index."),
+ } => {
+ return ToolResult::error(
+ "set_value requires window_id when element_index is used \
+ (omit only when supplying element_token, which carries it).",
+ )
+ }
+ cua_driver_core::element_token::ResolvedElement::None => {
+ return ToolResult::error(
+ "Missing required integer fields pid, window_id, and element_index.",
+ )
+ }
};
// No-raise guard: a WPF/XAML automation peer calls UIElement.Focus() →
@@ -3900,8 +4382,12 @@ impl Tool for SetValueTool {
let _noact = if crate::input::delivery::DeliveryMode::from_args(&args)
!= crate::input::delivery::DeliveryMode::Foreground
{
- Some(crate::input::NoActivateGuard::arm(windows::Win32::Foundation::HWND(hwnd as *mut _)))
- } else { None };
+ Some(crate::input::NoActivateGuard::arm(
+ windows::Win32::Foundation::HWND(hwnd as *mut _),
+ ))
+ } else {
+ None
+ };
// Glide the agent cursor onto the target element before writing its
// value, so a value write gets the same visual feedback as a click —
@@ -3910,9 +4396,13 @@ impl Tool for SetValueTool {
if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx) {
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: cx as f64, y: cy as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: cx as f64,
+ y: cy as f64,
+ },
+ );
}
let state = self.state.clone();
@@ -3921,12 +4411,17 @@ impl Tool for SetValueTool {
// get_window_state snapshot-replace on the same (pid, hwnd) can't
// Release it to zero while this Value/RangeValue SetValue is in
// flight. The guard is held for the whole closure.
- let element_guard = state.element_cache.get_element_retained(pid, hwnd, idx)
+ let element_guard = state
+ .element_cache
+ .get_element_retained(pid, hwnd, idx)
.ok_or_else(|| anyhow::anyhow!("Element {idx} not in cache."))?;
let ptr = element_guard.as_ptr();
- use windows::Win32::UI::Accessibility::{IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId};
use windows::core::{Interface, BSTR};
- let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
+ };
+ let elem: IUIAutomationElement =
+ unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
// Try ValuePattern first (text inputs, editable combos, etc).
// The SetValue is shielded by the EnableWindow bypass: a
// Chromium/Electron (or XAML) SetValue handler self-foregrounds via
@@ -3935,10 +4430,10 @@ impl Tool for SetValueTool {
// write still lands. (No-op shield for non-XAML/non-Chromium hosts.)
if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) } {
if let Ok(vp) = pattern.cast::<IUIAutomationValuePattern>() {
- let set = crate::uia::fg_bypass::run_with_uwp_bypass(
- hwnd as isize,
- || unsafe { vp.SetValue(&BSTR::from(value.as_str())) },
- );
+ let set =
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
+ vp.SetValue(&BSTR::from(value.as_str()))
+ });
if set.is_ok() {
std::mem::forget(elem);
return Ok("ValuePattern".to_string());
@@ -3953,15 +4448,16 @@ impl Tool for SetValueTool {
};
if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_RangeValuePatternId) } {
if let Ok(rv) = pattern.cast::<IUIAutomationRangeValuePattern>() {
- let parsed: f64 = value.parse().map_err(|_| anyhow::anyhow!(
- "set_value: target element exposes RangeValuePattern (Slider / \
+ let parsed: f64 = value.parse().map_err(|_| {
+ anyhow::anyhow!(
+ "set_value: target element exposes RangeValuePattern (Slider / \
ProgressBar / numeric range). `value` must be a parseable f64; \
got {value:?}."
- ))?;
- crate::uia::fg_bypass::run_with_uwp_bypass(
- hwnd as isize,
- || unsafe { rv.SetValue(parsed) },
- )?;
+ )
+ })?;
+ crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe {
+ rv.SetValue(parsed)
+ })?;
std::mem::forget(elem);
return Ok("RangeValuePattern".to_string());
}
@@ -3973,13 +4469,14 @@ impl Tool for SetValueTool {
SelectionItemPattern (RadioButton / ComboBoxItem), use the `click` \
tool instead."
);
- }).await;
+ })
+ .await;
match result {
- Ok(Ok(pattern_name)) => ToolResult::text(format!(
- "✅ Set AXValue on [{idx}] (UIA {pattern_name})."
- )),
+ Ok(Ok(pattern_name)) => {
+ ToolResult::text(format!("✅ Set AXValue on [{idx}] (UIA {pattern_name})."))
+ }
Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
}
@@ -4029,13 +4526,13 @@ impl Tool for ScrollTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- use crate::input::delivery::{DeliveryMode, EventKind, background_unavailable_error};
+ use crate::input::delivery::{DeliveryMode, EventKind};
use cua_driver_core::tool_args::ArgsExt;
// `direction` is required in both the pid path and the window-less
// desktop path, so resolve it before the pid check.
let direction = match args.get("direction").and_then(|v| v.as_str()) {
Some(d) => d.to_owned(),
- None => return ToolResult::error("Missing required string field direction."),
+ None => return ToolResult::error("Missing required string field direction."),
};
let amount = args.u64_or("amount", 3).clamp(1, 50) as u32;
@@ -4055,7 +4552,7 @@ impl Tool for ScrollTool {
"scroll: x,y given with no pid/window_id, but capture_scope is \
\"window\". Screen-absolute scroll requires desktop scope. Call \
set_config with capture_scope=desktop (and use get_desktop_state \
- to pick coordinates), or pass a pid/window_id."
+ to pick coordinates), or pass a pid/window_id.",
)
.with_structured(serde_json::json!({
"code": "desktop_scope_disabled",
@@ -4067,19 +4564,22 @@ impl Tool for ScrollTool {
let sy = args.f64_or("y", 0.0) as i32;
// Direction → (horizontal?, sign). Positive ticks = up / right.
let (horizontal, sign) = match direction.as_str() {
- "up" => (false, 1),
- "down" => (false, -1),
- "right" => (true, 1),
- "left" => (true, -1),
- other => return ToolResult::error(format!(
- "scroll: unknown direction \"{other}\" — expected up, down, left, right."
- )),
+ "up" => (false, 1),
+ "down" => (false, -1),
+ "right" => (true, 1),
+ "left" => (true, -1),
+ other => {
+ return ToolResult::error(format!(
+ "scroll: unknown direction \"{other}\" — expected up, down, left, right."
+ ))
+ }
};
let ticks = sign * amount as i32;
let dir_disp = direction.clone();
let result = tokio::task::spawn_blocking(move || {
crate::input::send_wheel_synthesized(sx, sy, ticks, horizontal)
- }).await;
+ })
+ .await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Scrolled {dir_disp} via SendInput wheel ({amount} tick(s)) at screen ({sx},{sy}) (desktop scope)."
@@ -4092,7 +4592,7 @@ impl Tool for ScrollTool {
// Swift error wording 1:1.
let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) => p,
- None => return ToolResult::error("Missing required integer field pid."),
+ None => return ToolResult::error("Missing required integer field pid."),
};
let pid = raw_pid as u32;
let delivery = DeliveryMode::from_args(&args);
@@ -4111,34 +4611,80 @@ impl Tool for ScrollTool {
Err(e) => return e,
};
let elem_idx: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index as u64),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index as u64)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
+ .map(|v| v as u64)
+ .or_else(|| args.opt_u64("window_id")),
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
is scoped per (pid, window_id). Pass the same window_id you used in \
- `get_window_state`.");
+ `get_window_state`.",
+ );
}
// Resolve HWND: use window_id if given, else first window for pid.
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
+ // WebView2/Tauri hosts can expose the indexed scroll container through
+ // UIA while their top-level HWND ignores WM_VSCROLL. Prefer the
+ // accessibility channel for an indexed target; the message path below
+ // remains the fallback for native Win32 scrollbars.
+ if let Some(idx) = elem_idx {
+ let state = self.state.clone();
+ let direction_for_uia = direction.clone();
+ let uia_result = tokio::task::spawn_blocking(move || {
+ let retained = state
+ .element_cache
+ .get_element_retained(pid, hwnd, idx as usize)
+ .ok_or_else(|| anyhow::anyhow!("element [{idx}] is not in the UIA cache"))?;
+ if !retained.is_uia() {
+ anyhow::bail!("element [{idx}] is not a UIA scroll element");
+ }
+ unsafe {
+ crate::uia::scroll::scroll_element(
+ retained.as_ptr(),
+ &direction_for_uia,
+ amount,
+ )
+ }
+ })
+ .await;
+ if matches!(uia_result, Ok(Ok(()))) {
+ return ToolResult::text(format!(
+ "Scrolled {direction} {amount} ticks via UIA (delivery_mode:background)."
+ ))
+ .with_structured(serde_json::json!({
+ "path": "uia",
+ "verified": false,
+ "delivery_mode": "background"
+ }));
+ }
+ }
+
// delivery_mode:"background" — WM_VSCROLL/HSCROLL is silently dropped by
// Chromium and may be by GTK. Surface the standard structured
// background_unavailable error: its remediation (bring_to_front +
@@ -4148,7 +4694,10 @@ impl Tool for ScrollTool {
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseScroll)
{
- return background_unavailable_error(hwnd, EventKind::MouseScroll);
+ return crate::input::delivery::background_unavailable_error(
+ hwnd,
+ EventKind::MouseScroll,
+ );
}
// delivery_mode:"foreground" — SendInput MOUSEEVENTF_WHEEL at the target
// window's screen center (send_wheel_synthesized). The wheel routes to
@@ -4158,13 +4707,15 @@ impl Tool for ScrollTool {
// count (page ≈ 3 detents).
if delivery == DeliveryMode::Foreground {
let (horizontal, sign) = match direction.as_str() {
- "up" => (false, 1),
- "down" => (false, -1),
- "right" => (true, 1),
- "left" => (true, -1),
- other => return ToolResult::error(format!(
- "scroll: unknown direction \"{other}\" — expected up, down, left, right."
- )),
+ "up" => (false, 1),
+ "down" => (false, -1),
+ "right" => (true, 1),
+ "left" => (true, -1),
+ other => {
+ return ToolResult::error(format!(
+ "scroll: unknown direction \"{other}\" — expected up, down, left, right."
+ ))
+ }
};
let per: i32 = if by == "page" { 3 } else { 1 };
let ticks = sign * (amount as i32) * per;
@@ -4174,49 +4725,54 @@ impl Tool for ScrollTool {
.into_iter()
.find(|w| w.hwnd == hwnd)
.map(|w| (w.x + w.width / 2, w.y + w.height / 2))
- }).await.ok().flatten();
+ })
+ .await
+ .ok()
+ .flatten();
let (cx, cy) = match center {
Some(c) => c,
- None => return ToolResult::error(format!(
- "scroll: could not resolve on-screen bounds for window {hwnd:#x} \
+ None => {
+ return ToolResult::error(format!(
+ "scroll: could not resolve on-screen bounds for window {hwnd:#x} \
(pid {pid}) to target the wheel. The window may be minimized or \
off-screen — call bring_to_front first."
- )),
+ ))
+ }
};
let dir_disp = direction.clone();
let tick_disp = ticks.abs();
let result = tokio::task::spawn_blocking(move || {
crate::input::send_wheel_synthesized(cx, cy, ticks, horizontal)
- }).await;
+ })
+ .await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Scrolled {dir_disp} via SendInput wheel ({tick_disp} tick(s)) at \
screen ({cx},{cy}) (delivery_mode:foreground)."
)),
Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
use windows::Win32::Foundation::{HWND, LPARAM, WPARAM};
use windows::Win32::UI::WindowsAndMessaging::{
- PostMessageW, WM_HSCROLL, WM_VSCROLL,
- SB_LINEDOWN, SB_LINEUP, SB_LINELEFT, SB_LINERIGHT,
- SB_PAGEDOWN, SB_PAGEUP, SB_PAGELEFT, SB_PAGERIGHT,
+ PostMessageW, SB_LINEDOWN, SB_LINELEFT, SB_LINERIGHT, SB_LINEUP, SB_PAGEDOWN,
+ SB_PAGELEFT, SB_PAGERIGHT, SB_PAGEUP, WM_HSCROLL, WM_VSCROLL,
};
let hwnd_win = HWND(hwnd as *mut _);
let use_page = by == "page";
let (msg, code) = match (direction.as_str(), use_page) {
- ("up", false) => (WM_VSCROLL, SB_LINEUP),
- ("up", true) => (WM_VSCROLL, SB_PAGEUP),
- ("down", false) => (WM_VSCROLL, SB_LINEDOWN),
- ("down", true) => (WM_VSCROLL, SB_PAGEDOWN),
- ("left", false) => (WM_HSCROLL, SB_LINELEFT),
- ("left", true) => (WM_HSCROLL, SB_PAGELEFT),
+ ("up", false) => (WM_VSCROLL, SB_LINEUP),
+ ("up", true) => (WM_VSCROLL, SB_PAGEUP),
+ ("down", false) => (WM_VSCROLL, SB_LINEDOWN),
+ ("down", true) => (WM_VSCROLL, SB_PAGEDOWN),
+ ("left", false) => (WM_HSCROLL, SB_LINELEFT),
+ ("left", true) => (WM_HSCROLL, SB_PAGELEFT),
("right", false) => (WM_HSCROLL, SB_LINERIGHT),
- ("right", true) => (WM_HSCROLL, SB_PAGERIGHT),
+ ("right", true) => (WM_HSCROLL, SB_PAGERIGHT),
_ => (WM_VSCROLL, SB_LINEDOWN),
};
for _ in 0..amount {
@@ -4225,7 +4781,8 @@ impl Tool for ScrollTool {
}
}
Ok(())
- }).await;
+ })
+ .await;
match result {
Ok(Ok(())) => {
@@ -4233,14 +4790,14 @@ impl Tool for ScrollTool {
// like "pagedown"; Windows uses Win32 SB_* constants — show
// the actual mechanism for traceability).
let mech_name = match (direction_display.as_str(), by_display.as_str()) {
- ("up", "page") => "SB_PAGEUP",
- ("down", "page") => "SB_PAGEDOWN",
- ("left", "page") => "SB_PAGELEFT",
+ ("up", "page") => "SB_PAGEUP",
+ ("down", "page") => "SB_PAGEDOWN",
+ ("left", "page") => "SB_PAGELEFT",
("right", "page") => "SB_PAGERIGHT",
- ("up", _) => "SB_LINEUP",
- ("down", _) => "SB_LINEDOWN",
- ("left", _) => "SB_LINELEFT",
- ("right", _) => "SB_LINERIGHT",
+ ("up", _) => "SB_LINEUP",
+ ("down", _) => "SB_LINEDOWN",
+ ("left", _) => "SB_LINELEFT",
+ ("right", _) => "SB_LINERIGHT",
_ => "SB_LINEDOWN",
};
ToolResult::text(format!(
@@ -4248,7 +4805,7 @@ impl Tool for ScrollTool {
))
}
Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
}
@@ -4281,20 +4838,18 @@ async fn chromium_click_short_circuit(
pid: u32,
gesture: &str,
) -> Option<ToolResult> {
- let is_chromium = tokio::task::spawn_blocking(move || {
- crate::input::is_chromium_target_window(hwnd)
- })
- .await
- .unwrap_or(false);
+ let is_chromium =
+ tokio::task::spawn_blocking(move || crate::input::is_chromium_target_window(hwnd))
+ .await
+ .unwrap_or(false);
if !is_chromium {
return None;
}
// Capture the pre-click foreground so the poller can restore it even if
// Chromium re-activates itself from a renderer-side handler (same pattern
// and rationale as the ClickTool Chromium branch).
- let prev_fg_addr = unsafe {
- windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
- };
+ let prev_fg_addr =
+ unsafe { windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize };
let button_owned = button.to_string();
let send_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized(hwnd, sx, sy, count, &button_owned)
@@ -4344,9 +4899,8 @@ fn winui3_uia_multi_invoke(
};
// Hold the frame non-activatable so the XAML Invoke handler's
// SetFocus()→SetForegroundWindow(self) is refused for the whole burst.
- let _noact = crate::input::NoActivateGuard::arm(
- windows::Win32::Foundation::HWND(hwnd as *mut _),
- );
+ let _noact =
+ crate::input::NoActivateGuard::arm(windows::Win32::Foundation::HWND(hwnd as *mut _));
let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
let outcome: Option<anyhow::Result<()>> = (|| {
let pattern = unsafe { elem.GetCurrentPattern(UIA_InvokePatternId) }.ok()?;
@@ -4399,11 +4953,10 @@ async fn winui3_background_gesture(
count: usize,
button: &str,
) -> Option<ToolResult> {
- let is_w = tokio::task::spawn_blocking(move || {
- crate::input::delivery::is_winui3_target_window(hwnd)
- })
- .await
- .unwrap_or(false);
+ let is_w =
+ tokio::task::spawn_blocking(move || crate::input::delivery::is_winui3_target_window(hwnd))
+ .await
+ .unwrap_or(false);
if !is_w {
return None;
}
@@ -4476,11 +5029,11 @@ impl Tool for DoubleClickTool {
})
}
async fn invoke(&self, args: Value) -> ToolResult {
- use crate::input::delivery::{DeliveryMode, EventKind, background_unavailable_error};
+ use crate::input::delivery::{DeliveryMode, EventKind};
// Swift error wording 1:1.
let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) => p,
- None => return ToolResult::error("Missing required integer field pid."),
+ None => return ToolResult::error("Missing required integer field pid."),
};
let pid = raw_pid as u32;
use cua_driver_core::tool_args::ArgsExt;
@@ -4496,13 +5049,15 @@ impl Tool for DoubleClickTool {
Err(e) => return e,
};
let elem_idx = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
+ .map(|v| v as u64)
+ .or_else(|| args.opt_u64("window_id")),
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
let x = args.opt_f64("x");
@@ -4510,8 +5065,8 @@ impl Tool for DoubleClickTool {
let delivery = DeliveryMode::from_args(&args);
let cursor_key = resolve_cursor_key(&args);
// Swift validates "both x and y or neither" and "no element_index without window_id".
- let has_xy = x.is_some() && y.is_some();
- let partial_xy = x.is_some() != y.is_some();
+ let has_xy = x.is_some() && y.is_some();
+ let partial_xy = x.is_some() != y.is_some();
if partial_xy {
return ToolResult::error("Provide both x and y together, not just one.");
}
@@ -4519,23 +5074,33 @@ impl Tool for DoubleClickTool {
return ToolResult::error("Provide either element_index or (x, y), not both.");
}
if elem_idx.is_none() && !has_xy {
- return ToolResult::error("Provide element_index or (x, y) to address the double-click target.");
+ return ToolResult::error(
+ "Provide element_index or (x, y) to address the double-click target.",
+ );
}
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
is scoped per (pid, window_id). Pass the same window_id you used in \
- `get_window_state`.");
+ `get_window_state`.",
+ );
}
// Resolve HWND: explicit, or auto from pid.
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -4543,17 +5108,33 @@ impl Tool for DoubleClickTool {
if let Some(idx) = elem_idx {
let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {
Some(v) => v,
- None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")),
+ None => {
+ return ToolResult::error(format!(
+ "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first."
+ ))
+ }
};
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache, pid, hwnd, idx, cx, cy, "double-clicking",
+ &self.state.element_cache,
+ pid,
+ hwnd,
+ idx,
+ cx,
+ cy,
+ "double-clicking",
) {
Ok(p) => p,
Err(msg) => return ToolResult::error(msg),
};
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: cx as f64,
+ y: cy as f64,
+ },
+ );
// delivery_mode:"background" (default) on WinUI3: the pen/touch injector
// click-activates the content island (8/8 foreground steals) and
// posted WM_*BUTTON to the frame/island no-ops (measured). A double
@@ -4563,7 +5144,9 @@ impl Tool for DoubleClickTool {
// swap. If the element has no InvokePattern, a background double-
// click isn't expressible → structured error.
if delivery == DeliveryMode::Background {
- if let Some(r) = winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 2, "left").await {
+ if let Some(r) =
+ winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 2, "left").await
+ {
return r;
}
}
@@ -4577,12 +5160,17 @@ impl Tool for DoubleClickTool {
{
let inj = tokio::task::spawn_blocking(move || {
crate::input::inject_click_screen(hwnd, cx, cy, 2, "left")
- }).await;
+ })
+ .await;
return match inj {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Injected double-click to [{idx}] at screen ({cx},{cy}) (background, no foreground swap)."
)),
- Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick),
+ Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause(
+ hwnd,
+ EventKind::MouseClick,
+ e.to_string(),
+ ),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -4593,7 +5181,8 @@ impl Tool for DoubleClickTool {
};
let send_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized(hwnd, cx, cy, 2, "left")
- }).await;
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
@@ -4604,28 +5193,42 @@ impl Tool for DoubleClickTool {
};
}
// Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput.
- if let Some(r) = chromium_click_short_circuit(hwnd, cx, cy, 2, "left", pid, "double-click").await {
+ if let Some(r) =
+ chromium_click_short_circuit(hwnd, cx, cy, 2, "left", pid, "double-click").await
+ {
return r;
}
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
crate::input::post_click_screen(hwnd, cx, cy, 2, "left")?;
// Swift text format 1:1: `"✅ Posted double-click to [N] role \"title\" at screen-point (X, Y)."`.
// UIA role/title placeholder pending element-cache enrichment.
- Ok(format!("✅ Posted double-click to [{idx}] at screen-point ({cx}, {cy})."))
- }).await;
+ Ok(format!(
+ "✅ Posted double-click to [{idx}] at screen-point ({cx}, {cy})."
+ ))
+ })
+ .await;
match result {
Ok(Ok(msg)) => ToolResult::text(msg),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else if let (Some(mut px), Some(mut py)) = (x, y) {
- let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);
+ let from_zoom = args
+ .get("from_zoom")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
if from_zoom {
match self.state.zoom_registry.get(pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(px, py); px = wx; py = wy; }
- None => return ToolResult::error(
- format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")
- ),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(px, py);
+ px = wx;
+ py = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
px *= ratio;
@@ -4637,13 +5240,18 @@ impl Tool for DoubleClickTool {
let (sx, sy) = (sx_i as f64, sy_i as f64);
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, sx, sy).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
+ );
// delivery_mode:"background" on WinUI3 (pixel path, no element_index): a
// background double-click on WinUI3 only lands via UIA Invoke on a
// cached element, which the pixel path doesn't have → structured
// error (caller retries with delivery_mode:foreground or uses element_index).
if delivery == DeliveryMode::Background {
- if let Some(r) = winui3_background_gesture(&self.state, pid, hwnd, None, 2, "left").await {
+ if let Some(r) =
+ winui3_background_gesture(&self.state, pid, hwnd, None, 2, "left").await
+ {
return r;
}
}
@@ -4654,12 +5262,17 @@ impl Tool for DoubleClickTool {
{
let inj = tokio::task::spawn_blocking(move || {
crate::input::inject_click_screen(hwnd, sx_i, sy_i, 2, "left")
- }).await;
+ })
+ .await;
return match inj {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Injected double-click to pid {pid} at screen ({sx_i},{sy_i}) (background, no foreground swap)."
)),
- Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick),
+ Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause(
+ hwnd,
+ EventKind::MouseClick,
+ e.to_string(),
+ ),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -4670,7 +5283,8 @@ impl Tool for DoubleClickTool {
};
let send_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized(hwnd, sx_i, sy_i, 2, "left")
- }).await;
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
@@ -4682,10 +5296,15 @@ impl Tool for DoubleClickTool {
}
let (xi, yi) = (px as i32, py as i32);
// Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput.
- if let Some(r) = chromium_click_short_circuit(hwnd, sx_i, sy_i, 2, "left", pid, "double-click").await {
+ if let Some(r) =
+ chromium_click_short_circuit(hwnd, sx_i, sy_i, 2, "left", pid, "double-click").await
+ {
return r;
}
- let result = tokio::task::spawn_blocking(move || crate::input::post_click_screen(hwnd, sx_i, sy_i, 2, "left")).await;
+ let result = tokio::task::spawn_blocking(move || {
+ crate::input::post_click_screen(hwnd, sx_i, sy_i, 2, "left")
+ })
+ .await;
match result {
Ok(Ok(())) => {
// Match Swift's pixel-path text 1:1.
@@ -4748,11 +5367,11 @@ impl Tool for RightClickTool {
})
}
async fn invoke(&self, args: Value) -> ToolResult {
- use crate::input::delivery::{DeliveryMode, EventKind, background_unavailable_error};
+ use crate::input::delivery::{DeliveryMode, EventKind};
// Swift error wording 1:1.
let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) => p,
- None => return ToolResult::error("Missing required integer field pid."),
+ None => return ToolResult::error("Missing required integer field pid."),
};
let pid = raw_pid as u32;
use cua_driver_core::tool_args::ArgsExt;
@@ -4768,13 +5387,15 @@ impl Tool for RightClickTool {
Err(e) => return e,
};
let elem_idx = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =>
- Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } => {
+ Some(*element_index)
+ }
cua_driver_core::element_token::ResolvedElement::None => None,
};
let hwnd_opt: Option<u64> = match &resolved {
- cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } =>
- window_id.map(|v| v as u64).or_else(|| args.opt_u64("window_id")),
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => window_id
+ .map(|v| v as u64)
+ .or_else(|| args.opt_u64("window_id")),
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
let x = args.opt_f64("x");
@@ -4782,7 +5403,7 @@ impl Tool for RightClickTool {
let delivery = DeliveryMode::from_args(&args);
let cursor_key = resolve_cursor_key(&args);
// Port Swift's full validation set.
- let has_xy = x.is_some() && y.is_some();
+ let has_xy = x.is_some() && y.is_some();
let partial_xy = x.is_some() != y.is_some();
if partial_xy {
return ToolResult::error("Provide both x and y together, not just one.");
@@ -4791,23 +5412,33 @@ impl Tool for RightClickTool {
return ToolResult::error("Provide either element_index or (x, y), not both.");
}
if elem_idx.is_none() && !has_xy {
- return ToolResult::error("Provide element_index or (x, y) to address the right-click target.");
+ return ToolResult::error(
+ "Provide element_index or (x, y) to address the right-click target.",
+ );
}
if elem_idx.is_some() && hwnd_opt.is_none() {
return ToolResult::error(
"window_id is required when element_index is used — the element_index cache \
is scoped per (pid, window_id). Pass the same window_id you used in \
- `get_window_state`.");
+ `get_window_state`.",
+ );
}
// Resolve HWND: explicit, or auto from pid.
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -4815,24 +5446,42 @@ impl Tool for RightClickTool {
if let Some(idx) = elem_idx {
let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {
Some(v) => v,
- None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")),
+ None => {
+ return ToolResult::error(format!(
+ "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first."
+ ))
+ }
};
let (cx, cy) = match resolve_onscreen_point_with_scroll(
- &self.state.element_cache, pid, hwnd, idx, cx, cy, "right-clicking",
+ &self.state.element_cache,
+ pid,
+ hwnd,
+ idx,
+ cx,
+ cy,
+ "right-clicking",
) {
Ok(p) => p,
Err(msg) => return ToolResult::error(msg),
};
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: cx as f64,
+ y: cy as f64,
+ },
+ );
// delivery_mode:"background" on WinUI3: a right-click can't both land and
// hold the contract — UIA has no right-click pattern, the content
// island ignores posted WM_RBUTTON (measured: RightTapped never
// fired), and the pen-barrel injector click-activates the frame.
// Return the structured error (retry with delivery_mode:foreground).
if delivery == DeliveryMode::Background {
- if let Some(r) = winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 1, "right").await {
+ if let Some(r) =
+ winui3_background_gesture(&self.state, pid, hwnd, Some(idx), 1, "right").await
+ {
return r;
}
}
@@ -4844,12 +5493,17 @@ impl Tool for RightClickTool {
{
let inj = tokio::task::spawn_blocking(move || {
crate::input::inject_click_screen(hwnd, cx, cy, 1, "right")
- }).await;
+ })
+ .await;
return match inj {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Injected right-click to [{idx}] at screen ({cx},{cy}) (background, no foreground swap)."
)),
- Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick),
+ Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause(
+ hwnd,
+ EventKind::MouseClick,
+ e.to_string(),
+ ),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -4859,7 +5513,8 @@ impl Tool for RightClickTool {
};
let send_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized(hwnd, cx, cy, 1, "right")
- }).await;
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
@@ -4870,7 +5525,9 @@ impl Tool for RightClickTool {
};
}
// Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput.
- if let Some(r) = chromium_click_short_circuit(hwnd, cx, cy, 1, "right", pid, "right-click").await {
+ if let Some(r) =
+ chromium_click_short_circuit(hwnd, cx, cy, 1, "right", pid, "right-click").await
+ {
return r;
}
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
@@ -4879,20 +5536,30 @@ impl Tool for RightClickTool {
// (`"✅ Shown menu for [N] role \"title\"."`). UIA role/title
// placeholder pending element-cache enrichment.
Ok(format!("✅ Shown menu for [{idx}] (screen ({cx}, {cy}))."))
- }).await;
+ })
+ .await;
match result {
Ok(Ok(msg)) => ToolResult::text(msg),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else if let (Some(mut px), Some(mut py)) = (x, y) {
- let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false);
+ let from_zoom = args
+ .get("from_zoom")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
if from_zoom {
match self.state.zoom_registry.get(pid) {
- Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(px, py); px = wx; py = wy; }
- None => return ToolResult::error(
- format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")
- ),
+ Some(ctx) => {
+ let (wx, wy) = ctx.zoom_to_window(px, py);
+ px = wx;
+ py = wy;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
px *= ratio;
@@ -4904,12 +5571,17 @@ impl Tool for RightClickTool {
let (sx, sy) = (sx_i as f64, sy_i as f64);
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, sx, sy).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
+ );
// delivery_mode:"background" on WinUI3: right-click can't land while
// holding the contract (no UIA right-click, posted WM_RBUTTON
// ignored, pen-barrel injector steals foreground) → structured error.
if delivery == DeliveryMode::Background {
- if let Some(r) = winui3_background_gesture(&self.state, pid, hwnd, None, 1, "right").await {
+ if let Some(r) =
+ winui3_background_gesture(&self.state, pid, hwnd, None, 1, "right").await
+ {
return r;
}
}
@@ -4920,12 +5592,17 @@ impl Tool for RightClickTool {
{
let inj = tokio::task::spawn_blocking(move || {
crate::input::inject_click_screen(hwnd, sx_i, sy_i, 1, "right")
- }).await;
+ })
+ .await;
return match inj {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Injected right-click to pid {pid} at screen ({sx_i},{sy_i}) (background, no foreground swap)."
)),
- Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick),
+ Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause(
+ hwnd,
+ EventKind::MouseClick,
+ e.to_string(),
+ ),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
@@ -4935,7 +5612,8 @@ impl Tool for RightClickTool {
};
let send_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized(hwnd, sx_i, sy_i, 1, "right")
- }).await;
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
@@ -4947,10 +5625,15 @@ impl Tool for RightClickTool {
}
let (xi, yi) = (px as i32, py as i32);
// Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput.
- if let Some(r) = chromium_click_short_circuit(hwnd, sx_i, sy_i, 1, "right", pid, "right-click").await {
+ if let Some(r) =
+ chromium_click_short_circuit(hwnd, sx_i, sy_i, 1, "right", pid, "right-click").await
+ {
return r;
}
- let result = tokio::task::spawn_blocking(move || crate::input::post_click_screen(hwnd, sx_i, sy_i, 1, "right")).await;
+ let result = tokio::task::spawn_blocking(move || {
+ crate::input::post_click_screen(hwnd, sx_i, sy_i, 1, "right")
+ })
+ .await;
match result {
Ok(Ok(())) => {
// Swift pixel-path text 1:1.
@@ -4958,7 +5641,7 @@ impl Tool for RightClickTool {
"✅ Posted right-click to pid {pid} at window-pixel ({xi}, {yi}) → screen-point ({sx_i}, {sy_i})."))
}
Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else {
// Guarded above by the early validation block — unreachable.
@@ -5006,7 +5689,7 @@ impl Tool for DragTool {
// Swift error wording 1:1.
let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) => p,
- None => return ToolResult::error("Missing required integer field pid."),
+ None => return ToolResult::error("Missing required integer field pid."),
};
let pid = raw_pid as u32;
let delivery = DeliveryMode::from_args(&args);
@@ -5015,46 +5698,66 @@ impl Tool for DragTool {
let cursor_key = resolve_cursor_key(&args);
// Accepts numeric JSON as either float or integer — coerce both to f64.
let coerce = |key: &str| -> Option<f64> {
- args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64))
+ args.opt_f64(key)
+ .or_else(|| args.opt_i64(key).map(|i| i as f64))
};
let from_x_opt = coerce("from_x");
let from_y_opt = coerce("from_y");
- let to_x_opt = coerce("to_x");
- let to_y_opt = coerce("to_y");
+ let to_x_opt = coerce("to_x");
+ let to_y_opt = coerce("to_y");
let (mut from_x, mut from_y, mut to_x, mut to_y) =
match (from_x_opt, from_y_opt, to_x_opt, to_y_opt) {
(Some(a), Some(b), Some(c), Some(d)) => (a, b, c, d),
- _ => return ToolResult::error(
- "from_x, from_y, to_x, and to_y are all required (window-local pixels)."),
+ _ => {
+ return ToolResult::error(
+ "from_x, from_y, to_x, and to_y are all required (window-local pixels).",
+ )
+ }
};
- let hwnd_opt = args.opt_u64("window_id");
+ let hwnd_opt = args.opt_u64("window_id");
let duration_ms = args.u64_or("duration_ms", 500);
- let steps = args.u64_or("steps", 20) as usize;
- let button = args.str_or("button", "left");
- let from_zoom = args.bool_or("from_zoom", false);
+ let steps = args.u64_or("steps", 20) as usize;
+ let button = args.str_or("button", "left");
+ let from_zoom = args.bool_or("from_zoom", false);
if from_zoom {
match self.state.zoom_registry.get(pid) {
Some(ctx) => {
let (wx, wy) = ctx.zoom_to_window(from_x, from_y);
let (wx2, wy2) = ctx.zoom_to_window(to_x, to_y);
- from_x = wx; from_y = wy; to_x = wx2; to_y = wy2;
+ from_x = wx;
+ from_y = wy;
+ to_x = wx2;
+ to_y = wy2;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
}
- None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")),
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
- from_x *= ratio; from_y *= ratio;
- to_x *= ratio; to_y *= ratio;
+ from_x *= ratio;
+ from_y *= ratio;
+ to_x *= ratio;
+ to_y *= ratio;
}
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -5063,7 +5766,7 @@ impl Tool for DragTool {
// tools — bitmap pixels are anchored to the DWM-frame top-left,
// not the client area top-left (see `bitmap_to_screen` doc).
let (sx_from, sy_from) = bitmap_to_screen(hwnd, from_x as i32, from_y as i32);
- let (sx_to, sy_to) = bitmap_to_screen(hwnd, to_x as i32, to_y as i32);
+ let (sx_to, sy_to) = bitmap_to_screen(hwnd, to_x as i32, to_y as i32);
// delivery_mode:"background" on WinUI3: a pointer drag can't both land and
// hold the contract — the content island only consumes real
@@ -5074,7 +5777,9 @@ impl Tool for DragTool {
// (a WinUI3 Slider can also be set with the set_value tool, which uses
// UIA RangeValuePattern and holds the contract).
if delivery == DeliveryMode::Background {
- if let Some(r) = winui3_background_gesture(&self.state, pid, hwnd, None, 1, &button).await {
+ if let Some(r) =
+ winui3_background_gesture(&self.state, pid, hwnd, None, 1, &button).await
+ {
return r;
}
}
@@ -5092,21 +5797,35 @@ impl Tool for DragTool {
let btn = button.clone();
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, sx_from as f64, sy_from as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: sx_from as f64, y: sy_from as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx_from as f64,
+ y: sy_from as f64,
+ },
+ );
let inj = tokio::task::spawn_blocking(move || {
crate::input::inject::inject_drag_screen(
- target, sx_from, sy_from, sx_to, sy_to, steps.max(8), &btn,
+ target,
+ sx_from,
+ sy_from,
+ sx_to,
+ sy_to,
+ steps.max(8),
+ &btn,
)
})
.await;
return match inj {
Ok(Ok(())) => {
overlay_glide_to(&cursor_key, sx_to as f64, sy_to as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: sx_to as f64, y: sy_to as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx_to as f64,
+ y: sy_to as f64,
+ },
+ );
ToolResult::text(format!(
"✅ Sent drag via synthetic-pen injection on pid {raw_pid} \
from screen ({sx_from},{sy_from}) → ({sx_to},{sy_to}) \
@@ -5131,13 +5850,22 @@ impl Tool for DragTool {
let send_result = tokio::task::spawn_blocking(move || {
crate::input::mouse::send_drag_synthesized(
hwnd,
- sx_from, sy_from,
- sx_to, sy_to,
- duration_ms, steps, &btn_fg,
+ sx_from,
+ sy_from,
+ sx_to,
+ sy_to,
+ duration_ms,
+ steps,
+ &btn_fg,
)
- }).await;
+ })
+ .await;
tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
- let button_suffix = if button == "left" { String::new() } else { format!(" ({} button)", button) };
+ let button_suffix = if button == "left" {
+ String::new()
+ } else {
+ format!(" ({} button)", button)
+ };
return match send_result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Sent drag{button_suffix} via SendInput on pid {raw_pid} \
@@ -5163,9 +5891,13 @@ impl Tool for DragTool {
// pre- and post-glides plus the press/release pulses are enough
// signal for a user watching the agent operate.
overlay_glide_to(&cursor_key, sx_from as f64, sy_from as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: sx_from as f64, y: sy_from as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx_from as f64,
+ y: sy_from as f64,
+ },
+ );
let button_c = button.clone();
let result = tokio::task::spawn_blocking(move || {
@@ -5174,26 +5906,39 @@ impl Tool for DragTool {
// not the top-level frame that would ignore it.
crate::input::mouse::post_drag_screen(
hwnd,
- sx_from, sy_from,
- sx_to, sy_to,
- duration_ms, steps, &button_c,
+ sx_from,
+ sy_from,
+ sx_to,
+ sy_to,
+ duration_ms,
+ steps,
+ &button_c,
)
- }).await;
+ })
+ .await;
// Drag finished — glide the visual cursor to the drag-end and
// pulse the release. Skipped on error so the cursor doesn't lie
// about a successful endpoint.
if matches!(&result, Ok(Ok(()))) {
overlay_glide_to(&cursor_key, sx_to as f64, sy_to as f64).await;
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
- x: sx_to as f64, y: sy_to as f64,
- });
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx_to as f64,
+ y: sy_to as f64,
+ },
+ );
}
match result {
Ok(Ok(())) => {
// Match Swift's text format verbatim.
- let button_suffix = if button == "left" { String::new() } else { format!(" ({} button)", button) };
+ let button_suffix = if button == "left" {
+ String::new()
+ } else {
+ format!(" ({} button)", button)
+ };
ToolResult::text(format!(
"✅ Posted drag{button_suffix} to pid {raw_pid} \
from window-pixel ({}, {}) → ({}, {}), \
@@ -5203,7 +5948,7 @@ impl Tool for DragTool {
))
}
Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
}
@@ -5290,8 +6035,8 @@ impl Tool for GetDesktopStateTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- use cua_driver_core::tool_args::ArgsExt;
use cua_driver_core::protocol::Content;
+ use cua_driver_core::tool_args::ArgsExt;
// Gate on the global capture_scope: a full-display capture is a
// desktop-scope operation, so it's only available when the agent has
@@ -5367,7 +6112,11 @@ impl Tool for GetDesktopStateTool {
structured["screenshot_file_path"] = json!(fp);
}
- ToolResult { content, is_error: None, structured_content: Some(structured) }
+ ToolResult {
+ content,
+ is_error: None,
+ structured_content: Some(structured),
+ }
}
}
@@ -5381,16 +6130,23 @@ impl Tool for GetCursorPositionTool {
fn def(&self) -> &ToolDef {
GCP_DEF.get_or_init(|| ToolDef {
name: "get_cursor_position".into(),
- description: "Return the current mouse cursor position in screen points (origin top-left).".into(),
+ description:
+ "Return the current mouse cursor position in screen points (origin top-left)."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
use windows::Win32::Foundation::POINT;
use windows::Win32::UI::WindowsAndMessaging::GetCursorPos;
let mut pt = POINT { x: 0, y: 0 };
- unsafe { let _ = GetCursorPos(&mut pt); }
+ unsafe {
+ let _ = GetCursorPos(&mut pt);
+ }
// Text format matches Swift `GetCursorPositionTool` 1:1.
ToolResult::text(format!("✅ Cursor at ({}, {})", pt.x, pt.y))
.with_structured(json!({ "x": pt.x, "y": pt.y }))
@@ -5410,12 +6166,17 @@ impl Tool for MoveCursorTool {
fn def(&self) -> &ToolDef {
MCURSOR_DEF.get_or_init(|| ToolDef {
name: "move_cursor".into(),
- description: "Move the agent cursor overlay to (x, y). Does NOT move the real mouse cursor.".into(),
+ description:
+ "Move the agent cursor overlay to (x, y). Does NOT move the real mouse cursor."
+ .into(),
input_schema: json!({"type":"object","required":["x","y"],"properties":{
"session": cua_driver_core::tool_schema::session_schema(),
"x":{"type":"number"},"y":{"type":"number"},"cursor_id":{"type":"string"}
},"additionalProperties":false}),
- read_only: false, destructive: false, idempotent: true, open_world: false,
+ read_only: false,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, args: Value) -> ToolResult {
@@ -5426,15 +6187,26 @@ impl Tool for MoveCursorTool {
// > NO_CURSOR. An anonymous run (no session) has no cursor to move.
let cursor_key = resolve_cursor_key(&args);
if !cursor_key.is_empty() {
- self.state.cursor_registry.update_position(&cursor_key, x, y);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_key, x, y);
}
// End pointing upper-left (45°) — matches Swift's
// `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so
// the cursor settles to the natural macOS-style pose.
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::MoveTo {
- x, y, end_heading_radians: std::f64::consts::FRAC_PI_4,
- });
- let shown = if cursor_key.is_empty() { "default" } else { cursor_key.as_str() };
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::MoveTo {
+ x,
+ y,
+ end_heading_radians: std::f64::consts::FRAC_PI_4,
+ },
+ );
+ let shown = if cursor_key.is_empty() {
+ "default"
+ } else {
+ cursor_key.as_str()
+ };
ToolResult::text(format!("Agent cursor '{shown}' moved to ({x:.1}, {y:.1})."))
}
}
@@ -5473,13 +6245,16 @@ impl Tool for SetAgentCursorEnabledTool {
// Swift error wording 1:1.
let enabled = match args.get("enabled").and_then(|v| v.as_bool()) {
Some(v) => v,
- None => return ToolResult::error("Missing required boolean field `enabled`."),
+ None => return ToolResult::error("Missing required boolean field `enabled`."),
};
let cursor_key = resolve_cursor_key(&args);
if !cursor_key.is_empty() {
self.state.cursor_registry.set_enabled(&cursor_key, enabled);
}
- crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::SetEnabled(enabled));
+ crate::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::SetEnabled(enabled),
+ );
// Match Swift text format 1:1: `"✅ Agent cursor enabled."`
// (or `"✅ Agent cursor disabled."`).
ToolResult::text(if enabled {
@@ -5557,19 +6332,33 @@ impl Tool for SetAgentCursorMotionTool {
let icon_owned = icon.to_owned();
match tokio::task::spawn_blocking(move || {
cursor_overlay::resolve_cursor_icon(&icon_owned)
- }).await {
- Ok(Ok(resolution)) => shape_cmd = Some(cursor_overlay::OverlayCommand::from_cursor_icon(resolution)),
+ })
+ .await
+ {
+ Ok(Ok(resolution)) => {
+ shape_cmd = Some(cursor_overlay::OverlayCommand::from_cursor_icon(resolution))
+ }
Ok(Err(e)) => return ToolResult::error(format!("Invalid cursor_icon: {e}")),
Err(e) => return ToolResult::error(format!("Task error: {e}")),
}
}
// 1. Per-instance appearance fields (Rust-only).
self.state.cursor_registry.update_config(&cursor_id, |cfg| {
- if let Some(v) = args.get("cursor_icon").and_then(|v| v.as_str()) { cfg.cursor_icon = Some(v.to_owned()); }
- if let Some(v) = args.get("cursor_color").and_then(|v| v.as_str()) { cfg.cursor_color = Some(v.to_owned()); }
- if let Some(v) = args.get("cursor_label").and_then(|v| v.as_str()) { cfg.cursor_label = Some(v.to_owned()); }
- if let Some(v) = num(args.get("cursor_size")) { cfg.cursor_size = Some(v); }
- if let Some(v) = num(args.get("cursor_opacity")) { cfg.cursor_opacity = Some(v.clamp(0.0, 1.0)); }
+ if let Some(v) = args.get("cursor_icon").and_then(|v| v.as_str()) {
+ cfg.cursor_icon = Some(v.to_owned());
+ }
+ if let Some(v) = args.get("cursor_color").and_then(|v| v.as_str()) {
+ cfg.cursor_color = Some(v.to_owned());
+ }
+ if let Some(v) = args.get("cursor_label").and_then(|v| v.as_str()) {
+ cfg.cursor_label = Some(v.to_owned());
+ }
+ if let Some(v) = num(args.get("cursor_size")) {
+ cfg.cursor_size = Some(v);
+ }
+ if let Some(v) = num(args.get("cursor_opacity")) {
+ cfg.cursor_opacity = Some(v.clamp(0.0, 1.0));
+ }
});
if let Some(cmd) = shape_cmd {
crate::overlay::send_command(cursor_id.clone(), cmd);
@@ -5589,7 +6378,10 @@ impl Tool for SetAgentCursorMotionTool {
None, // press_duration_ms — not in Swift tool surface
num(args.get("turn_radius")),
);
- crate::overlay::send_command(cursor_id.clone(), cursor_overlay::OverlayCommand::SetMotion(updated.clone()));
+ crate::overlay::send_command(
+ cursor_id.clone(),
+ cursor_overlay::OverlayCommand::SetMotion(updated.clone()),
+ );
// Match Swift text format 1:1.
let summary = format!(
"cursor motion: startHandle={sh} endHandle={eh} arcSize={asz} arcFlow={af} \
@@ -5634,9 +6426,13 @@ impl Tool for GetAgentCursorStateTool {
description: "Report the current agent-cursor configuration: enabled flag, \
motion knobs (startHandle, endHandle, arcSize, arcFlow, spring), glide \
duration, post-click dwell, and idle-hide delay. Durations come back in \
- milliseconds to match the setter's units. Pure read-only — no side effects.".into(),
+ milliseconds to match the setter's units. Pure read-only — no side effects."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, args: Value) -> ToolResult {
@@ -5644,7 +6440,7 @@ impl Tool for GetAgentCursorStateTool {
// > "default"), mirroring macOS get_agent_cursor_state scoping.
let cursor_key = resolve_cursor_key(&args);
let enabled = crate::overlay::is_enabled(&cursor_key);
- let motion = crate::overlay::current_motion(&cursor_key);
+ let motion = crate::overlay::current_motion(&cursor_key);
// Swift text format 1:1: single-line camelCase key=value pairs.
let summary = format!(
"cursor: enabled={enabled} startHandle={sh} endHandle={eh} arcSize={asz} \
@@ -5659,21 +6455,21 @@ impl Tool for GetAgentCursorStateTool {
);
// Rust-only structured payload: the same fields + the multi-cursor
// instance map. Cursor instances are a Rust-only extension.
- let cursors = serde_json::to_value(self.state.cursor_registry.all_states()).unwrap_or_default();
- ToolResult::text(format!("✅ {summary}"))
- .with_structured(json!({
- "enabled": enabled,
- "start_handle": motion.start_handle,
- "end_handle": motion.end_handle,
- "arc_size": motion.arc_size,
- "arc_flow": motion.arc_flow,
- "spring": motion.spring,
- "glide_duration_ms": motion.glide_duration_ms,
- "dwell_after_click_ms": motion.dwell_after_click_ms,
- "idle_hide_ms": motion.idle_hide_ms,
- "turn_radius": motion.turn_radius,
- "cursors": cursors,
- }))
+ let cursors =
+ serde_json::to_value(self.state.cursor_registry.all_states()).unwrap_or_default();
+ ToolResult::text(format!("✅ {summary}")).with_structured(json!({
+ "enabled": enabled,
+ "start_handle": motion.start_handle,
+ "end_handle": motion.end_handle,
+ "arc_size": motion.arc_size,
+ "arc_flow": motion.arc_flow,
+ "spring": motion.spring,
+ "glide_duration_ms": motion.glide_duration_ms,
+ "dwell_after_click_ms": motion.dwell_after_click_ms,
+ "idle_hide_ms": motion.idle_hide_ms,
+ "turn_radius": motion.turn_radius,
+ "cursors": cursors,
+ }))
}
}
@@ -5742,7 +6538,9 @@ impl Tool for SetAgentCursorStyleTool {
let path_owned = path.to_owned();
match tokio::task::spawn_blocking(move || {
cursor_overlay::CursorShape::load(&path_owned)
- }).await {
+ })
+ .await
+ {
Ok(Ok(shape)) => {
let path_owned2 = path.to_owned();
self.state.cursor_registry.update_config(&cursor_id, |c| {
@@ -5750,7 +6548,9 @@ impl Tool for SetAgentCursorStyleTool {
});
Some(cursor_overlay::OverlayCommand::SetShape(Some(shape)))
}
- Ok(Err(e)) => return ToolResult::error(format!("Failed to load image_path: {e}")),
+ Ok(Err(e)) => {
+ return ToolResult::error(format!("Failed to load image_path: {e}"))
+ }
Err(e) => return ToolResult::error(format!("Task error: {e}")),
}
}
@@ -5759,34 +6559,36 @@ impl Tool for SetAgentCursorStyleTool {
};
// gradient_colors
- let gradient_colors: Vec<[u8; 4]> = if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {
- let mut out = vec![];
- for v in arr {
- if let Some(hex) = v.as_str() {
- match parse_hex_color(hex) {
- Some(c) => out.push(c),
- None => return ToolResult::error(format!("Invalid hex color: {hex}")),
+ let gradient_colors: Vec<[u8; 4]> =
+ if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {
+ let mut out = vec![];
+ for v in arr {
+ if let Some(hex) = v.as_str() {
+ match parse_hex_color(hex) {
+ Some(c) => out.push(c),
+ None => return ToolResult::error(format!("Invalid hex color: {hex}")),
+ }
}
}
- }
- out
- } else {
- vec![]
- };
+ out
+ } else {
+ vec![]
+ };
// bloom_color
- let bloom_color: Option<Option<[u8; 4]>> = if let Some(hex) = args.get("bloom_color").and_then(|v| v.as_str()) {
- if hex.is_empty() {
- Some(None)
- } else {
- match parse_hex_color(hex) {
- Some(c) => Some(Some(c)),
- None => return ToolResult::error(format!("Invalid bloom_color: {hex}")),
+ let bloom_color: Option<Option<[u8; 4]>> =
+ if let Some(hex) = args.get("bloom_color").and_then(|v| v.as_str()) {
+ if hex.is_empty() {
+ Some(None)
+ } else {
+ match parse_hex_color(hex) {
+ Some(c) => Some(Some(c)),
+ None => return ToolResult::error(format!("Invalid bloom_color: {hex}")),
+ }
}
- }
- } else {
- None
- };
+ } else {
+ None
+ };
// Dispatch to overlay
if let Some(cmd) = shape_cmd {
@@ -5795,10 +6597,13 @@ impl Tool for SetAgentCursorStyleTool {
let gradient_provided = args.get("gradient_colors").is_some();
let bloom_provided = args.get("bloom_color").is_some();
if gradient_provided || bloom_provided {
- crate::overlay::send_command(cursor_id.clone(), cursor_overlay::OverlayCommand::SetGradient {
- gradient_colors,
- bloom_color: bloom_color.flatten(),
- });
+ crate::overlay::send_command(
+ cursor_id.clone(),
+ cursor_overlay::OverlayCommand::SetGradient {
+ gradient_colors,
+ bloom_color: bloom_color.flatten(),
+ },
+ );
}
// Swift `SetAgentCursorStyleTool` text format: only include fields
@@ -5807,7 +6612,8 @@ impl Tool for SetAgentCursorStyleTool {
// field is empty.
let mut parts: Vec<String> = Vec::new();
if let Some(arr) = args.get("gradient_colors").and_then(|v| v.as_array()) {
- let hexes: Vec<String> = arr.iter()
+ let hexes: Vec<String> = arr
+ .iter()
.filter_map(|v| v.as_str().map(str::to_owned))
.collect();
if !hexes.is_empty() {
@@ -5815,10 +6621,14 @@ impl Tool for SetAgentCursorStyleTool {
}
}
if let Some(s) = args.get("bloom_color").and_then(|v| v.as_str()) {
- if !s.is_empty() { parts.push(format!("bloom_color={s}")); }
+ if !s.is_empty() {
+ parts.push(format!("bloom_color={s}"));
+ }
}
if let Some(s) = image_path {
- if !s.is_empty() { parts.push(format!("image_path={s}")); }
+ if !s.is_empty() {
+ parts.push(format!("image_path={s}"));
+ }
}
let summary = if parts.is_empty() {
"reverted to default".to_owned()
@@ -5860,7 +6670,10 @@ impl Tool for CheckPermissionsTool {
name: "check_permissions".into(),
description: "Check required permissions for cua-driver-rs on Windows.".into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
@@ -5925,7 +6738,11 @@ impl Tool for CheckPermissionsTool {
"Process integrity level: {il_name} (RID 0x{rid:04X}, {})\n\
UIA accessibility: available (no special permission needed)\n\
PostMessage injection: available",
- if is_elevated { "elevated" } else { "non-elevated" }
+ if is_elevated {
+ "elevated"
+ } else {
+ "non-elevated"
+ }
),
None => format!(
"Process integrity level: {il_name} (token query failed)\n\
@@ -5933,14 +6750,13 @@ impl Tool for CheckPermissionsTool {
PostMessage injection: available"
),
};
- ToolResult::text(status_text)
- .with_structured(json!({
- "elevated": is_elevated,
- "integrity_level": il_name,
- "integrity_level_rid": il_rid,
- "uia": true,
- "post_message": true
- }))
+ ToolResult::text(status_text).with_structured(json!({
+ "elevated": is_elevated,
+ "integrity_level": il_name,
+ "integrity_level_rid": il_rid,
+ "uia": true,
+ "post_message": true
+ }))
}
}
@@ -5991,7 +6807,14 @@ if ($null -ne $rid) {{ Write-Output ([int]$rid) }}
pid = pid
);
let out = std::process::Command::new("powershell.exe")
- .args(["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", &script])
+ .args([
+ "-NoProfile",
+ "-NonInteractive",
+ "-ExecutionPolicy",
+ "Bypass",
+ "-Command",
+ &script,
+ ])
.output()
.ok()?;
if !out.status.success() {
@@ -6102,10 +6925,9 @@ impl Tool for SetConfigTool {
let mut cfg = self.state.config.write().unwrap();
let mut applied = false;
// Swift-compatible {key, value} shape.
- if let (Some(key), Some(val)) = (
- args.get("key").and_then(|v| v.as_str()),
- args.get("value"),
- ) {
+ if let (Some(key), Some(val)) =
+ (args.get("key").and_then(|v| v.as_str()), args.get("value"))
+ {
match key {
"capture_mode" => match val.as_str() {
Some(s) => {
@@ -6169,17 +6991,23 @@ impl Tool for SetConfigTool {
// Legacy per-field shape.
if let Some(mode) = args.get("capture_mode").and_then(|v| v.as_str()) {
cfg.capture_mode = mode.to_owned();
- if let Err(e) = pip_preview::write_config_key("capture_mode", Value::String(mode.to_owned())) {
+ if let Err(e) =
+ pip_preview::write_config_key("capture_mode", Value::String(mode.to_owned()))
+ {
tracing::warn!("set_config: failed to persist capture_mode: {e}");
}
applied = true;
}
if let Some(scope) = args.get("capture_scope").and_then(|v| v.as_str()) {
if !matches!(scope, "window" | "desktop") {
- return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."));
+ return ToolResult::error(format!(
+ "`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."
+ ));
}
cfg.capture_scope = scope.to_owned();
- if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.to_owned())) {
+ if let Err(e) =
+ pip_preview::write_config_key("capture_scope", Value::String(scope.to_owned()))
+ {
tracing::warn!("set_config: failed to persist capture_scope: {e}");
}
applied = true;
@@ -6192,28 +7020,42 @@ impl Tool for SetConfigTool {
applied = true;
}
if let Some(enabled) = args.get("experimental_pip").and_then(|v| v.as_bool()) {
- if let Err(e) = pip_preview::write_config_key("experimental_pip", Value::Bool(enabled)) {
+ if let Err(e) = pip_preview::write_config_key("experimental_pip", Value::Bool(enabled))
+ {
return ToolResult::error(format!("failed to persist experimental_pip: {e}"));
}
applied = true;
}
- if let Some(geom) = args.get("experimental_pip_geometry").and_then(|v| v.as_str()) {
+ if let Some(geom) = args
+ .get("experimental_pip_geometry")
+ .and_then(|v| v.as_str())
+ {
if pip_preview::PipGeometry::parse(geom).is_none() {
return ToolResult::error(format!(
"experimental_pip_geometry `{geom}` is not a valid WxH or WxH+X+Y string"
));
}
- if let Err(e) = pip_preview::write_config_key("experimental_pip_geometry", Value::String(geom.to_owned())) {
- return ToolResult::error(format!("failed to persist experimental_pip_geometry: {e}"));
+ if let Err(e) = pip_preview::write_config_key(
+ "experimental_pip_geometry",
+ Value::String(geom.to_owned()),
+ ) {
+ return ToolResult::error(format!(
+ "failed to persist experimental_pip_geometry: {e}"
+ ));
}
applied = true;
}
if !applied {
- return ToolResult::error("Missing required string field `key` (or a known legacy per-field).");
+ return ToolResult::error(
+ "Missing required string field `key` (or a known legacy per-field).",
+ );
}
// Emit the same pretty-JSON payload as `get_config` (matches Swift's
// `set_config` return shape — both tools echo the full config after).
- let cursor_enabled = self.state.cursor_registry.all_states()
+ let cursor_enabled = self
+ .state
+ .cursor_registry
+ .all_states()
.first()
.map(|s| s.config.enabled)
.unwrap_or(true);
@@ -6248,19 +7090,29 @@ impl Tool for GetAccessibilityTreeTool {
description: "Return a lightweight snapshot of the desktop: running processes and \
on-screen visible windows with their bounds and owner pid.\n\n\
For the full UIA subtree of a single window (with interactive element indices \
- you can click by), use get_window_state instead — this is a fast discovery read.".into(),
+ you can click by), use get_window_state instead — this is a fast discovery read."
+ .into(),
input_schema: json!({"type":"object","properties":{},"additionalProperties":false}),
- read_only: true, destructive: false, idempotent: true, open_world: false,
+ read_only: true,
+ destructive: false,
+ idempotent: true,
+ open_world: false,
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
let (procs, windows) = tokio::task::spawn_blocking(|| {
- (crate::win32::list_processes(), crate::win32::list_windows(None))
- }).await.unwrap_or_default();
+ (
+ crate::win32::list_processes(),
+ crate::win32::list_windows(None),
+ )
+ })
+ .await
+ .unwrap_or_default();
let mut lines = vec![format!(
"{} running process(es), {} visible window(s)",
- procs.len(), windows.len()
+ procs.len(),
+ windows.len()
)];
for p in &procs {
lines.push(format!("- {} (pid {})", p.name, p.pid));
@@ -6269,14 +7121,19 @@ impl Tool for GetAccessibilityTreeTool {
lines.push(String::new());
lines.push("Windows:".to_owned());
for w in &windows {
- let title = if w.title.is_empty() { "(no title)".to_owned() }
- else { format!("\"{}\"", w.title) };
+ let title = if w.title.is_empty() {
+ "(no title)".to_owned()
+ } else {
+ format!("\"{}\"", w.title)
+ };
lines.push(format!(
"- pid={} {} [window_id: {}] {}x{}+{}+{}",
w.pid, title, w.hwnd, w.width, w.height, w.x, w.y
));
}
- lines.push("→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned());
+ lines.push(
+ "→ Call get_window_state(pid, window_id) to inspect a window's UI.".to_owned(),
+ );
}
let structured = json!({
@@ -6334,15 +7191,18 @@ impl Tool for ZoomTool {
async fn invoke(&self, args: Value) -> ToolResult {
let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) => p,
- None => return ToolResult::error("Missing required integer field pid."),
+ None => return ToolResult::error("Missing required integer field pid."),
};
let pid = Some(raw_pid as u32);
let hwnd = match args.get("window_id").and_then(|v| v.as_u64()) {
Some(v) => v,
- None => return ToolResult::error("Missing required integer field window_id."),
+ None => return ToolResult::error("Missing required integer field window_id."),
};
use cua_driver_core::tool_args::ArgsExt;
- let coerce = |k: &str| args.opt_f64(k).or_else(|| args.opt_i64(k).map(|i| i as f64));
+ let coerce = |k: &str| {
+ args.opt_f64(k)
+ .or_else(|| args.opt_i64(k).map(|i| i as f64))
+ };
let (x1, y1, x2, y2) = match (coerce("x1"), coerce("y1"), coerce("x2"), coerce("y2")) {
(Some(a), Some(b), Some(c), Some(d)) => (a, b, c, d),
_ => return ToolResult::error("Missing required region coordinates (x1, y1, x2, y2)."),
@@ -6362,23 +7222,31 @@ impl Tool for ZoomTool {
// runs on a fresh native-resolution capture. Scale by the stored resize
// ratio so the crop lands on the intended region; the zoom context then
// holds native-pixel values, which is what `from_zoom` clicks expect.
- let ratio = self.state.resize_registry.ratio(raw_pid as u32).unwrap_or(1.0);
+ let ratio = self
+ .state
+ .resize_registry
+ .ratio(raw_pid as u32)
+ .unwrap_or(1.0);
let (nx1, ny1, nx2, ny2) = (x1 * ratio, y1 * ratio, x2 * ratio, y2 * ratio);
let state = self.state.clone();
let result = tokio::task::spawn_blocking(move || {
let png = crate::capture::screenshot_window_bytes(hwnd)?;
cursor_overlay::capture_utils::crop_png_to_jpeg(&png, nx1, ny1, nx2, ny2, 500)
- }).await;
+ })
+ .await;
match result {
Ok(Ok(crop)) => {
if let Some(p) = pid {
- state.zoom_registry.set(p, ZoomContext {
- origin_x: crop.origin_x,
- origin_y: crop.origin_y,
- scale_inv: crop.scale_inv,
- });
+ state.zoom_registry.set(
+ p,
+ ZoomContext {
+ origin_x: crop.origin_x,
+ origin_y: crop.origin_y,
+ scale_inv: crop.scale_inv,
+ },
+ );
}
use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
let b64 = B64.encode(&crop.jpeg_bytes);
@@ -6389,12 +7257,10 @@ impl Tool for ZoomTool {
To click a target in this image, use \
`click(pid, x, y, from_zoom=true)` where x,y are pixel \
coordinates in THIS zoomed image — the driver maps them \
- back automatically.".to_owned();
+ back automatically."
+ .to_owned();
ToolResult {
- content: vec![
- Content::image_jpeg(b64),
- Content::text(summary),
- ],
+ content: vec![Content::image_jpeg(b64), Content::text(summary)],
is_error: None,
// Surface 7: `mime_type` mirrors the MCP image part's `mimeType`
// onto the structured payload (additive — `format` stays).
@@ -6412,7 +7278,9 @@ impl Tool for ZoomTool {
// ── type_text_chars ───────────────────────────────────────────────────────────
-pub struct TypeTextCharsTool { _state: Arc<ToolState> }
+pub struct TypeTextCharsTool {
+ _state: Arc<ToolState>,
+}
static TYPE_CHARS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
#[async_trait]
@@ -6440,7 +7308,10 @@ impl Tool for TypeTextCharsTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let pid = args.u64_or("pid", 0) as u32;
- let text_raw = match args.require_str("text") { Ok(v) => v, Err(e) => return e };
+ let text_raw = match args.require_str("text") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Same trailing-protocol-tag scrub as the main TypeTextTool — see
// cua_driver_core::text_sanitize for rationale.
let text = cua_driver_core::text_sanitize::strip_trailing_agent_protocol_tags(&text_raw)
@@ -6450,19 +7321,29 @@ impl Tool for TypeTextCharsTool {
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("No windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "No windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
let text_len = text.chars().count();
let result = tokio::task::spawn_blocking(move || {
crate::input::post_type_text_with_delay(hwnd, &text, delay_ms)
- }).await;
+ })
+ .await;
match result {
- Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) with {delay_ms}ms delay.")),
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) with {delay_ms}ms delay."
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -6526,17 +7407,27 @@ impl Tool for BringToFrontTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_u32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
let hwnd_opt = args.opt_u64("window_id");
// Resolve hwnd: explicit, or auto from pid.
let hwnd = match hwnd_opt {
Some(h) => h,
None => {
- let windows = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))).await.unwrap_or_default();
+ let windows =
+ tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid)))
+ .await
+ .unwrap_or_default();
match windows.first() {
Some(w) => w.hwnd,
- None => return ToolResult::error(format!("bring_to_front: no windows found for pid {pid}. Provide window_id.")),
+ None => {
+ return ToolResult::error(format!(
+ "bring_to_front: no windows found for pid {pid}. Provide window_id."
+ ))
+ }
}
}
};
@@ -6547,11 +7438,11 @@ impl Tool for BringToFrontTool {
// Edge launch focus-steal recovery case.
let outcome = tokio::task::spawn_blocking(move || -> Result<(u64, u64, bool), String> {
use windows::Win32::Foundation::HWND;
+ use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};
use windows::Win32::UI::WindowsAndMessaging::{
GetForegroundWindow, GetWindowThreadProcessId, IsWindow, SetForegroundWindow,
SetWindowPos, HWND_NOTOPMOST, HWND_TOPMOST, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE,
};
- use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};
let target = HWND(hwnd as *mut _);
if !unsafe { IsWindow(target) }.as_bool() {
@@ -6568,8 +7459,26 @@ impl Tool for BringToFrontTool {
// is not gated by the foreground-lock / UIAccess. This is the same
// technique the delivery_mode:"foreground" pointer path uses.
let raised = unsafe {
- let a = SetWindowPos(target, HWND_TOPMOST, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE).is_ok();
- let b = SetWindowPos(target, HWND_NOTOPMOST, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE).is_ok();
+ let a = SetWindowPos(
+ target,
+ HWND_TOPMOST,
+ 0,
+ 0,
+ 0,
+ 0,
+ SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
+ )
+ .is_ok();
+ let b = SetWindowPos(
+ target,
+ HWND_NOTOPMOST,
+ 0,
+ 0,
+ 0,
+ 0,
+ SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
+ )
+ .is_ok();
a && b
};
@@ -6591,7 +7500,8 @@ impl Tool for BringToFrontTool {
}
let now_fg = unsafe { GetForegroundWindow() };
Ok((prev_fg_addr, now_fg.0 as u64, raised))
- }).await;
+ })
+ .await;
match outcome {
Ok(Ok((prev, now, raised))) => {
@@ -6625,7 +7535,9 @@ impl Tool for BringToFrontTool {
}))
}
Ok(Err(e)) => ToolResult::error(format!("bring_to_front: {e}")),
- Err(join) => ToolResult::error(format!("bring_to_front: blocking-task join error: {join}")),
+ Err(join) => {
+ ToolResult::error(format!("bring_to_front: blocking-task join error: {join}"))
+ }
}
}
}
@@ -6658,8 +7570,14 @@ impl Tool for KillAppTool {
async fn invoke(&self, args: Value) -> ToolResult {
let pid_v: u32 = match args.get("pid").and_then(|v| v.as_u64()) {
Some(p) if p > 0 && p <= u32::MAX as u64 => p as u32,
- Some(_) => return ToolResult::error("kill_app: `pid` must be a positive integer".to_string()),
- None => return ToolResult::error("kill_app: missing required integer field `pid`".to_string()),
+ Some(_) => {
+ return ToolResult::error("kill_app: `pid` must be a positive integer".to_string())
+ }
+ None => {
+ return ToolResult::error(
+ "kill_app: missing required integer field `pid`".to_string(),
+ )
+ }
};
// Run the syscalls on a blocking thread — TerminateProcess + the
@@ -6668,8 +7586,8 @@ impl Tool for KillAppTool {
let outcome = tokio::task::spawn_blocking(move || -> Result<(), String> {
use windows::Win32::Foundation::{CloseHandle, WAIT_OBJECT_0};
use windows::Win32::System::Threading::{
- OpenProcess, TerminateProcess, WaitForSingleObject,
- PROCESS_TERMINATE, PROCESS_SYNCHRONIZE,
+ OpenProcess, TerminateProcess, WaitForSingleObject, PROCESS_SYNCHRONIZE,
+ PROCESS_TERMINATE,
};
// SAFETY: OpenProcess returns a Result<HANDLE> in windows-rs;
@@ -6677,11 +7595,13 @@ impl Tool for KillAppTool {
// target AND wait for the OS to fully reap it (so the caller's
// follow-up list_apps reflects the change immediately).
let h = unsafe { OpenProcess(PROCESS_TERMINATE | PROCESS_SYNCHRONIZE, false, pid_v) }
- .map_err(|e| format!(
+ .map_err(|e| {
+ format!(
"OpenProcess(pid={pid_v}, PROCESS_TERMINATE|PROCESS_SYNCHRONIZE) failed: {e}. \
The process may not exist, or the daemon lacks the right to terminate it. \
Try listing processes first to confirm the pid is correct."
- ))?;
+ )
+ })?;
let term_result = unsafe { TerminateProcess(h, 1) };
@@ -6704,14 +7624,15 @@ impl Tool for KillAppTool {
(PPL, e.g. csrss / system services)."
)),
}
- }).await;
+ })
+ .await;
match outcome {
Ok(Ok(())) => ToolResult::text(format!("✅ Terminated pid {pid_v}.")),
Ok(Err(e)) => ToolResult::error(format!("kill_app: {e}")),
- Err(join_err) => ToolResult::error(format!(
- "kill_app: blocking-task join error: {join_err}"
- )),
+ Err(join_err) => {
+ ToolResult::error(format!("kill_app: blocking-task join error: {join_err}"))
+ }
}
}
}
@@ -6744,7 +7665,8 @@ impl Tool for DebugWindowInfoTool {
CUIAutomation succeeds — the focused UIA element with the list of \
patterns it supports (ValuePattern, InvokePattern, TextPattern, \
TogglePattern, etc.). Used to design / debug input routing for \
- XAML / UWP / WinUI3 targets; see CUA-543.".into(),
+ XAML / UWP / WinUI3 targets; see CUA-543."
+ .into(),
input_schema: json!({"type":"object","required":["pid"],"properties":{
"pid":{"type":"integer","description":"PID of the process to inspect."}
},"additionalProperties":false}),
@@ -6758,7 +7680,9 @@ impl Tool for DebugWindowInfoTool {
async fn invoke(&self, args: Value) -> ToolResult {
let pid_v: u32 = match args.get("pid").and_then(|v| v.as_u64()) {
Some(p) if p > 0 && p <= u32::MAX as u64 => p as u32,
- _ => return ToolResult::error("debug_window_info: missing or invalid `pid`".to_string()),
+ _ => {
+ return ToolResult::error("debug_window_info: missing or invalid `pid`".to_string())
+ }
};
let outcome = tokio::task::spawn_blocking(move || -> serde_json::Value {
@@ -7000,20 +7924,40 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
let mut r = ToolRegistry::new();
r.register(Box::new(ListAppsTool));
r.register(Box::new(ListWindowsTool));
- r.register(Box::new(GetWindowStateTool { state: state.clone() }));
+ r.register(Box::new(GetWindowStateTool {
+ state: state.clone(),
+ }));
r.register(Box::new(LaunchAppTool));
r.register(Box::new(KillAppTool));
r.register(Box::new(BringToFrontTool));
r.register(Box::new(DebugWindowInfoTool));
- r.register(Box::new(ClickTool { state: state.clone() }));
- r.register(Box::new(DoubleClickTool { state: state.clone() }));
- r.register(Box::new(RightClickTool { state: state.clone() }));
- r.register(Box::new(DragTool { state: state.clone() }));
- r.register(Box::new(TypeTextTool { state: state.clone() }));
- r.register(Box::new(PressKeyTool { state: state.clone() }));
- r.register(Box::new(HotkeyTool { state: state.clone() }));
- r.register(Box::new(SetValueTool { state: state.clone() }));
- r.register(Box::new(ScrollTool { state: state.clone() }));
+ r.register(Box::new(ClickTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(DoubleClickTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(RightClickTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(DragTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(TypeTextTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(PressKeyTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(HotkeyTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetValueTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(ScrollTool {
+ state: state.clone(),
+ }));
// `screenshot` / `ScreenshotCompatTool` removed from the tool surface
// — `get_window_state` (which now always returns a screenshot) is the
// single canonical path for getting a window screenshot. Reasons:
@@ -7035,30 +7979,50 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
r.register(Box::new(GetScreenSizeTool));
r.register(Box::new(GetDesktopStateTool));
r.register(Box::new(GetCursorPositionTool));
- r.register(Box::new(MoveCursorTool { state: state.clone() }));
- r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));
- r.register(Box::new(SetAgentCursorMotionTool { state: state.clone() }));
- r.register(Box::new(GetAgentCursorStateTool { state: state.clone() }));
- r.register(Box::new(SetAgentCursorStyleTool { state: state.clone() }));
+ r.register(Box::new(MoveCursorTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetAgentCursorEnabledTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetAgentCursorMotionTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(GetAgentCursorStateTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetAgentCursorStyleTool {
+ state: state.clone(),
+ }));
r.register(Box::new(CheckPermissionsTool));
// `health_report` — single-call cross-platform driver diagnostics.
// Stable schema_version="1" contract for downstream consumers. Windows skips
// tcc_* and bundle_identity with "not applicable on Windows".
- r.register(Box::new(cua_driver_core::health_report::HealthReportTool::new(
- std::sync::Arc::new(crate::health_report::WindowsHealthProvider),
- )));
- r.register(Box::new(GetConfigTool { state: state.clone() }));
- r.register(Box::new(SetConfigTool { state: state.clone() }));
+ r.register(Box::new(
+ cua_driver_core::health_report::HealthReportTool::new(std::sync::Arc::new(
+ crate::health_report::WindowsHealthProvider,
+ )),
+ ));
+ r.register(Box::new(GetConfigTool {
+ state: state.clone(),
+ }));
+ r.register(Box::new(SetConfigTool {
+ state: state.clone(),
+ }));
r.register(Box::new(GetAccessibilityTreeTool));
- r.register(Box::new(ZoomTool { state: state.clone() }));
+ r.register(Box::new(ZoomTool {
+ state: state.clone(),
+ }));
// `type_text_chars` is intentionally NOT registered — Swift treats it
// as a deprecated alias for `type_text` resolved at invoke time in
// mcp-server's `ToolRegistry::invoke`. Keeping it out of the registry
// means it doesn't show up in `tools/list` either, matching Swift's
// ToolRegistry.swift (`type_text_chars` not in `handlers`).
- let _: &TypeTextCharsTool = &TypeTextCharsTool { _state: state.clone() }; // touch struct so it stays in this crate for now
- // Cross-platform `page` tool definition lives in mcp-server; Windows plugs
- // in its UIA TextPattern + FindAll backend (CDP for execute_javascript).
+ let _: &TypeTextCharsTool = &TypeTextCharsTool {
+ _state: state.clone(),
+ }; // touch struct so it stays in this crate for now
+ // Cross-platform `page` tool definition lives in mcp-server; Windows plugs
+ // in its UIA TextPattern + FindAll backend (CDP for execute_javascript).
r.register(Box::new(cua_driver_core::page::PageTool::new(
std::sync::Arc::new(super::page::WindowsPageBackend::new()),
)));
@@ -7079,25 +8043,43 @@ mod cursor_key_resolution_tests {
// source — it stays the recording/config lifecycle key.
assert_eq!(resolve_cursor_key(&json!({})), NO_CURSOR);
assert_eq!(resolve_cursor_key(&json!({ "pid": 1 })), NO_CURSOR);
- assert_eq!(resolve_cursor_key(&json!({ "_session_id": "mcp-1-2" })), NO_CURSOR);
+ assert_eq!(
+ resolve_cursor_key(&json!({ "_session_id": "mcp-1-2" })),
+ NO_CURSOR
+ );
}
#[test]
fn explicit_session_owns_a_cursor() {
- assert_eq!(resolve_cursor_key(&json!({ "session": "research-run" })), "research-run");
+ assert_eq!(
+ resolve_cursor_key(&json!({ "session": "research-run" })),
+ "research-run"
+ );
}
#[test]
fn cursor_id_is_a_legacy_alias_and_session_wins() {
- assert_eq!(resolve_cursor_key(&json!({ "cursor_id": "user-handle" })), "user-handle");
- assert_eq!(resolve_cursor_key(&json!({ "session": "s1", "cursor_id": "c1" })), "s1");
+ assert_eq!(
+ resolve_cursor_key(&json!({ "cursor_id": "user-handle" })),
+ "user-handle"
+ );
+ assert_eq!(
+ resolve_cursor_key(&json!({ "session": "s1", "cursor_id": "c1" })),
+ "s1"
+ );
}
#[test]
fn empty_strings_fall_through_to_no_cursor() {
// An empty `session` falls through to `cursor_id`; both empty → NO_CURSOR.
- assert_eq!(resolve_cursor_key(&json!({ "session": "", "cursor_id": "c1" })), "c1");
- assert_eq!(resolve_cursor_key(&json!({ "session": "", "cursor_id": "" })), NO_CURSOR);
+ assert_eq!(
+ resolve_cursor_key(&json!({ "session": "", "cursor_id": "c1" })),
+ "c1"
+ );
+ assert_eq!(
+ resolve_cursor_key(&json!({ "session": "", "cursor_id": "" })),
+ NO_CURSOR
+ );
}
#[test]
@@ -7105,8 +8087,10 @@ mod cursor_key_resolution_tests {
// The regression this whole port fixes: two concurrent runs each declare
// their own `session`, so they resolve DISTINCT cursor keys and own
// separate overlay cursors instead of clobbering one shared cursor.
- let a = resolve_cursor_key(&json!({ "pid": 10, "element_index": 1, "session": "calc-2plus1" }));
- let b = resolve_cursor_key(&json!({ "pid": 20, "element_index": 1, "session": "calc-5plus6" }));
+ let a =
+ resolve_cursor_key(&json!({ "pid": 10, "element_index": 1, "session": "calc-2plus1" }));
+ let b =
+ resolve_cursor_key(&json!({ "pid": 20, "element_index": 1, "session": "calc-5plus6" }));
assert_eq!(a, "calc-2plus1");
assert_eq!(b, "calc-5plus6");
assert_ne!(a, b);
@@ -7134,19 +8118,28 @@ mod launch_focus_restore_decision_tests {
// asked for navigation in the default browser. The freshly-launched
// browser instance IS the legitimate foreground; restoring would
// hide the page the user wanted to see.
- let shape = LaunchTargetShape { has_urls: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_urls: true,
+ ..empty()
+ };
assert!(!should_restore_foreground_after_launch(shape));
}
#[test]
fn name_only_restores() {
- let shape = LaunchTargetShape { has_name: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_name: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
#[test]
fn path_only_restores() {
- let shape = LaunchTargetShape { has_path: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_path: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
@@ -7157,19 +8150,28 @@ mod launch_focus_restore_decision_tests {
// app-identifying launch" — the caller (`LaunchAppTool::invoke`)
// gates the polling spawn on `aumid_for_uwp.is_none()` so we don't
// double-restore.
- let shape = LaunchTargetShape { has_aumid: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_aumid: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
#[test]
fn bundle_id_only_restores() {
- let shape = LaunchTargetShape { has_bundle_id: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_bundle_id: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
#[test]
fn launch_path_only_restores() {
- let shape = LaunchTargetShape { has_launch_path: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_launch_path: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
@@ -7178,19 +8180,31 @@ mod launch_focus_restore_decision_tests {
// App-identifying field present alongside urls — the user wants
// the named app to open these URLs in the background, NOT for the
// urls to take over the default browser's foreground. Restore.
- let shape = LaunchTargetShape { has_name: true, has_urls: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_name: true,
+ has_urls: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
#[test]
fn path_with_urls_restores() {
- let shape = LaunchTargetShape { has_path: true, has_urls: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_path: true,
+ has_urls: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
#[test]
fn aumid_with_urls_restores() {
- let shape = LaunchTargetShape { has_aumid: true, has_urls: true, ..empty() };
+ let shape = LaunchTargetShape {
+ has_aumid: true,
+ has_urls: true,
+ ..empty()
+ };
assert!(should_restore_foreground_after_launch(shape));
}
@@ -7209,13 +8223,19 @@ mod chromium_flag_injection_tests {
#[test]
fn detects_bare_browser_names() {
for name in [
- "msedge", "chrome", "brave", "opera", "vivaldi",
- "chromium", "thorium", "iridium", "browser", "arc",
+ "msedge", "chrome", "brave", "opera", "vivaldi", "chromium", "thorium", "iridium",
+ "browser", "arc",
] {
assert!(is_chromium_browser_target(name), "{name} should match");
- assert!(is_chromium_browser_target(&format!("{name}.exe")), "{name}.exe should match");
+ assert!(
+ is_chromium_browser_target(&format!("{name}.exe")),
+ "{name}.exe should match"
+ );
// Case-insensitive.
- assert!(is_chromium_browser_target(&name.to_uppercase()), "uppercase {name} should match");
+ assert!(
+ is_chromium_browser_target(&name.to_uppercase()),
+ "uppercase {name} should match"
+ );
}
}
@@ -7245,7 +8265,10 @@ mod chromium_flag_injection_tests {
fn does_not_match_non_chromium_apps() {
for name in ["firefox", "notepad", "explorer", "code", "soffice"] {
assert!(!is_chromium_browser_target(name), "{name} should NOT match");
- assert!(!is_chromium_browser_target(&format!("{name}.exe")), "{name}.exe should NOT match");
+ assert!(
+ !is_chromium_browser_target(&format!("{name}.exe")),
+ "{name}.exe should NOT match"
+ );
}
// Empty target.
assert!(!is_chromium_browser_target(""));
@@ -7266,7 +8289,10 @@ mod chromium_flag_injection_tests {
let mut args = vec!["--disable-features=Foo,Bar".to_string()];
inject_chromium_anti_throttling_flags(&mut args);
// Should NOT have two --disable-features= entries.
- let dfe: Vec<_> = args.iter().filter(|a| a.starts_with("--disable-features=")).collect();
+ let dfe: Vec<_> = args
+ .iter()
+ .filter(|a| a.starts_with("--disable-features="))
+ .collect();
assert_eq!(dfe.len(), 1);
assert!(dfe[0].contains("CalculateNativeWinOcclusion"));
assert!(dfe[0].contains("Foo"));
@@ -7292,8 +8318,12 @@ mod chromium_flag_injection_tests {
// URL must still be present.
assert!(args.iter().any(|a| a == "file:///C:/test_page.html"));
// All three flags now in args.
- assert!(args.iter().any(|a| a == "--disable-features=CalculateNativeWinOcclusion"));
- assert!(args.iter().any(|a| a == "--disable-backgrounding-occluded-windows"));
+ assert!(args
+ .iter()
+ .any(|a| a == "--disable-features=CalculateNativeWinOcclusion"));
+ assert!(args
+ .iter()
+ .any(|a| a == "--disable-backgrounding-occluded-windows"));
assert!(args.iter().any(|a| a == "--disable-renderer-backgrounding"));
}
}
@@ -7302,14 +8332,15 @@ mod chromium_flag_injection_tests {
mod click_button_schema_tests {
use super::ClickTool;
use cua_driver_core::tool::Tool;
- use std::sync::Arc;
/// Surface 5: schema must keep advertising the three canonical button
/// values. Windows was already shipping `button` (pre-Surface-5) so this
/// test is the freeze test — guards against an inadvertent rename/removal.
#[test]
fn schema_advertises_button_enum() {
- let tool = ClickTool { state: super::ToolState::new() };
+ let tool = ClickTool {
+ state: super::ToolState::new(),
+ };
let d = tool.def();
let props = d.input_schema.get("properties").expect("properties");
let button = props.get("button").expect("button field present");
@@ -7338,7 +8369,10 @@ mod desktop_scope_tests {
#[test]
fn windowless_true_for_xy_under_desktop_scope_click_shape() {
// Click arg shape: {x, y}.
- assert!(is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "desktop"));
+ assert!(is_windowless_desktop_action(
+ &json!({"x": 10, "y": 20}),
+ "desktop"
+ ));
}
#[test]
@@ -7352,7 +8386,10 @@ mod desktop_scope_tests {
#[test]
fn windowless_false_when_pid_present() {
- assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20, "pid": 5}), "desktop"));
+ assert!(!is_windowless_desktop_action(
+ &json!({"x": 10, "y": 20, "pid": 5}),
+ "desktop"
+ ));
}
#[test]
@@ -7365,7 +8402,10 @@ mod desktop_scope_tests {
#[test]
fn windowless_false_under_window_scope() {
- assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "window"));
+ assert!(!is_windowless_desktop_action(
+ &json!({"x": 10, "y": 20}),
+ "window"
+ ));
}
#[test]
@@ -7374,7 +8414,10 @@ mod desktop_scope_tests {
assert!(!is_windowless_desktop_action(&json!({"y": 20}), "desktop"));
assert!(!is_windowless_desktop_action(&json!({}), "desktop"));
// Non-numeric x/y must not qualify.
- assert!(!is_windowless_desktop_action(&json!({"x": "10", "y": "20"}), "desktop"));
+ assert!(!is_windowless_desktop_action(
+ &json!({"x": "10", "y": "20"}),
+ "desktop"
+ ));
}
// ── DriverConfig default ──────────────────────────────────────────────────
@@ -7392,7 +8435,10 @@ mod desktop_scope_tests {
assert!(d.read_only, "get_desktop_state must be read_only");
let props = d.input_schema["properties"].as_object().unwrap();
assert!(!props.contains_key("pid"), "must not accept pid");
- assert!(!props.contains_key("window_id"), "must not accept window_id");
+ assert!(
+ !props.contains_key("window_id"),
+ "must not accept window_id"
+ );
assert!(props.contains_key("session"));
assert!(props.contains_key("screenshot_out_file"));
assert_eq!(d.input_schema["additionalProperties"], json!(false));
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
index c51aeb0e1f..e5e7fb8504 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
@@ -162,14 +162,24 @@ impl ElementCache {
}
fn update_with_kind(&self, pid: u32, hwnd: u64, nodes: &[UiaNode], kind: SnapshotKind) {
- let actionable: Vec<&UiaNode> = nodes.iter().filter(|n| n.element_index.is_some()).collect();
+ let actionable: Vec<&UiaNode> =
+ nodes.iter().filter(|n| n.element_index.is_some()).collect();
let elements: Vec<usize> = actionable.iter().map(|n| n.element_ptr).collect();
- let centers: Vec<(i32, i32)> = actionable.iter().map(|n| (n.center_x, n.center_y)).collect();
+ let centers: Vec<(i32, i32)> = actionable
+ .iter()
+ .map(|n| (n.center_x, n.center_y))
+ .collect();
let rects: Vec<Option<(i32, i32, i32, i32)>> = actionable.iter().map(|n| n.rect).collect();
let msaa_roles: Vec<Option<i32>> = actionable.iter().map(|n| n.msaa_role).collect();
self.core.insert(
CacheKey { pid, hwnd },
- CachedSnapshot { kind, elements, centers, rects, msaa_roles },
+ CachedSnapshot {
+ kind,
+ elements,
+ centers,
+ rects,
+ msaa_roles,
+ },
);
}
@@ -221,18 +231,50 @@ impl ElementCache {
.flatten()
}
- pub fn get_element_center(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<(i32, i32)> {
+ pub fn get_element_center(
+ &self,
+ pid: u32,
+ hwnd: u64,
+ element_index: usize,
+ ) -> Option<(i32, i32)> {
self.core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| s.centers.get(element_index).copied())
+ .with_snapshot(&CacheKey { pid, hwnd }, |s| {
+ s.centers.get(element_index).copied()
+ })
.flatten()
}
+ /// Focus a cached UIA element without activating its top-level window.
+ /// The caller owns the no-activation guard for background delivery.
+ pub fn focus_element(&self, pid: u32, hwnd: u64, element_index: usize) -> anyhow::Result<()> {
+ let retained = self
+ .get_element_retained(pid, hwnd, element_index)
+ .ok_or_else(|| anyhow::anyhow!("element [{element_index}] is not in the UIA cache"))?;
+ if !retained.is_uia() {
+ anyhow::bail!("element [{element_index}] is an MSAA element, not a UIA element");
+ }
+ let ptr = retained.as_ptr();
+ let element: IUIAutomationElement =
+ unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
+ let result = unsafe { element.SetFocus() };
+ // `retained` owns the AddRef; `from_raw` borrows that same pointer.
+ std::mem::forget(element);
+ result.map_err(|e| anyhow::anyhow!("UIA SetFocus failed: {e}"))
+ }
+
/// Cached screen rect for the element. Used by the click tool to
/// compute the right-edge dispatch point for `action:"expand"` on
/// MSAA BUTTONDROPDOWN.
- pub fn get_element_rect(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<(i32, i32, i32, i32)> {
+ pub fn get_element_rect(
+ &self,
+ pid: u32,
+ hwnd: u64,
+ element_index: usize,
+ ) -> Option<(i32, i32, i32, i32)> {
self.core
- .with_snapshot(&CacheKey { pid, hwnd }, |s| s.rects.get(element_index).copied().flatten())
+ .with_snapshot(&CacheKey { pid, hwnd }, |s| {
+ s.rects.get(element_index).copied().flatten()
+ })
.flatten()
}
@@ -269,4 +311,3 @@ impl Default for ElementCache {
#[cfg(test)]
#[path = "cache_uaf_repro.rs"]
mod cache_uaf_repro;
-
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
index dc6f68259a..914bb1112c 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
@@ -10,17 +10,17 @@
//! take >4s when reading each property individually).
use windows::core::{Interface, BSTR};
-use windows::Win32::System::Com::{CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED};
+use windows::Win32::System::Com::{
+ CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED,
+};
use windows::Win32::UI::Accessibility::{
CUIAutomation, IUIAutomation, IUIAutomationCacheRequest, IUIAutomationElement,
- UIA_AutomationIdPropertyId, UIA_BoundingRectanglePropertyId,
- UIA_ControlTypePropertyId, UIA_HelpTextPropertyId,
- UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_NamePropertyId,
- UIA_ProcessIdPropertyId, UIA_ValueValuePropertyId,
- UIA_InvokePatternId, UIA_SelectionItemPatternId,
- UIA_TogglePatternId, UIA_ExpandCollapsePatternId, UIA_TextPatternId,
- UIA_ValuePatternId, UIA_RangeValuePatternId, UIA_ScrollPatternId,
- TreeScope_Children, TreeScope_Subtree,
+ TreeScope_Children, TreeScope_Subtree, UIA_AutomationIdPropertyId,
+ UIA_BoundingRectanglePropertyId, UIA_ControlTypePropertyId, UIA_ExpandCollapsePatternId,
+ UIA_HelpTextPropertyId, UIA_InvokePatternId, UIA_IsEnabledPropertyId,
+ UIA_IsOffscreenPropertyId, UIA_NamePropertyId, UIA_ProcessIdPropertyId,
+ UIA_RangeValuePatternId, UIA_ScrollPatternId, UIA_SelectionItemPatternId, UIA_TextPatternId,
+ UIA_TogglePatternId, UIA_ValuePatternId, UIA_ValueValuePropertyId,
};
pub mod cache;
@@ -113,21 +113,26 @@ unsafe fn walk_tree_unsafe(
) -> UiaTreeResult {
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
- let automation: IUIAutomation = match CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) {
- Ok(a) => a,
- Err(e) => return UiaTreeResult {
- tree_markdown: format!("UIA init failed: {e}"),
- nodes: Vec::new(),
- },
- };
+ let automation: IUIAutomation =
+ match CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) {
+ Ok(a) => a,
+ Err(e) => {
+ return UiaTreeResult {
+ tree_markdown: format!("UIA init failed: {e}"),
+ nodes: Vec::new(),
+ }
+ }
+ };
// Build a cache request that fetches everything we need in ONE bulk RPC.
let cache_req: IUIAutomationCacheRequest = match automation.CreateCacheRequest() {
Ok(r) => r,
- Err(e) => return UiaTreeResult {
- tree_markdown: format!("CreateCacheRequest failed: {e}"),
- nodes: Vec::new(),
- },
+ Err(e) => {
+ return UiaTreeResult {
+ tree_markdown: format!("CreateCacheRequest failed: {e}"),
+ nodes: Vec::new(),
+ }
+ }
};
// Properties to pre-fetch.
@@ -204,10 +209,12 @@ unsafe fn walk_tree_unsafe(
// the difference matters for performance-sensitive providers.
let uncached = match automation.ElementFromHandle(hwnd_win) {
Ok(e) => e,
- Err(e) => return UiaTreeResult {
- tree_markdown: format!("ElementFromHandle failed: {e}"),
- nodes: Vec::new(),
- },
+ Err(e) => {
+ return UiaTreeResult {
+ tree_markdown: format!("ElementFromHandle failed: {e}"),
+ nodes: Vec::new(),
+ }
+ }
};
// A single transient provider error (commonly E_FAIL / 0x80004005 from a
// control rebuilding its automation subtree mid-walk) must not take down
@@ -353,7 +360,10 @@ unsafe fn walk_tree_unsafe(
returns alongside this tree; \
(b) `press_key` with `delivery_mode:\"foreground\"` (Esc / Enter / Y / N).)\n"
);
- return UiaTreeResult { tree_markdown: stub, nodes: Vec::new() };
+ return UiaTreeResult {
+ tree_markdown: stub,
+ nodes: Vec::new(),
+ };
}
}
@@ -364,7 +374,10 @@ unsafe fn walk_tree_unsafe(
raw_md
};
- UiaTreeResult { tree_markdown, nodes }
+ UiaTreeResult {
+ tree_markdown,
+ nodes,
+ }
}
/// Resolve the owning process id of `hwnd` via `GetWindowThreadProcessId`.
@@ -373,7 +386,11 @@ unsafe fn pid_from_hwnd(hwnd: windows::Win32::Foundation::HWND) -> Option<u32> {
use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId;
let mut pid: u32 = 0;
let tid = GetWindowThreadProcessId(hwnd, Some(&mut pid));
- if tid == 0 || pid == 0 { None } else { Some(pid) }
+ if tid == 0 || pid == 0 {
+ None
+ } else {
+ Some(pid)
+ }
}
/// Root-walk UIA fallback for apps whose top-level window is a
@@ -399,19 +416,31 @@ unsafe fn walk_root_by_pid(
) {
let root = match automation.GetRootElement() {
Ok(r) => r,
- Err(e) => { tracing::debug!(target: "uia", "GetRootElement failed: {e}"); return; }
+ Err(e) => {
+ tracing::debug!(target: "uia", "GetRootElement failed: {e}");
+ return;
+ }
};
let true_cond = match automation.CreateTrueCondition() {
Ok(c) => c,
- Err(e) => { tracing::debug!(target: "uia", "CreateTrueCondition failed: {e}"); return; }
+ Err(e) => {
+ tracing::debug!(target: "uia", "CreateTrueCondition failed: {e}");
+ return;
+ }
};
let kids = match root.FindAll(TreeScope_Children, &true_cond) {
Ok(a) => a,
- Err(e) => { tracing::debug!(target: "uia", "root.FindAll(Children) failed: {e}"); return; }
+ Err(e) => {
+ tracing::debug!(target: "uia", "root.FindAll(Children) failed: {e}");
+ return;
+ }
};
let count = kids.Length().unwrap_or(0);
for i in 0..count {
- let elem = match kids.GetElement(i) { Ok(e) => e, Err(_) => continue };
+ let elem = match kids.GetElement(i) {
+ Ok(e) => e,
+ Err(_) => continue,
+ };
// Read ProcessId without a cache — root.FindAll didn't use one.
// VARIANT for VT_I4 (UIA's ProcessId type) puts the int at
// Anonymous.Anonymous.Anonymous.lVal; mirrors the read_cached_bool
@@ -419,12 +448,18 @@ unsafe fn walk_root_by_pid(
let pid: u32 = match elem.GetCurrentPropertyValue(UIA_ProcessIdPropertyId) {
Ok(v) => {
let raw = v.as_raw();
- if raw.Anonymous.Anonymous.vt != 3 /* VT_I4 */ { continue; }
+ if raw.Anonymous.Anonymous.vt != 3
+ /* VT_I4 */
+ {
+ continue;
+ }
raw.Anonymous.Anonymous.Anonymous.lVal as u32
}
Err(_) => continue,
};
- if pid != target_pid { continue; }
+ if pid != target_pid {
+ continue;
+ }
// Match — pull a cached subtree from this element using the same
// cache_req shape as the primary path so walk_cached sees the same
// properties + patterns.
@@ -494,12 +529,16 @@ unsafe fn walk_cached_bounded(
let automation_id = read_cached_bstr(element, UIA_AutomationIdPropertyId);
let help_text = read_cached_bstr(element, UIA_HelpTextPropertyId);
let is_enabled = read_cached_bool(element, UIA_IsEnabledPropertyId).unwrap_or(true);
- let is_offscreen = read_cached_bool(element, UIA_IsOffscreenPropertyId).unwrap_or(false);
-
- let actions = detect_cached_actions(element, is_enabled);
- let is_actionable = !actions.is_empty() && is_enabled && !is_offscreen;
- let has_content = name.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false)
- || value.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false);
+ let actions = detect_cached_actions(element, &control_type, is_enabled);
+ let is_actionable = !actions.is_empty() && is_enabled;
+ let has_content = name
+ .as_deref()
+ .map(|s| !s.trim().is_empty())
+ .unwrap_or(false)
+ || value
+ .as_deref()
+ .map(|s| !s.trim().is_empty())
+ .unwrap_or(false);
let mut emitted_parent: Option<usize> = parent_index;
if is_actionable || has_content {
@@ -574,7 +613,9 @@ unsafe fn walk_cached_bounded(
fn read_cached_control_type(element: &IUIAutomationElement) -> String {
unsafe {
- element.CachedControlType().ok()
+ element
+ .CachedControlType()
+ .ok()
.map(|ct| control_type_name(ct.0))
.unwrap_or_else(|| "Unknown".into())
}
@@ -584,7 +625,11 @@ fn read_cached_bstr_name(element: &IUIAutomationElement) -> Option<String> {
unsafe {
let bstr = element.CachedName().ok()?;
let s = bstr.to_string();
- if s.trim().is_empty() { None } else { Some(s) }
+ if s.trim().is_empty() {
+ None
+ } else {
+ Some(s)
+ }
}
}
@@ -592,21 +637,31 @@ fn read_cached_bstr_value(element: &IUIAutomationElement) -> Option<String> {
read_cached_bstr(element, UIA_ValueValuePropertyId)
}
-fn read_cached_bstr(element: &IUIAutomationElement, property_id: windows::Win32::UI::Accessibility::UIA_PROPERTY_ID) -> Option<String> {
+fn read_cached_bstr(
+ element: &IUIAutomationElement,
+ property_id: windows::Win32::UI::Accessibility::UIA_PROPERTY_ID,
+) -> Option<String> {
unsafe {
let variant = element.GetCachedPropertyValue(property_id).ok()?;
if variant.as_raw().Anonymous.Anonymous.vt == 8 {
let bstr = BSTR::from_raw(variant.as_raw().Anonymous.Anonymous.Anonymous.bstrVal);
let s = bstr.to_string();
std::mem::forget(bstr);
- if s.trim().is_empty() { None } else { Some(s) }
+ if s.trim().is_empty() {
+ None
+ } else {
+ Some(s)
+ }
} else {
None
}
}
}
-fn read_cached_bool(element: &IUIAutomationElement, property_id: windows::Win32::UI::Accessibility::UIA_PROPERTY_ID) -> Option<bool> {
+fn read_cached_bool(
+ element: &IUIAutomationElement,
+ property_id: windows::Win32::UI::Accessibility::UIA_PROPERTY_ID,
+) -> Option<bool> {
unsafe {
let variant = element.GetCachedPropertyValue(property_id).ok()?;
if variant.as_raw().Anonymous.Anonymous.vt == 11 {
@@ -620,7 +675,9 @@ fn read_cached_bool(element: &IUIAutomationElement, property_id: windows::Win32:
/// Read bounding rect as (center_x, center_y, Some((l,t,r,b))). Returns
/// rect=None when the element has no meaningful BoundingRectangle (offscreen
/// containers, structure-only elements).
-fn read_cached_bounding_rect_full(element: &IUIAutomationElement) -> (i32, i32, Option<(i32, i32, i32, i32)>) {
+fn read_cached_bounding_rect_full(
+ element: &IUIAutomationElement,
+) -> (i32, i32, Option<(i32, i32, i32, i32)>) {
unsafe {
match element.CachedBoundingRectangle() {
Ok(r) if r.right > r.left && r.bottom > r.top => (
@@ -633,8 +690,14 @@ fn read_cached_bounding_rect_full(element: &IUIAutomationElement) -> (i32, i32,
}
}
-fn detect_cached_actions(element: &IUIAutomationElement, is_enabled: bool) -> Vec<String> {
- if !is_enabled { return vec![]; }
+fn detect_cached_actions(
+ element: &IUIAutomationElement,
+ control_type: &str,
+ is_enabled: bool,
+) -> Vec<String> {
+ if !is_enabled {
+ return vec![];
+ }
let mut actions = Vec::new();
unsafe {
if element.GetCachedPattern(UIA_InvokePatternId).is_ok() {
@@ -646,7 +709,10 @@ fn detect_cached_actions(element: &IUIAutomationElement, is_enabled: bool) -> Ve
if element.GetCachedPattern(UIA_SelectionItemPatternId).is_ok() {
actions.push("select".into());
}
- if element.GetCachedPattern(UIA_ExpandCollapsePatternId).is_ok() {
+ if element
+ .GetCachedPattern(UIA_ExpandCollapsePatternId)
+ .is_ok()
+ {
actions.push("expand".into());
}
if element.GetCachedPattern(UIA_ValuePatternId).is_ok() {
@@ -666,6 +732,13 @@ fn detect_cached_actions(element: &IUIAutomationElement, is_enabled: bool) -> Ve
actions.push("scroll".into());
}
}
+ if actions.is_empty() && control_type == "MenuItem" {
+ // Some WPF MenuItem peers show up in ControlView with a usable name
+ // and bounding rectangle, but without cached Invoke/ExpandCollapse
+ // patterns. Index them anyway so click can retry live patterns and then
+ // fall back to the coordinate injector if UIA still reports no pattern.
+ actions.push("invoke".into());
+ }
actions
}
@@ -713,7 +786,8 @@ fn control_type_name(id: i32) -> String {
50039 => "SemanticZoom",
50040 => "AppBar",
_ => "Unknown",
- }.into()
+ }
+ .into()
}
pub(crate) fn format_node_line(node: &UiaNode) -> String {
@@ -724,9 +798,15 @@ pub(crate) fn format_node_line(node: &UiaNode) -> String {
s.push_str(&format!(" \"{}\"", n));
}
let mut attrs = Vec::new();
- if let Some(v) = &node.value { attrs.push(format!("value=\"{}\"", v)); }
- if let Some(id) = &node.automation_id { attrs.push(format!("id={}", id)); }
- if let Some(h) = &node.help_text { attrs.push(format!("help=\"{}\"", h)); }
+ if let Some(v) = &node.value {
+ attrs.push(format!("value=\"{}\"", v));
+ }
+ if let Some(id) = &node.automation_id {
+ attrs.push(format!("id={}", id));
+ }
+ if let Some(h) = &node.help_text {
+ attrs.push(format!("help=\"{}\"", h));
+ }
if !node.actions.is_empty() {
attrs.push(format!("actions=[{}]", node.actions.join(",")));
}
@@ -735,8 +815,12 @@ pub(crate) fn format_node_line(node: &UiaNode) -> String {
}
} else {
s.push_str(&format!("- {}", node.control_type));
- if let Some(n) = &node.name { s.push_str(&format!(" \"{}\"", n)); }
- if let Some(v) = &node.value { s.push_str(&format!(" = \"{}\"", v)); }
+ if let Some(n) = &node.name {
+ s.push_str(&format!(" \"{}\"", n));
+ }
+ if let Some(v) = &node.value {
+ s.push_str(&format!(" = \"{}\"", v));
+ }
}
s
}
@@ -744,7 +828,9 @@ pub(crate) fn format_node_line(node: &UiaNode) -> String {
fn render_lines(lines: &[(usize, String)]) -> String {
let mut out = String::new();
for (depth, line) in lines {
- for _ in 0..*depth { out.push_str(" "); }
+ for _ in 0..*depth {
+ out.push_str(" ");
+ }
out.push_str(line);
out.push('\n');
}
@@ -764,13 +850,19 @@ fn filter_tree(markdown: &str, query: &str) -> String {
ancestors.push("");
last_emitted.push(None);
}
- for d in (depth + 1)..ancestors.len() { last_emitted[d] = None; }
+ for d in (depth + 1)..ancestors.len() {
+ last_emitted[d] = None;
+ }
ancestors[depth] = line;
if line.to_lowercase().contains(&needle) {
for d in 0..depth {
- if ancestors[d].is_empty() { continue; }
- if last_emitted[d] == Some(ancestors[d]) { continue; }
+ if ancestors[d].is_empty() {
+ continue;
+ }
+ if last_emitted[d] == Some(ancestors[d]) {
+ continue;
+ }
last_emitted[d] = Some(ancestors[d]);
output.push(ancestors[d]);
}
@@ -779,7 +871,9 @@ fn filter_tree(markdown: &str, query: &str) -> String {
}
}
- if output.is_empty() { return String::new(); }
+ if output.is_empty() {
+ return String::new();
+ }
let mut r = output.join("\n");
r.push('\n');
r
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/scroll.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/scroll.rs
index 50ae3dd17f..a084bd590b 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/scroll.rs
@@ -101,6 +101,48 @@ pub unsafe fn scroll_into_view_and_recenter(
result
}
+/// Scroll a cached UIA scroll container in the requested direction.
+///
+/// WebView2/Tauri content can expose a real `ScrollPattern` even though its
+/// top-level host HWND ignores `WM_VSCROLL`. Keeping this on the UIA channel
+/// makes indexed background scrolls reach that container without activating
+/// the host window.
+pub unsafe fn scroll_element(
+ element_ptr: usize,
+ direction: &str,
+ amount: u32,
+) -> anyhow::Result<()> {
+ if element_ptr == 0 {
+ anyhow::bail!("cached UIA scroll element is null");
+ }
+ let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
+ let elem: IUIAutomationElement = IUIAutomationElement::from_raw(element_ptr as *mut _);
+ let pattern = elem
+ .GetCurrentPattern(UIA_ScrollPatternId)
+ .map_err(|e| anyhow::anyhow!("UIA ScrollPattern unavailable: {e}"))?;
+ let scroll = pattern
+ .cast::<IUIAutomationScrollPattern>()
+ .map_err(|e| anyhow::anyhow!("UIA ScrollPattern cast failed: {e}"))?;
+ let vertical = match direction {
+ "up" => ScrollAmount_LargeDecrement,
+ "down" => ScrollAmount_LargeIncrement,
+ "left" => ScrollAmount_LargeDecrement,
+ "right" => ScrollAmount_LargeIncrement,
+ other => anyhow::bail!("unknown scroll direction {other:?}"),
+ };
+ let horizontal = matches!(direction, "left" | "right");
+ for _ in 0..amount.max(1) {
+ let result = if horizontal {
+ scroll.Scroll(vertical, ScrollAmount_NoAmount)
+ } else {
+ scroll.Scroll(ScrollAmount_NoAmount, vertical)
+ };
+ result.map_err(|e| anyhow::anyhow!("UIA scroll failed: {e}"))?;
+ }
+ std::mem::forget(elem);
+ Ok(())
+}
+
/// Strategy 1: `ScrollItemPattern::ScrollIntoView` on the element itself.
unsafe fn scroll_item_into_view(host_hwnd: u64, elem: &IUIAutomationElement) -> Option<(i32, i32)> {
let pattern = elem.GetCurrentPattern(UIA_ScrollItemPatternId).ok()?;
@@ -132,7 +174,10 @@ unsafe fn scroll_ancestor_into_view(
for _ in 0..16 {
if let Ok(p) = cur.GetCurrentPattern(UIA_ScrollPatternId) {
if let Ok(sp) = p.cast::<IUIAutomationScrollPattern>() {
- let scrollable = sp.CurrentVerticallyScrollable().map(|b| b.as_bool()).unwrap_or(false);
+ let scrollable = sp
+ .CurrentVerticallyScrollable()
+ .map(|b| b.as_bool())
+ .unwrap_or(false);
if scrollable {
container = Some(cur.clone());
scroll_pat = Some(sp);
diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js
index 89ad75543c..cdc48cc36c 100644
--- a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js
+++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js
@@ -28,6 +28,7 @@ function createWindow() {
width: 940,
height: 780,
title: fixedTitle,
+ show: false,
autoHideMenuBar: true,
webPreferences: {
nodeIntegration: false,
@@ -41,6 +42,21 @@ function createWindow() {
// 'cua-driver Web Harness' (the page's title).
mainWindow.on('page-title-updated', e => e.preventDefault());
mainWindow.setTitle(fixedTitle);
+ mainWindow.webContents.setWindowOpenHandler(() => ({
+ action: 'allow',
+ overrideBrowserWindowOptions: {
+ width: 320,
+ height: 220,
+ show: false,
+ autoHideMenuBar: true,
+ title: 'CuaTestHarness Child Window',
+ },
+ }));
+ mainWindow.webContents.on('did-create-window', child => {
+ child.once('ready-to-show', () => {
+ child.showInactive();
+ });
+ });
mainWindow
.loadFile(path.join(__dirname, 'web', 'index.html'))
@@ -50,6 +66,7 @@ function createWindow() {
// our fixedTitle and break the harness-window-discovery test.
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.setTitle(fixedTitle);
+ mainWindow.showInactive();
}
})
.catch(err => {
diff --git a/libs/cua-driver/tests/fixtures/shared/web/index.html b/libs/cua-driver/tests/fixtures/shared/web/index.html
index d7e00af120..b1f93c6944 100644
--- a/libs/cua-driver/tests/fixtures/shared/web/index.html
+++ b/libs/cua-driver/tests/fixtures/shared/web/index.html
@@ -161,6 +161,14 @@
<h3 id="section-target" data-cua-id="section-target">SECTION_TARGET_MARKER_v1</h3>
</fieldset>
+ <fieldset>
+ <legend>child_window</legend>
+ <div class="row">
+ <button id="btn-open-child-window" data-cua-id="btn-open-child-window">Open child window</button>
+ <span class="mirror" id="lbl-child-window-state" data-cua-id="lbl-child-window-state">child_windows=0</span>
+ </div>
+ </fieldset>
+
<script>
// ----- state -----
let counter = 0;
@@ -292,6 +300,17 @@
$('lbl-nav-state').textContent = `hash=${location.hash}`;
}, 30);
});
+
+ let childWindowCount = 0;
+ $('btn-open-child-window').addEventListener('click', () => {
+ childWindowCount++;
+ $('lbl-child-window-state').textContent = `child_windows=${childWindowCount}`;
+ const child = window.open('', `cua-harness-child-${childWindowCount}`, 'width=320,height=220');
+ if (child) {
+ child.document.write(`<!doctype html><title>Cua child ${childWindowCount}</title><p>CHILD_WINDOW_MARKER_v1</p>`);
+ child.document.close();
+ }
+ });
</script>
</body>
</html>