Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): refuse minimized Windows element actions

webdev8a9b7baa32bb
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
index 8671836fbe..362f76118a 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs
@@ -787,8 +787,16 @@ fn write_turn(
}
};
let click_family = matches!(tool_name.as_str(), "click" | "double_click" | "right_click");
+ let action_refused = action_record
+ .is_some_and(|record| record.effect == crate::action_record::ActionEffect::Refused);
let refused_before_target_resolution = click_family && result_is_error && click_point.is_none();
- let click_expected = click_family && !refused_before_target_resolution;
+ // A target may resolve successfully and still be refused before input
+ // dispatch (for example, a minimized Windows element). Retaining the
+ // resolved point in action.json is useful diagnostic context, but a
+ // crosshair would falsely imply that a click was delivered.
+ let refused_before_dispatch = click_family && result_is_error && action_refused;
+ let click_expected =
+ click_family && !refused_before_target_resolution && !refused_before_dispatch;
let mut payload = serde_json::json!({
"tool": tool_name,
@@ -820,14 +828,16 @@ fn write_turn(
// the pre-action image. This also keeps modal-dismiss evidence available
// after the modal HWND has closed.
let mut click_captured = false;
- if let (Some((cx, cy)), Some(screenshot), Some(marker)) = (
- click_point,
- before.screenshot.as_deref(),
- CLICK_MARKER_FN.get(),
- ) {
- if let Some(click_png) = marker(screenshot, cx, cy) {
- std::fs::write(turn_dir.join("click.png"), click_png)?;
- click_captured = true;
+ if click_expected {
+ if let (Some((cx, cy)), Some(screenshot), Some(marker)) = (
+ click_point,
+ before.screenshot.as_deref(),
+ CLICK_MARKER_FN.get(),
+ ) {
+ if let Some(click_png) = marker(screenshot, cx, cy) {
+ std::fs::write(turn_dir.join("click.png"), click_png)?;
+ click_captured = true;
+ }
}
}
write_evidence_manifest(
@@ -839,6 +849,8 @@ fn write_turn(
click_captured,
if refused_before_target_resolution {
"action_refused_before_target_resolution"
+ } else if refused_before_dispatch {
+ "action_refused_before_dispatch"
} else {
"not_a_click_action"
},
@@ -1064,6 +1076,49 @@ mod tests {
"action_refused_before_target_resolution"
);
+ let pending = session
+ .begin_turn(
+ "click",
+ &serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}),
+ now_ms(),
+ )
+ .expect("resolved refusal should reserve an evidence turn");
+ let refusal_record = crate::action_record::ActionExecutionRecord::builder(
+ crate::action_record::ActionEffect::Refused,
+ crate::action_record::ActionTransport::WindowsTargetedInjection,
+ crate::action_record::RequestedDelivery::Background,
+ )
+ .build()
+ .expect("valid refusal record");
+ session.finish_turn_with_outcome(
+ pending,
+ "refused before dispatch",
+ Some(&refusal_record),
+ true,
+ );
+ let resolved_refusal_turn = output_dir.join("turn-00004");
+ let resolved_refusal_action: Value = serde_json::from_slice(
+ &std::fs::read(resolved_refusal_turn.join("action.json"))
+ .expect("read resolved refusal action"),
+ )
+ .expect("parse resolved refusal action");
+ assert_eq!(resolved_refusal_action["click_point"]["x"], 3.0);
+ assert_eq!(resolved_refusal_action["action_truth"]["effect"], "refused");
+ assert!(!resolved_refusal_turn.join("click.png").exists());
+ let resolved_refusal_manifest: Value = serde_json::from_slice(
+ &std::fs::read(resolved_refusal_turn.join("evidence.json"))
+ .expect("read resolved refusal evidence"),
+ )
+ .expect("parse resolved refusal evidence");
+ assert_eq!(
+ resolved_refusal_manifest["click"]["status"],
+ "not_applicable"
+ );
+ assert_eq!(
+ resolved_refusal_manifest["click"]["classification"],
+ "action_refused_before_dispatch"
+ );
+
let files = [
"action.json",
"app_state.json",
@@ -1081,11 +1136,13 @@ mod tests {
}
std::fs::remove_dir(directory).expect("remove turn fixture directory");
}
- for file in files.into_iter().filter(|file| *file != "click.png") {
- std::fs::remove_file(refused_turn.join(file))
- .expect("remove refused turn fixture file");
+ for directory in [&refused_turn, &resolved_refusal_turn] {
+ for file in files.iter().copied().filter(|file| *file != "click.png") {
+ std::fs::remove_file(directory.join(file))
+ .expect("remove refused turn fixture file");
+ }
+ std::fs::remove_dir(directory).expect("remove refused turn fixture directory");
}
- std::fs::remove_dir(refused_turn).expect("remove refused turn fixture directory");
std::fs::remove_dir(&output_dir).expect("remove recording fixture directory");
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
index 23a42cab59..848a7607da 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
@@ -315,6 +315,7 @@ pub enum RefusalCode {
BrowserReconnectExhausted,
BrowserInputIncomplete,
BrowserActionUnavailable,
+ WindowMinimized,
}
impl RefusalCode {
@@ -336,6 +337,7 @@ impl RefusalCode {
"browser_reconnect_exhausted" => Some(Self::BrowserReconnectExhausted),
"browser_input_incomplete" => Some(Self::BrowserInputIncomplete),
"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),
+ "window_minimized" => Some(Self::WindowMinimized),
_ => None,
}
}
@@ -1493,6 +1495,12 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
.as_str()
.is_some_and(|summary| summary.starts_with("✅ bring_to_front:"))
});
+ let refused_minimized_action = action.as_ref().is_some_and(|value| {
+ value["result_error"].as_bool() == Some(true)
+ && value["result_summary"]
+ .as_str()
+ .is_some_and(|summary| summary.starts_with("refused (window_minimized):"))
+ });
let restored_state_captured = manifest
.as_ref()
.is_some_and(|value| value["after"]["state"]["status"].as_str() == Some("captured"));
@@ -1506,6 +1514,8 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
&& successful_restore
&& restored_state_captured
&& classification(phase, "screenshot") == Some("capture_failed"))))
+ || (refused_minimized_action
+ && classification(phase, "screenshot") == Some("target_minimized"))
};
for (phase, kind) in [("before", "screenshot"), ("after", "screenshot")] {
@@ -1577,15 +1587,23 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
let refused_before_target_resolution = action.as_ref().is_some_and(|value| {
value["result_error"].as_bool() == Some(true) && value.get("click_point").is_none()
});
- if refused_before_target_resolution {
+ let refused_before_dispatch = action.as_ref().is_some_and(|value| {
+ value["result_error"].as_bool() == Some(true)
+ && value["action_truth"]["effect"].as_str() == Some("refused")
+ });
+ if refused_before_target_resolution || refused_before_dispatch {
let click = manifest.as_ref().map(|value| &value["click"]);
+ let expected_classification = if refused_before_target_resolution {
+ "action_refused_before_target_resolution"
+ } else {
+ "action_refused_before_dispatch"
+ };
if !click.is_some_and(|value| {
value["status"].as_str() == Some("not_applicable")
- && value["classification"].as_str()
- == Some("action_refused_before_target_resolution")
+ && value["classification"].as_str() == Some(expected_classification)
}) {
errors.push(format!(
- "invalid refused-click evidence for {cell_id}/{turn_name}: expected not_applicable/action_refused_before_target_resolution"
+ "invalid refused-click evidence for {cell_id}/{turn_name}: expected not_applicable/{expected_classification}"
));
}
return;
@@ -2222,6 +2240,42 @@ mod tests {
.expect("a minimized target cannot provide a pre-restore screenshot");
}
+ #[test]
+ fn validator_accepts_missing_images_for_typed_minimized_refusal() {
+ let (root, case, result, turn) = complete_turn_fixture();
+ std::fs::write(
+ turn.join("action.json"),
+ br#"{
+ "tool":"click",
+ "arguments":{"pid":1,"window_id":2,"element_index":3},
+ "result_summary":"refused (window_minimized): restore before retrying",
+ "result_error":true,
+ "click_point":{"x":10,"y":20},
+ "action_truth":{"effect":"refused"}
+ }"#,
+ )
+ .expect("write minimized refusal action");
+ std::fs::write(
+ turn.join("evidence.json"),
+ br#"{
+ "schema":"cua-turn-evidence/v1",
+ "before":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}},
+ "after":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}},
+ "click":{"status":"not_applicable","classification":"action_refused_before_dispatch"}
+ }"#,
+ )
+ .expect("write minimized refusal evidence manifest");
+ for file in ["before.png", "after.png", "screenshot.png", "click.png"] {
+ let path = turn.join(file);
+ if path.exists() {
+ std::fs::remove_file(path).expect("remove unavailable image");
+ }
+ }
+
+ validate_catalog(&[case], &[result], Some(root.path()), true)
+ .expect("a typed minimized refusal cannot provide target images");
+ }
+
#[test]
fn validator_accepts_restored_state_when_host_capture_remains_unavailable() {
let (root, case, result, turn) = complete_turn_fixture();
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
index c37857fc00..33e1847824 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
@@ -48,6 +48,8 @@ use cua_driver_testkit::e2e::{
use cua_driver_testkit::observer::TargetWindow;
use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel};
use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolResponse};
+use windows::Win32::Foundation::HWND;
+use windows::Win32::UI::WindowsAndMessaging::{IsIconic, ShowWindow, SW_MINIMIZE};
// ── harness launcher ─────────────────────────────────────────────────────────
@@ -549,6 +551,95 @@ fn harness_wpf_counter_invoke() {
);
}
+#[test]
+#[ignore]
+fn harness_wpf_minimized_element_click_refuses_without_side_effects() {
+ let case = CaseSpec::delivered(
+ "windows-wpf-minimized-element-click-refusal",
+ "wpf",
+ "wpf",
+ "left_click",
+ Targeting::Ax,
+ Delivery::Background,
+ Scope::Window,
+ DriverRoute::Composite,
+ vec![
+ OracleKind::FixtureState,
+ OracleKind::Focus,
+ OracleKind::ZOrder,
+ OracleKind::Cursor,
+ OracleKind::NoLeakedInput,
+ OracleKind::Protocol,
+ ],
+ )
+ .expecting_refusal(vec![RefusalCode::WindowMinimized]);
+
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_named("windows-wpf-minimized-element-click-refusal")
+ .expect("required source-built driver did not start");
+ *evidence = recording_evidence(driver.recording_dir());
+ let state_dir = tempfile::tempdir().expect("create WPF fixture state directory");
+ let state_path = state_dir.path().join("state.json");
+ let pid = launch_harness_with_state_file(&mut driver, Some(&state_path))
+ .expect("required WPF harness did not launch");
+ let (wid, _) = driver
+ .find_window(pid as i64, "CuaTestHarness WPF")
+ .expect("main window");
+ wait_for_fixture_file_text(&state_path, "lbl-counter", "counter=0");
+ let ready = snapshot(&mut driver, pid, wid);
+ let idx = ax::element_index_by_id(ready.text(), "btn-increment")
+ .expect("btn-increment not in pre-minimize snapshot");
+
+ let sentinel = ForegroundSentinel::launch(&mut driver);
+ let hwnd = HWND(wid as *mut _);
+ let _ = unsafe { ShowWindow(hwnd, SW_MINIMIZE) };
+ let deadline = Instant::now() + Duration::from_secs(2);
+ while !unsafe { IsIconic(hwnd) }.as_bool() {
+ assert!(Instant::now() < deadline, "WPF target did not minimize");
+ std::thread::sleep(Duration::from_millis(25));
+ }
+ driver.start_behavior_recording();
+
+ let (response, mut passed) = sentinel
+ .observe_desktop(|| {
+ driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": wid,
+ "element_index": idx,
+ "snapshot_id": ready.snapshot_id(),
+ "delivery_mode": "background"
+ }),
+ )
+ })
+ .unwrap_or_else(|error| panic!("minimized refusal leaked desktop state: {error}"));
+ assert!(
+ response.is_error(),
+ "minimized click reported success: {}",
+ response.text()
+ );
+ assert_eq!(response.structured()["status"], "refused");
+ assert_eq!(response.structured()["refusal"]["code"], "window_minimized");
+ assert!(
+ unsafe { IsIconic(hwnd) }.as_bool(),
+ "refused click restored or otherwise changed the minimized target"
+ );
+ std::thread::sleep(Duration::from_millis(300));
+ wait_for_fixture_file_text(&state_path, "lbl-counter", "counter=0");
+
+ passed.extend([OracleKind::FixtureState, OracleKind::Protocol]);
+ passed.sort();
+ passed.dedup();
+ Observation::refused(
+ RefusalCode::WindowMinimized,
+ passed,
+ response.text(),
+ Evidence::default(),
+ )
+ });
+}
+
#[test]
#[ignore]
fn harness_wpf_left_click_px_background() {
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
index 0a13ad1a4f..649f1989b6 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
@@ -408,6 +408,122 @@ pub fn point_in_window_bounds(hwnd: u64, x: i32, y: i32) -> bool {
x >= r.left && x < r.right && y >= r.top && y < r.bottom
}
+fn point_in_rect(x: i32, y: i32, left: i32, top: i32, width: i32, height: i32) -> bool {
+ if width <= 0 || height <= 0 {
+ return false;
+ }
+ let (x, y, left, top, width, height) = (
+ i64::from(x),
+ i64::from(y),
+ i64::from(left),
+ i64::from(top),
+ i64::from(width),
+ i64::from(height),
+ );
+ x >= left && x < left + width && y >= top && y < top + height
+}
+
+/// True when `(x, y)` belongs to the live Windows virtual desktop.
+///
+/// Unlike a fixed negative-coordinate threshold, this admits every layout the
+/// OS actually reports. It also rejects the iconic `(-32000, -32000)` region
+/// when a transient `IsIconic` query races with UIA/GetWindowRect state.
+pub fn point_on_virtual_desktop(x: i32, y: i32) -> bool {
+ use windows::Win32::UI::WindowsAndMessaging::{
+ GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN,
+ SM_YVIRTUALSCREEN,
+ };
+ unsafe {
+ point_in_rect(
+ x,
+ y,
+ GetSystemMetrics(SM_XVIRTUALSCREEN),
+ GetSystemMetrics(SM_YVIRTUALSCREEN),
+ GetSystemMetrics(SM_CXVIRTUALSCREEN),
+ GetSystemMetrics(SM_CYVIRTUALSCREEN),
+ )
+ }
+}
+
+/// True when `hwnd` is minimized (iconic).
+///
+/// Authoritative counterpart to [`is_iconic_sentinel_point`]: the capture path
+/// already refuses iconic windows outright (see `capture.rs`), and the input
+/// path needs the same signal so an element action can fail loudly instead of
+/// posting into the off-screen iconic region.
+pub fn window_is_iconic(hwnd: u64) -> bool {
+ use windows::Win32::Foundation::HWND;
+ use windows::Win32::UI::WindowsAndMessaging::IsIconic;
+ if hwnd == 0 {
+ return false;
+ }
+ unsafe { IsIconic(HWND(hwnd as *mut _)).as_bool() }
+}
+
+#[cfg(test)]
+mod iconic_sentinel_tests {
+ use super::*;
+
+ /// The exact rect Win32 hands back for a minimized window, as documented on
+ /// the capture-path guard in `capture.rs`.
+ const ICONIC_RECT: (i32, i32, i32, i32) = (-32000, -32000, -31840, -31972);
+
+ #[test]
+ fn iconic_rect_passes_the_positive_extent_check_that_guards_the_bounds_path() {
+ // This is why #2015 is silent rather than refused: the sentinel rect is
+ // a *well-formed* rectangle, so validity checks phrased as
+ // "right > left && bottom > top" admit it.
+ let (left, top, right, bottom) = ICONIC_RECT;
+ assert!(right > left, "sentinel width is positive: {right} > {left}");
+ assert!(
+ bottom > top,
+ "sentinel height is positive: {bottom} > {top}"
+ );
+ }
+
+ #[test]
+ fn virtual_desktop_membership_has_no_magic_negative_ceiling() {
+ assert!(point_in_rect(-31920, 50, -40000, 0, 50000, 2000));
+ }
+
+ #[test]
+ fn iconic_center_is_outside_an_ordinary_virtual_desktop() {
+ let (left, top, right, bottom) = ICONIC_RECT;
+ let (cx, cy) = ((left + right) / 2, (top + bottom) / 2);
+ assert!(!point_in_rect(cx, cy, -2560, -1080, 6400, 3240));
+ }
+
+ #[test]
+ fn iconic_value_on_either_axis_is_outside_the_desktop() {
+ assert!(!point_in_rect(640, -32000, -2560, -1080, 6400, 3240));
+ assert!(!point_in_rect(-32000, 480, -2560, -1080, 6400, 3240));
+ }
+
+ #[test]
+ fn real_negative_origin_monitor_points_remain_valid() {
+ for (x, y) in [
+ (-1920, 0),
+ (-1795, 383),
+ (-2560, 0),
+ (0, -1080),
+ (-1920, -1080),
+ (0, 0),
+ (1920, 1080),
+ ] {
+ assert!(point_in_rect(x, y, -2560, -1080, 6400, 3240));
+ }
+ }
+
+ #[test]
+ fn virtual_desktop_edges_and_invalid_extents_fail_closed() {
+ assert!(point_in_rect(-1795, 383, -2560, -1080, 6400, 3240));
+ assert!(!point_in_rect(-2561, 383, -2560, -1080, 6400, 3240));
+ assert!(!point_in_rect(3840, 0, -2560, -1080, 6400, 3240));
+ assert!(!point_in_rect(0, 0, 0, 0, 0, 1080));
+ assert!(!point_in_rect(0, 0, 0, 0, 1920, -1));
+ }
+}
+
/// One pen press-drag-release from screen `(sx0,sy0)` to `(sx1,sy1)`, with
/// `steps` interpolated in-contact UPDATE points between the down and the up.
/// A single synthetic pen device is created for the whole stroke. The barrel
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
index dbf99e2ef5..9d9c05321c 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
@@ -16,7 +16,8 @@ pub mod mouse;
pub(crate) use inject::{force_foreground_assisted, force_foreground_attached};
pub use inject::{
- inject_click_screen, point_in_window_bounds, ForegroundLockGuard, NoActivateGuard,
+ inject_click_screen, point_in_window_bounds, point_on_virtual_desktop, window_is_iconic,
+ ForegroundLockGuard, NoActivateGuard,
};
pub use keyboard::{
is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 152aeb9782..69da996fb6 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -42,8 +42,8 @@ fn pin_overlay_above(key: &str, hwnd: u64) {
/// Returns:
/// - `Ok((cx,cy))` unchanged when already in bounds — fast path, no COM call;
/// - `Ok((nx,ny))` with the post-scroll center when scrolling brought it in;
-/// - `Err(message)` when it is still off-screen — preserving the existing clean
-/// failure (a clear error beats a misfire onto the taskbar).
+/// - `Err(result)` with a typed refusal when minimized or still off-screen — a
+/// clear error beats a misfire onto the taskbar.
///
/// Background-safe: ScrollIntoView is delivered over the accessibility channel
/// (wrapped in the UWP fg-steal bypass) and does not raise or activate the
@@ -57,8 +57,39 @@ fn resolve_onscreen_point_with_scroll(
cx: i32,
cy: i32,
action_verb: &str,
-) -> Result<(i32, i32), String> {
+) -> Result<(i32, i32), ToolResult> {
+ // Issue #2015: refuse the iconic sentinel BEFORE the containment check
+ // below, which structurally cannot catch it. A minimized window's
+ // `GetWindowRect` is the off-screen iconic position
+ // `(-32000, -32000)-(-31840, -31972)`, and UIA mirrors that same sentinel
+ // into every element's `BoundingRectangle`. Both sides of
+ // `point_in_window_bounds` are then poisoned identically, so a sentinel
+ // center tests as *inside* its sentinel window rect, the guard passes, and
+ // the click is posted to coordinates no monitor covers — the tool reports
+ // success and nothing happens. `capture.rs` already refuses iconic windows
+ // outright; the input path must refuse them too rather than silently no-op.
+ if let Some(refusal) = minimized_element_refusal(hwnd, idx, cx, cy, action_verb) {
+ return Err(refusal);
+ }
let cached_center_in_window = crate::input::point_in_window_bounds(hwnd, cx, cy);
+ if cached_center_in_window && !crate::input::point_on_virtual_desktop(cx, cy) {
+ let message = format!(
+ "Element [{idx}] resolves to ({cx},{cy}) inside window 0x{hwnd:x}, but \
+ that point is outside the live Windows virtual desktop. The bounds \
+ may be a stale iconic position, so {action_verb} was refused before \
+ input dispatch. Restore or move the window onto a display, then \
+ re-snapshot with get_window_state before retrying."
+ );
+ return Err(ToolResult::error(message.clone()).with_structured(json!({
+ "status": "refused",
+ "code": "element_not_visible",
+ "effect": "refused",
+ "refusal": {
+ "code": "element_not_visible",
+ "message": message,
+ }
+ })));
+ }
// A center can still be clipped by a ScrollViewer or a docked sibling while
// remaining inside the outer HWND rectangle. UIA's IsOffscreen property is
// authoritative for that case; without it a foreground tap can land on the
@@ -84,12 +115,60 @@ fn resolve_onscreen_point_with_scroll(
} else if cached_center_in_window {
return Ok((cx, cy));
}
- Err(format!(
+ let message = format!(
"Element [{idx}] resolves to ({cx},{cy}) but is not visibly actionable — \
tried UIA ScrollIntoView but it is still off-screen, so {action_verb} \
would land on whatever covers that point. Make the window taller or \
scroll the region into view, then retry."
- ))
+ );
+ Err(ToolResult::error(message.clone()).with_structured(json!({
+ "status": "refused",
+ "code": "element_not_visible",
+ "effect": "refused",
+ "refusal": {
+ "code": "element_not_visible",
+ "message": message,
+ }
+ })))
+}
+
+fn minimized_element_refusal(
+ hwnd: u64,
+ idx: usize,
+ cx: i32,
+ cy: i32,
+ action_verb: &str,
+) -> Option<ToolResult> {
+ crate::input::window_is_iconic(hwnd).then(|| {
+ let message = format!(
+ "Element [{idx}] resolves to ({cx},{cy}), which is the Win32 iconic \
+ sentinel rather than a location on any display — window 0x{hwnd:x} \
+ is minimized, so {action_verb} would \
+ be posted off-screen and silently do nothing. Call bring_to_front \
+ with this window_id to restore it, then re-snapshot with \
+ get_window_state before retrying."
+ );
+ ToolResult::error(format!("refused (window_minimized): {message}")).with_structured(json!({
+ "status": "refused",
+ "code": "window_minimized",
+ "effect": "refused",
+ "refusal": {
+ "code": "window_minimized",
+ "message": message,
+ }
+ }))
+ })
+}
+
+fn tool_result_text(result: ToolResult) -> String {
+ result
+ .content
+ .into_iter()
+ .find_map(|content| match content {
+ cua_driver_core::protocol::Content::Text { text, .. } => Some(text),
+ _ => None,
+ })
+ .unwrap_or_else(|| "element action was refused".to_owned())
}
/// Convert (px, py) — "window-local screenshot pixels, top-left origin
@@ -3229,7 +3308,7 @@ impl Tool for ClickTool {
"a foreground click",
) {
Ok(p) => p,
- Err(msg) => return ToolResult::error(msg),
+ Err(result) => return result,
};
let btn_fg = btn.clone();
let mods_owned = modifiers.clone();
@@ -3253,6 +3332,12 @@ impl Tool for ClickTool {
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
}
+ // UIA and posted-message routes can report success without a
+ // user-visible effect on an iconic target. Refuse before any
+ // background route selects or dispatches its transport.
+ if let Some(result) = minimized_element_refusal(hwnd, idx, cx, cy, "clicking") {
+ return result;
+ }
// delivery_mode:"background" on WinUI3 for double / right / middle: single
// left falls through to the UIA Invoke path below (already drives
// WinUI3). Double-left lands via a double UIA Invoke; right/middle
@@ -3317,7 +3402,7 @@ impl Tool for ClickTool {
let (cx, cy) = resolve_onscreen_point_with_scroll(
&state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking",
)
- .map_err(|message| anyhow::anyhow!(message))?;
+ .map_err(|result| anyhow::anyhow!(tool_result_text(result)))?;
return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn)
.map(|()| format!(
"✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)."
@@ -3432,7 +3517,7 @@ impl Tool for ClickTool {
cy,
"clicking",
)
- .map_err(|message| anyhow::anyhow!(message))?;
+ .map_err(|result| anyhow::anyhow!(tool_result_text(result)))?;
return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn)
.map(|()| {
format!(
@@ -4059,7 +4144,7 @@ impl Tool for TypeTextTool {
"foreground typing",
) {
Ok(point) => point,
- Err(message) => return ToolResult::error(message),
+ Err(result) => return result,
};
let focus_result = tokio::task::spawn_blocking(move || {
crate::input::send_click_synthesized_active_mods(hwnd, cx, cy, 1, "left", &[])
@@ -4744,7 +4829,7 @@ impl Tool for PressKeyTool {
"focusing for key delivery",
) {
Ok(point) => point,
- Err(message) => return ToolResult::error(message),
+ Err(result) => return result,
};
let (mut px, mut py) = screen_to_bitmap(hwnd, cx, cy);
if let Some(ratio) = self.state.resize_registry.ratio(pid) {
@@ -6089,7 +6174,7 @@ impl Tool for DoubleClickTool {
"double-clicking",
) {
Ok(p) => p,
- Err(msg) => return ToolResult::error(msg),
+ Err(result) => return result,
};
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;
@@ -6436,7 +6521,7 @@ impl Tool for RightClickTool {
"right-clicking",
) {
Ok(p) => p,
- Err(msg) => return ToolResult::error(msg),
+ Err(result) => return result,
};
pin_overlay_above(&cursor_key, hwnd);
overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;