Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): refuse minimized Windows element actions
webdev8a9b7baa32bb
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rsindex 8671836fbe..362f76118a 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs@@ -787,8 +787,16 @@ fn write_turn(}};let click_family = matches!(tool_name.as_str(), "click" | "double_click" | "right_click");+ let action_refused = action_record+ .is_some_and(|record| record.effect == crate::action_record::ActionEffect::Refused);let refused_before_target_resolution = click_family && result_is_error && click_point.is_none();- let click_expected = click_family && !refused_before_target_resolution;+ // A target may resolve successfully and still be refused before input+ // dispatch (for example, a minimized Windows element). Retaining the+ // resolved point in action.json is useful diagnostic context, but a+ // crosshair would falsely imply that a click was delivered.+ let refused_before_dispatch = click_family && result_is_error && action_refused;+ let click_expected =+ click_family && !refused_before_target_resolution && !refused_before_dispatch;let mut payload = serde_json::json!({"tool": tool_name,@@ -820,14 +828,16 @@ fn write_turn(// the pre-action image. This also keeps modal-dismiss evidence available// after the modal HWND has closed.let mut click_captured = false;- if let (Some((cx, cy)), Some(screenshot), Some(marker)) = (- click_point,- before.screenshot.as_deref(),- CLICK_MARKER_FN.get(),- ) {- if let Some(click_png) = marker(screenshot, cx, cy) {- std::fs::write(turn_dir.join("click.png"), click_png)?;- click_captured = true;+ if click_expected {+ if let (Some((cx, cy)), Some(screenshot), Some(marker)) = (+ click_point,+ before.screenshot.as_deref(),+ CLICK_MARKER_FN.get(),+ ) {+ if let Some(click_png) = marker(screenshot, cx, cy) {+ std::fs::write(turn_dir.join("click.png"), click_png)?;+ click_captured = true;+ }}}write_evidence_manifest(@@ -839,6 +849,8 @@ fn write_turn(click_captured,if refused_before_target_resolution {"action_refused_before_target_resolution"+ } else if refused_before_dispatch {+ "action_refused_before_dispatch"} else {"not_a_click_action"},@@ -1064,6 +1076,49 @@ mod tests {"action_refused_before_target_resolution");+ let pending = session+ .begin_turn(+ "click",+ &serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}),+ now_ms(),+ )+ .expect("resolved refusal should reserve an evidence turn");+ let refusal_record = crate::action_record::ActionExecutionRecord::builder(+ crate::action_record::ActionEffect::Refused,+ crate::action_record::ActionTransport::WindowsTargetedInjection,+ crate::action_record::RequestedDelivery::Background,+ )+ .build()+ .expect("valid refusal record");+ session.finish_turn_with_outcome(+ pending,+ "refused before dispatch",+ Some(&refusal_record),+ true,+ );+ let resolved_refusal_turn = output_dir.join("turn-00004");+ let resolved_refusal_action: Value = serde_json::from_slice(+ &std::fs::read(resolved_refusal_turn.join("action.json"))+ .expect("read resolved refusal action"),+ )+ .expect("parse resolved refusal action");+ assert_eq!(resolved_refusal_action["click_point"]["x"], 3.0);+ assert_eq!(resolved_refusal_action["action_truth"]["effect"], "refused");+ assert!(!resolved_refusal_turn.join("click.png").exists());+ let resolved_refusal_manifest: Value = serde_json::from_slice(+ &std::fs::read(resolved_refusal_turn.join("evidence.json"))+ .expect("read resolved refusal evidence"),+ )+ .expect("parse resolved refusal evidence");+ assert_eq!(+ resolved_refusal_manifest["click"]["status"],+ "not_applicable"+ );+ assert_eq!(+ resolved_refusal_manifest["click"]["classification"],+ "action_refused_before_dispatch"+ );+let files = ["action.json","app_state.json",@@ -1081,11 +1136,13 @@ mod tests {}std::fs::remove_dir(directory).expect("remove turn fixture directory");}- for file in files.into_iter().filter(|file| *file != "click.png") {- std::fs::remove_file(refused_turn.join(file))- .expect("remove refused turn fixture file");+ for directory in [&refused_turn, &resolved_refusal_turn] {+ for file in files.iter().copied().filter(|file| *file != "click.png") {+ std::fs::remove_file(directory.join(file))+ .expect("remove refused turn fixture file");+ }+ std::fs::remove_dir(directory).expect("remove refused turn fixture directory");}- std::fs::remove_dir(refused_turn).expect("remove refused turn fixture directory");std::fs::remove_dir(&output_dir).expect("remove recording fixture directory");}diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rsindex 23a42cab59..848a7607da 100644--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs@@ -315,6 +315,7 @@ pub enum RefusalCode {BrowserReconnectExhausted,BrowserInputIncomplete,BrowserActionUnavailable,+ WindowMinimized,}impl RefusalCode {@@ -336,6 +337,7 @@ impl RefusalCode {"browser_reconnect_exhausted" => Some(Self::BrowserReconnectExhausted),"browser_input_incomplete" => Some(Self::BrowserInputIncomplete),"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),+ "window_minimized" => Some(Self::WindowMinimized),_ => None,}}@@ -1493,6 +1495,12 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {.as_str().is_some_and(|summary| summary.starts_with("✅ bring_to_front:"))});+ let refused_minimized_action = action.as_ref().is_some_and(|value| {+ value["result_error"].as_bool() == Some(true)+ && value["result_summary"]+ .as_str()+ .is_some_and(|summary| summary.starts_with("refused (window_minimized):"))+ });let restored_state_captured = manifest.as_ref().is_some_and(|value| value["after"]["state"]["status"].as_str() == Some("captured"));@@ -1506,6 +1514,8 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {&& successful_restore&& restored_state_captured&& classification(phase, "screenshot") == Some("capture_failed"))))+ || (refused_minimized_action+ && classification(phase, "screenshot") == Some("target_minimized"))};for (phase, kind) in [("before", "screenshot"), ("after", "screenshot")] {@@ -1577,15 +1587,23 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {let refused_before_target_resolution = action.as_ref().is_some_and(|value| {value["result_error"].as_bool() == Some(true) && value.get("click_point").is_none()});- if refused_before_target_resolution {+ let refused_before_dispatch = action.as_ref().is_some_and(|value| {+ value["result_error"].as_bool() == Some(true)+ && value["action_truth"]["effect"].as_str() == Some("refused")+ });+ if refused_before_target_resolution || refused_before_dispatch {let click = manifest.as_ref().map(|value| &value["click"]);+ let expected_classification = if refused_before_target_resolution {+ "action_refused_before_target_resolution"+ } else {+ "action_refused_before_dispatch"+ };if !click.is_some_and(|value| {value["status"].as_str() == Some("not_applicable")- && value["classification"].as_str()- == Some("action_refused_before_target_resolution")+ && value["classification"].as_str() == Some(expected_classification)}) {errors.push(format!(- "invalid refused-click evidence for {cell_id}/{turn_name}: expected not_applicable/action_refused_before_target_resolution"+ "invalid refused-click evidence for {cell_id}/{turn_name}: expected not_applicable/{expected_classification}"));}return;@@ -2222,6 +2240,42 @@ mod tests {.expect("a minimized target cannot provide a pre-restore screenshot");}+ #[test]+ fn validator_accepts_missing_images_for_typed_minimized_refusal() {+ let (root, case, result, turn) = complete_turn_fixture();+ std::fs::write(+ turn.join("action.json"),+ br#"{+ "tool":"click",+ "arguments":{"pid":1,"window_id":2,"element_index":3},+ "result_summary":"refused (window_minimized): restore before retrying",+ "result_error":true,+ "click_point":{"x":10,"y":20},+ "action_truth":{"effect":"refused"}+ }"#,+ )+ .expect("write minimized refusal action");+ std::fs::write(+ turn.join("evidence.json"),+ br#"{+ "schema":"cua-turn-evidence/v1",+ "before":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}},+ "after":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}},+ "click":{"status":"not_applicable","classification":"action_refused_before_dispatch"}+ }"#,+ )+ .expect("write minimized refusal evidence manifest");+ for file in ["before.png", "after.png", "screenshot.png", "click.png"] {+ let path = turn.join(file);+ if path.exists() {+ std::fs::remove_file(path).expect("remove unavailable image");+ }+ }++ validate_catalog(&[case], &[result], Some(root.path()), true)+ .expect("a typed minimized refusal cannot provide target images");+ }+#[test]fn validator_accepts_restored_state_when_host_capture_remains_unavailable() {let (root, case, result, turn) = complete_turn_fixture();diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rsindex c37857fc00..33e1847824 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs@@ -48,6 +48,8 @@ use cua_driver_testkit::e2e::{use cua_driver_testkit::observer::TargetWindow;use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel};use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolResponse};+use windows::Win32::Foundation::HWND;+use windows::Win32::UI::WindowsAndMessaging::{IsIconic, ShowWindow, SW_MINIMIZE};// ── harness launcher ─────────────────────────────────────────────────────────@@ -549,6 +551,95 @@ fn harness_wpf_counter_invoke() {);}+#[test]+#[ignore]+fn harness_wpf_minimized_element_click_refuses_without_side_effects() {+ let case = CaseSpec::delivered(+ "windows-wpf-minimized-element-click-refusal",+ "wpf",+ "wpf",+ "left_click",+ Targeting::Ax,+ Delivery::Background,+ Scope::Window,+ DriverRoute::Composite,+ vec![+ OracleKind::FixtureState,+ OracleKind::Focus,+ OracleKind::ZOrder,+ OracleKind::Cursor,+ OracleKind::NoLeakedInput,+ OracleKind::Protocol,+ ],+ )+ .expecting_refusal(vec![RefusalCode::WindowMinimized]);++ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_named("windows-wpf-minimized-element-click-refusal")+ .expect("required source-built driver did not start");+ *evidence = recording_evidence(driver.recording_dir());+ let state_dir = tempfile::tempdir().expect("create WPF fixture state directory");+ let state_path = state_dir.path().join("state.json");+ let pid = launch_harness_with_state_file(&mut driver, Some(&state_path))+ .expect("required WPF harness did not launch");+ let (wid, _) = driver+ .find_window(pid as i64, "CuaTestHarness WPF")+ .expect("main window");+ wait_for_fixture_file_text(&state_path, "lbl-counter", "counter=0");+ let ready = snapshot(&mut driver, pid, wid);+ let idx = ax::element_index_by_id(ready.text(), "btn-increment")+ .expect("btn-increment not in pre-minimize snapshot");++ let sentinel = ForegroundSentinel::launch(&mut driver);+ let hwnd = HWND(wid as *mut _);+ let _ = unsafe { ShowWindow(hwnd, SW_MINIMIZE) };+ let deadline = Instant::now() + Duration::from_secs(2);+ while !unsafe { IsIconic(hwnd) }.as_bool() {+ assert!(Instant::now() < deadline, "WPF target did not minimize");+ std::thread::sleep(Duration::from_millis(25));+ }+ driver.start_behavior_recording();++ let (response, mut passed) = sentinel+ .observe_desktop(|| {+ driver.call(+ "click",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": wid,+ "element_index": idx,+ "snapshot_id": ready.snapshot_id(),+ "delivery_mode": "background"+ }),+ )+ })+ .unwrap_or_else(|error| panic!("minimized refusal leaked desktop state: {error}"));+ assert!(+ response.is_error(),+ "minimized click reported success: {}",+ response.text()+ );+ assert_eq!(response.structured()["status"], "refused");+ assert_eq!(response.structured()["refusal"]["code"], "window_minimized");+ assert!(+ unsafe { IsIconic(hwnd) }.as_bool(),+ "refused click restored or otherwise changed the minimized target"+ );+ std::thread::sleep(Duration::from_millis(300));+ wait_for_fixture_file_text(&state_path, "lbl-counter", "counter=0");++ passed.extend([OracleKind::FixtureState, OracleKind::Protocol]);+ passed.sort();+ passed.dedup();+ Observation::refused(+ RefusalCode::WindowMinimized,+ passed,+ response.text(),+ Evidence::default(),+ )+ });+}+#[test]#[ignore]fn harness_wpf_left_click_px_background() {diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rsindex 0a13ad1a4f..649f1989b6 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs@@ -408,6 +408,122 @@ pub fn point_in_window_bounds(hwnd: u64, x: i32, y: i32) -> bool {x >= r.left && x < r.right && y >= r.top && y < r.bottom}+fn point_in_rect(x: i32, y: i32, left: i32, top: i32, width: i32, height: i32) -> bool {+ if width <= 0 || height <= 0 {+ return false;+ }+ let (x, y, left, top, width, height) = (+ i64::from(x),+ i64::from(y),+ i64::from(left),+ i64::from(top),+ i64::from(width),+ i64::from(height),+ );+ x >= left && x < left + width && y >= top && y < top + height+}++/// True when `(x, y)` belongs to the live Windows virtual desktop.+///+/// Unlike a fixed negative-coordinate threshold, this admits every layout the+/// OS actually reports. It also rejects the iconic `(-32000, -32000)` region+/// when a transient `IsIconic` query races with UIA/GetWindowRect state.+pub fn point_on_virtual_desktop(x: i32, y: i32) -> bool {+ use windows::Win32::UI::WindowsAndMessaging::{+ GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN,+ SM_YVIRTUALSCREEN,+ };+ unsafe {+ point_in_rect(+ x,+ y,+ GetSystemMetrics(SM_XVIRTUALSCREEN),+ GetSystemMetrics(SM_YVIRTUALSCREEN),+ GetSystemMetrics(SM_CXVIRTUALSCREEN),+ GetSystemMetrics(SM_CYVIRTUALSCREEN),+ )+ }+}++/// True when `hwnd` is minimized (iconic).+///+/// Authoritative counterpart to [`is_iconic_sentinel_point`]: the capture path+/// already refuses iconic windows outright (see `capture.rs`), and the input+/// path needs the same signal so an element action can fail loudly instead of+/// posting into the off-screen iconic region.+pub fn window_is_iconic(hwnd: u64) -> bool {+ use windows::Win32::Foundation::HWND;+ use windows::Win32::UI::WindowsAndMessaging::IsIconic;+ if hwnd == 0 {+ return false;+ }+ unsafe { IsIconic(HWND(hwnd as *mut _)).as_bool() }+}++#[cfg(test)]+mod iconic_sentinel_tests {+ use super::*;++ /// The exact rect Win32 hands back for a minimized window, as documented on+ /// the capture-path guard in `capture.rs`.+ const ICONIC_RECT: (i32, i32, i32, i32) = (-32000, -32000, -31840, -31972);++ #[test]+ fn iconic_rect_passes_the_positive_extent_check_that_guards_the_bounds_path() {+ // This is why #2015 is silent rather than refused: the sentinel rect is+ // a *well-formed* rectangle, so validity checks phrased as+ // "right > left && bottom > top" admit it.+ let (left, top, right, bottom) = ICONIC_RECT;+ assert!(right > left, "sentinel width is positive: {right} > {left}");+ assert!(+ bottom > top,+ "sentinel height is positive: {bottom} > {top}"+ );+ }++ #[test]+ fn virtual_desktop_membership_has_no_magic_negative_ceiling() {+ assert!(point_in_rect(-31920, 50, -40000, 0, 50000, 2000));+ }++ #[test]+ fn iconic_center_is_outside_an_ordinary_virtual_desktop() {+ let (left, top, right, bottom) = ICONIC_RECT;+ let (cx, cy) = ((left + right) / 2, (top + bottom) / 2);+ assert!(!point_in_rect(cx, cy, -2560, -1080, 6400, 3240));+ }++ #[test]+ fn iconic_value_on_either_axis_is_outside_the_desktop() {+ assert!(!point_in_rect(640, -32000, -2560, -1080, 6400, 3240));+ assert!(!point_in_rect(-32000, 480, -2560, -1080, 6400, 3240));+ }++ #[test]+ fn real_negative_origin_monitor_points_remain_valid() {+ for (x, y) in [+ (-1920, 0),+ (-1795, 383),+ (-2560, 0),+ (0, -1080),+ (-1920, -1080),+ (0, 0),+ (1920, 1080),+ ] {+ assert!(point_in_rect(x, y, -2560, -1080, 6400, 3240));+ }+ }++ #[test]+ fn virtual_desktop_edges_and_invalid_extents_fail_closed() {+ assert!(point_in_rect(-1795, 383, -2560, -1080, 6400, 3240));+ assert!(!point_in_rect(-2561, 383, -2560, -1080, 6400, 3240));+ assert!(!point_in_rect(3840, 0, -2560, -1080, 6400, 3240));+ assert!(!point_in_rect(0, 0, 0, 0, 0, 1080));+ assert!(!point_in_rect(0, 0, 0, 0, 1920, -1));+ }+}+/// One pen press-drag-release from screen `(sx0,sy0)` to `(sx1,sy1)`, with/// `steps` interpolated in-contact UPDATE points between the down and the up./// A single synthetic pen device is created for the whole stroke. The barreldiff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rsindex dbf99e2ef5..9d9c05321c 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs@@ -16,7 +16,8 @@ pub mod mouse;pub(crate) use inject::{force_foreground_assisted, force_foreground_attached};pub use inject::{- inject_click_screen, point_in_window_bounds, ForegroundLockGuard, NoActivateGuard,+ inject_click_screen, point_in_window_bounds, point_on_virtual_desktop, window_is_iconic,+ ForegroundLockGuard, NoActivateGuard,};pub use keyboard::{is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 152aeb9782..69da996fb6 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -42,8 +42,8 @@ fn pin_overlay_above(key: &str, hwnd: u64) {/// Returns:/// - `Ok((cx,cy))` unchanged when already in bounds — fast path, no COM call;/// - `Ok((nx,ny))` with the post-scroll center when scrolling brought it in;-/// - `Err(message)` when it is still off-screen — preserving the existing clean-/// failure (a clear error beats a misfire onto the taskbar).+/// - `Err(result)` with a typed refusal when minimized or still off-screen — a+/// clear error beats a misfire onto the taskbar.////// Background-safe: ScrollIntoView is delivered over the accessibility channel/// (wrapped in the UWP fg-steal bypass) and does not raise or activate the@@ -57,8 +57,39 @@ fn resolve_onscreen_point_with_scroll(cx: i32,cy: i32,action_verb: &str,-) -> Result<(i32, i32), String> {+) -> Result<(i32, i32), ToolResult> {+ // Issue #2015: refuse the iconic sentinel BEFORE the containment check+ // below, which structurally cannot catch it. A minimized window's+ // `GetWindowRect` is the off-screen iconic position+ // `(-32000, -32000)-(-31840, -31972)`, and UIA mirrors that same sentinel+ // into every element's `BoundingRectangle`. Both sides of+ // `point_in_window_bounds` are then poisoned identically, so a sentinel+ // center tests as *inside* its sentinel window rect, the guard passes, and+ // the click is posted to coordinates no monitor covers — the tool reports+ // success and nothing happens. `capture.rs` already refuses iconic windows+ // outright; the input path must refuse them too rather than silently no-op.+ if let Some(refusal) = minimized_element_refusal(hwnd, idx, cx, cy, action_verb) {+ return Err(refusal);+ }let cached_center_in_window = crate::input::point_in_window_bounds(hwnd, cx, cy);+ if cached_center_in_window && !crate::input::point_on_virtual_desktop(cx, cy) {+ let message = format!(+ "Element [{idx}] resolves to ({cx},{cy}) inside window 0x{hwnd:x}, but \+ that point is outside the live Windows virtual desktop. The bounds \+ may be a stale iconic position, so {action_verb} was refused before \+ input dispatch. Restore or move the window onto a display, then \+ re-snapshot with get_window_state before retrying."+ );+ return Err(ToolResult::error(message.clone()).with_structured(json!({+ "status": "refused",+ "code": "element_not_visible",+ "effect": "refused",+ "refusal": {+ "code": "element_not_visible",+ "message": message,+ }+ })));+ }// A center can still be clipped by a ScrollViewer or a docked sibling while// remaining inside the outer HWND rectangle. UIA's IsOffscreen property is// authoritative for that case; without it a foreground tap can land on the@@ -84,12 +115,60 @@ fn resolve_onscreen_point_with_scroll(} else if cached_center_in_window {return Ok((cx, cy));}- Err(format!(+ let message = format!("Element [{idx}] resolves to ({cx},{cy}) but is not visibly actionable — \tried UIA ScrollIntoView but it is still off-screen, so {action_verb} \would land on whatever covers that point. Make the window taller or \scroll the region into view, then retry."- ))+ );+ Err(ToolResult::error(message.clone()).with_structured(json!({+ "status": "refused",+ "code": "element_not_visible",+ "effect": "refused",+ "refusal": {+ "code": "element_not_visible",+ "message": message,+ }+ })))+}++fn minimized_element_refusal(+ hwnd: u64,+ idx: usize,+ cx: i32,+ cy: i32,+ action_verb: &str,+) -> Option<ToolResult> {+ crate::input::window_is_iconic(hwnd).then(|| {+ let message = format!(+ "Element [{idx}] resolves to ({cx},{cy}), which is the Win32 iconic \+ sentinel rather than a location on any display — window 0x{hwnd:x} \+ is minimized, so {action_verb} would \+ be posted off-screen and silently do nothing. Call bring_to_front \+ with this window_id to restore it, then re-snapshot with \+ get_window_state before retrying."+ );+ ToolResult::error(format!("refused (window_minimized): {message}")).with_structured(json!({+ "status": "refused",+ "code": "window_minimized",+ "effect": "refused",+ "refusal": {+ "code": "window_minimized",+ "message": message,+ }+ }))+ })+}++fn tool_result_text(result: ToolResult) -> String {+ result+ .content+ .into_iter()+ .find_map(|content| match content {+ cua_driver_core::protocol::Content::Text { text, .. } => Some(text),+ _ => None,+ })+ .unwrap_or_else(|| "element action was refused".to_owned())}/// Convert (px, py) — "window-local screenshot pixels, top-left origin@@ -3229,7 +3308,7 @@ impl Tool for ClickTool {"a foreground click",) {Ok(p) => p,- Err(msg) => return ToolResult::error(msg),+ Err(result) => return result,};let btn_fg = btn.clone();let mods_owned = modifiers.clone();@@ -3253,6 +3332,12 @@ impl Tool for ClickTool {Err(e) => ToolResult::error(format!("Task error: {e}")),};}+ // UIA and posted-message routes can report success without a+ // user-visible effect on an iconic target. Refuse before any+ // background route selects or dispatches its transport.+ if let Some(result) = minimized_element_refusal(hwnd, idx, cx, cy, "clicking") {+ return result;+ }// delivery_mode:"background" on WinUI3 for double / right / middle: single// left falls through to the UIA Invoke path below (already drives// WinUI3). Double-left lands via a double UIA Invoke; right/middle@@ -3317,7 +3402,7 @@ impl Tool for ClickTool {let (cx, cy) = resolve_onscreen_point_with_scroll(&state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking",)- .map_err(|message| anyhow::anyhow!(message))?;+ .map_err(|result| anyhow::anyhow!(tool_result_text(result)))?;return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn).map(|()| format!("✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)."@@ -3432,7 +3517,7 @@ impl Tool for ClickTool {cy,"clicking",)- .map_err(|message| anyhow::anyhow!(message))?;+ .map_err(|result| anyhow::anyhow!(tool_result_text(result)))?;return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn).map(|()| {format!(@@ -4059,7 +4144,7 @@ impl Tool for TypeTextTool {"foreground typing",) {Ok(point) => point,- Err(message) => return ToolResult::error(message),+ Err(result) => return result,};let focus_result = tokio::task::spawn_blocking(move || {crate::input::send_click_synthesized_active_mods(hwnd, cx, cy, 1, "left", &[])@@ -4744,7 +4829,7 @@ impl Tool for PressKeyTool {"focusing for key delivery",) {Ok(point) => point,- Err(message) => return ToolResult::error(message),+ Err(result) => return result,};let (mut px, mut py) = screen_to_bitmap(hwnd, cx, cy);if let Some(ratio) = self.state.resize_registry.ratio(pid) {@@ -6089,7 +6174,7 @@ impl Tool for DoubleClickTool {"double-clicking",) {Ok(p) => p,- Err(msg) => return ToolResult::error(msg),+ Err(result) => return result,};pin_overlay_above(&cursor_key, hwnd);overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;@@ -6436,7 +6521,7 @@ impl Tool for RightClickTool {"right-clicking",) {Ok(p) => p,- Err(msg) => return ToolResult::error(msg),+ Err(result) => return result,};pin_overlay_above(&cursor_key, hwnd);overlay_glide_to(&cursor_key, cx as f64, cy as f64).await;