Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): refuse Windows Chromium clicks from a minimized-window snapshot (#5686)

Philip Leongde783ee0a6ee
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rs
index de06bf2537..ec67d5bd77 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rs
@@ -12,19 +12,24 @@
#![cfg(target_os = "windows")]
-use std::cell::Cell;
+use std::cell::{Cell, RefCell};
use std::io::Read;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use cua_driver_testkit::e2e::{
- execute_case, native_background_case, recording_evidence, DriverRoute, Observation, Targeting,
+ execute_case, native_background_case, recording_evidence, CaseSpec, Delivery, DriverRoute,
+ Evidence, Observation, OracleKind, RefusalCode, Scope, Targeting,
};
use cua_driver_testkit::observer::TargetWindow;
use cua_driver_testkit::sentinel::run_with_background_oracles;
use cua_driver_testkit::{
- harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse,
+ ax, harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse,
+};
+use windows::Win32::Foundation::HWND;
+use windows::Win32::UI::WindowsAndMessaging::{
+ IsIconic, ShowWindow, SW_MINIMIZE, SW_SHOWNOACTIVATE,
};
// ── workspace paths ──────────────────────────────────────────────────────────
@@ -101,19 +106,22 @@ where
title_substr,
|_, _, _, _| {},
|pid, wid, driver, _| f(pid, wid, driver),
+ |(), passed| Observation::delivered_with_fixture_state(passed),
);
}
-fn run_web_case_with_preparation<P, F>(
+fn run_web_case_with_preparation<P, F, R, O>(
case: cua_driver_testkit::e2e::CaseSpec,
toolkit: &str,
host_exe: PathBuf,
title_substr: &str,
prepare: P,
f: F,
+ observe: O,
) where
P: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal),
- F: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal),
+ F: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal) -> R,
+ O: FnOnce(R, Vec<OracleKind>) -> Observation,
{
let cell_id = case.cell_id.clone();
execute_case(case, |evidence| {
@@ -201,7 +209,7 @@ fn run_web_case_with_preparation<P, F>(
std::thread::sleep(Duration::from_millis(50));
}
prepare(pid, wid, &mut driver, &journal);
- let (_, passed) = run_with_background_oracles(
+ let (result, passed) = run_with_background_oracles(
&mut driver,
TargetWindow {
pid: pid as u32,
@@ -210,7 +218,7 @@ fn run_web_case_with_preparation<P, F>(
|driver| f(pid, wid, driver, &journal),
)
.unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));
- Observation::delivered_with_fixture_state(passed)
+ observe(result, passed)
});
}
@@ -384,6 +392,7 @@ fn harness_webview_left_click_px_background() {
click.text(),
);
},
+ |(), passed| Observation::delivered_with_fixture_state(passed),
);
}
@@ -530,3 +539,100 @@ fn harness_electron_click_element() {
},
);
}
+
+// ── Electron minimized snapshot ──────────────────────────────────────────────
+
+fn wait_for_iconic(hwnd: HWND, iconic: bool) {
+ let deadline = Instant::now() + Duration::from_secs(2);
+ while unsafe { IsIconic(hwnd) }.as_bool() != iconic {
+ assert!(
+ Instant::now() < deadline,
+ "Electron window did not reach iconic={iconic}"
+ );
+ std::thread::sleep(Duration::from_millis(25));
+ }
+}
+
+#[test]
+#[ignore]
+fn harness_electron_minimized_snapshot_click_refuses_without_side_effects() {
+ let case = CaseSpec::delivered(
+ "windows-electron-minimized-snapshot-click-refusal",
+ "electron",
+ "electron",
+ "left_click",
+ Targeting::Ax,
+ Delivery::Background,
+ Scope::Window,
+ DriverRoute::PostMessage,
+ vec![
+ OracleKind::FixtureState,
+ OracleKind::Focus,
+ OracleKind::ZOrder,
+ OracleKind::Cursor,
+ OracleKind::NoLeakedInput,
+ OracleKind::Protocol,
+ ],
+ )
+ .expecting_refusal(vec![RefusalCode::ElementNotVisible]);
+ let token = RefCell::new(None);
+ run_web_case_with_preparation(
+ case,
+ "electron",
+ electron_exe(),
+ "CuaTestHarness Electron",
+ |pid, wid, driver, journal| {
+ wait_for_journal_text(journal, "lbl-counter", "counter=0");
+ // A visible read first, so the minimized read below sees the
+ // renderer's accessibility tree.
+ let _ = snapshot(driver, pid, wid);
+ let hwnd = HWND(wid as *mut _);
+ let _ = unsafe { ShowWindow(hwnd, SW_MINIMIZE) };
+ wait_for_iconic(hwnd, true);
+ let minimized = snapshot(driver, pid, wid);
+ let idx = ax::element_index_by_id(minimized.tree_text(), "btn-increment")
+ .expect("btn-increment not in the minimized snapshot");
+ token.replace(Some(minimized.element_token(idx).to_owned()));
+ let _ = unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE) };
+ wait_for_iconic(hwnd, false);
+ },
+ |pid, wid, driver, journal| {
+ let token = token
+ .take()
+ .expect("minimized snapshot did not set a token");
+ let response = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid,
+ "window_id": wid,
+ "element_token": token,
+ "delivery_mode": "background"
+ }),
+ );
+ assert!(
+ response.is_error(),
+ "click with a minimized-snapshot token reported success: {}",
+ response.text()
+ );
+ assert_eq!(response.structured()["status"], "refused");
+ assert_eq!(
+ response.structured()["refusal"]["code"],
+ "element_not_visible"
+ );
+ std::thread::sleep(Duration::from_millis(300));
+ wait_for_journal_text(journal, "lbl-counter", "counter=0");
+ response.text().to_owned()
+ },
+ |message, mut passed| {
+ passed.extend([OracleKind::FixtureState, OracleKind::Protocol]);
+ passed.sort();
+ passed.dedup();
+ Observation::refused(
+ RefusalCode::ElementNotVisible,
+ passed,
+ message,
+ Evidence::default(),
+ )
+ },
+ );
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
index f7cbfea4f7..388fc60b19 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
@@ -318,6 +318,7 @@ pub enum RefusalCode {
BrowserActionUnavailable,
ModifiedPointerUnavailable,
WindowMinimized,
+ ElementNotVisible,
ForegroundUnavailable,
}
@@ -345,6 +346,7 @@ impl RefusalCode {
"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),
"modified_pointer_unavailable" => Some(Self::ModifiedPointerUnavailable),
"window_minimized" => Some(Self::WindowMinimized),
+ "element_not_visible" => Some(Self::ElementNotVisible),
"foreground_unavailable" => Some(Self::ForegroundUnavailable),
_ => None,
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 896e77dd0e..9964932134 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -1447,7 +1447,8 @@ impl Tool for GetWindowStateTool {
} else {
crate::uia::snapshot::SnapshotKind::Uia
};
- let payload = crate::uia::snapshot::UiaSnapshot::from_nodes(&tree.nodes, kind);
+ let payload = crate::uia::snapshot::UiaSnapshot::from_nodes(&tree.nodes, kind)
+ .with_read_while_minimized(crate::input::window_is_iconic(hwnd));
(tree, payload, budget.outcome())
});
let started = std::time::Instant::now();
@@ -3935,6 +3936,27 @@ impl Tool for ClickTool {
&& count == 1
&& crate::input::is_chromium_target_window(hwnd)
{
+ // This route posts at the cached center without
+ // `resolve_onscreen_point_with_scroll`, so check it here.
+ if admitted
+ .as_ref()
+ .is_some_and(|element| element.read_while_minimized)
+ {
+ let message = format!(
+ "Element [{idx}] resolves to ({cx},{cy}), which was read while \
+ window 0x{hwnd:x} was minimized. Re-snapshot with get_window_state \
+ before retrying."
+ );
+ return ToolResult::error(message.clone()).with_structured(json!({
+ "status": "refused",
+ "code": "element_not_visible",
+ "effect": "refused",
+ "refusal": {
+ "code": "element_not_visible",
+ "message": message,
+ }
+ }));
+ }
let posted = tokio::task::spawn_blocking({
let admitted = admitted.clone();
move || {
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rs
index 5f2e69a7ba..d58c62ab5c 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rs
@@ -12,6 +12,7 @@ pub struct RetainedElement {
pub center: (i32, i32),
pub rect: Option<(i32, i32, i32, i32)>,
pub msaa_role: Option<i32>,
+ pub read_while_minimized: bool,
}
impl RetainedElement {
@@ -72,6 +73,7 @@ impl Clone for RetainedElement {
center: self.center,
rect: self.rect,
msaa_role: self.msaa_role,
+ read_while_minimized: self.read_while_minimized,
}
}
}
@@ -111,10 +113,18 @@ impl UiaSnapshot {
center: (node.center_x, node.center_y),
rect: node.rect,
msaa_role: node.msaa_role,
+ read_while_minimized: false,
})
.collect(),
}
}
+
+ pub fn with_read_while_minimized(mut self, minimized: bool) -> Self {
+ for element in &mut self.elements {
+ element.read_while_minimized = minimized;
+ }
+ self
+ }
}
impl SnapshotPayload for UiaSnapshot {
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rs
index 276517bb69..68c4c542b7 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rs
@@ -114,6 +114,7 @@ fn snapshot_with(ptrs: Vec<usize>) -> UiaSnapshot {
center: (0, 0),
rect: None,
msaa_role: None,
+ read_while_minimized: false,
})
.collect(),
}