Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): refuse Windows Chromium clicks from a minimized-window snapshot (#5686)
Philip Leongde783ee0a6ee
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rsindex de06bf2537..ec67d5bd77 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_web_test.rs@@ -12,19 +12,24 @@#![cfg(target_os = "windows")]-use std::cell::Cell;+use std::cell::{Cell, RefCell};use std::io::Read;use std::path::PathBuf;use std::process::{Command, Stdio};use std::time::{Duration, Instant};use cua_driver_testkit::e2e::{- execute_case, native_background_case, recording_evidence, DriverRoute, Observation, Targeting,+ execute_case, native_background_case, recording_evidence, CaseSpec, Delivery, DriverRoute,+ Evidence, Observation, OracleKind, RefusalCode, Scope, Targeting,};use cua_driver_testkit::observer::TargetWindow;use cua_driver_testkit::sentinel::run_with_background_oracles;use cua_driver_testkit::{- harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse,+ ax, harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse,+};+use windows::Win32::Foundation::HWND;+use windows::Win32::UI::WindowsAndMessaging::{+ IsIconic, ShowWindow, SW_MINIMIZE, SW_SHOWNOACTIVATE,};// ── workspace paths ──────────────────────────────────────────────────────────@@ -101,19 +106,22 @@ wheretitle_substr,|_, _, _, _| {},|pid, wid, driver, _| f(pid, wid, driver),+ |(), passed| Observation::delivered_with_fixture_state(passed),);}-fn run_web_case_with_preparation<P, F>(+fn run_web_case_with_preparation<P, F, R, O>(case: cua_driver_testkit::e2e::CaseSpec,toolkit: &str,host_exe: PathBuf,title_substr: &str,prepare: P,f: F,+ observe: O,) whereP: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal),- F: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal),+ F: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal) -> R,+ O: FnOnce(R, Vec<OracleKind>) -> Observation,{let cell_id = case.cell_id.clone();execute_case(case, |evidence| {@@ -201,7 +209,7 @@ fn run_web_case_with_preparation<P, F>(std::thread::sleep(Duration::from_millis(50));}prepare(pid, wid, &mut driver, &journal);- let (_, passed) = run_with_background_oracles(+ let (result, passed) = run_with_background_oracles(&mut driver,TargetWindow {pid: pid as u32,@@ -210,7 +218,7 @@ fn run_web_case_with_preparation<P, F>(|driver| f(pid, wid, driver, &journal),).unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));- Observation::delivered_with_fixture_state(passed)+ observe(result, passed)});}@@ -384,6 +392,7 @@ fn harness_webview_left_click_px_background() {click.text(),);},+ |(), passed| Observation::delivered_with_fixture_state(passed),);}@@ -530,3 +539,100 @@ fn harness_electron_click_element() {},);}++// ── Electron minimized snapshot ──────────────────────────────────────────────++fn wait_for_iconic(hwnd: HWND, iconic: bool) {+ let deadline = Instant::now() + Duration::from_secs(2);+ while unsafe { IsIconic(hwnd) }.as_bool() != iconic {+ assert!(+ Instant::now() < deadline,+ "Electron window did not reach iconic={iconic}"+ );+ std::thread::sleep(Duration::from_millis(25));+ }+}++#[test]+#[ignore]+fn harness_electron_minimized_snapshot_click_refuses_without_side_effects() {+ let case = CaseSpec::delivered(+ "windows-electron-minimized-snapshot-click-refusal",+ "electron",+ "electron",+ "left_click",+ Targeting::Ax,+ Delivery::Background,+ Scope::Window,+ DriverRoute::PostMessage,+ vec![+ OracleKind::FixtureState,+ OracleKind::Focus,+ OracleKind::ZOrder,+ OracleKind::Cursor,+ OracleKind::NoLeakedInput,+ OracleKind::Protocol,+ ],+ )+ .expecting_refusal(vec![RefusalCode::ElementNotVisible]);+ let token = RefCell::new(None);+ run_web_case_with_preparation(+ case,+ "electron",+ electron_exe(),+ "CuaTestHarness Electron",+ |pid, wid, driver, journal| {+ wait_for_journal_text(journal, "lbl-counter", "counter=0");+ // A visible read first, so the minimized read below sees the+ // renderer's accessibility tree.+ let _ = snapshot(driver, pid, wid);+ let hwnd = HWND(wid as *mut _);+ let _ = unsafe { ShowWindow(hwnd, SW_MINIMIZE) };+ wait_for_iconic(hwnd, true);+ let minimized = snapshot(driver, pid, wid);+ let idx = ax::element_index_by_id(minimized.tree_text(), "btn-increment")+ .expect("btn-increment not in the minimized snapshot");+ token.replace(Some(minimized.element_token(idx).to_owned()));+ let _ = unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE) };+ wait_for_iconic(hwnd, false);+ },+ |pid, wid, driver, journal| {+ let token = token+ .take()+ .expect("minimized snapshot did not set a token");+ let response = driver.call(+ "click",+ serde_json::json!({+ "pid": pid,+ "window_id": wid,+ "element_token": token,+ "delivery_mode": "background"+ }),+ );+ assert!(+ response.is_error(),+ "click with a minimized-snapshot token reported success: {}",+ response.text()+ );+ assert_eq!(response.structured()["status"], "refused");+ assert_eq!(+ response.structured()["refusal"]["code"],+ "element_not_visible"+ );+ std::thread::sleep(Duration::from_millis(300));+ wait_for_journal_text(journal, "lbl-counter", "counter=0");+ response.text().to_owned()+ },+ |message, mut passed| {+ passed.extend([OracleKind::FixtureState, OracleKind::Protocol]);+ passed.sort();+ passed.dedup();+ Observation::refused(+ RefusalCode::ElementNotVisible,+ passed,+ message,+ Evidence::default(),+ )+ },+ );+}diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rsindex f7cbfea4f7..388fc60b19 100644--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs@@ -318,6 +318,7 @@ pub enum RefusalCode {BrowserActionUnavailable,ModifiedPointerUnavailable,WindowMinimized,+ ElementNotVisible,ForegroundUnavailable,}@@ -345,6 +346,7 @@ impl RefusalCode {"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),"modified_pointer_unavailable" => Some(Self::ModifiedPointerUnavailable),"window_minimized" => Some(Self::WindowMinimized),+ "element_not_visible" => Some(Self::ElementNotVisible),"foreground_unavailable" => Some(Self::ForegroundUnavailable),_ => None,}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 896e77dd0e..9964932134 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -1447,7 +1447,8 @@ impl Tool for GetWindowStateTool {} else {crate::uia::snapshot::SnapshotKind::Uia};- let payload = crate::uia::snapshot::UiaSnapshot::from_nodes(&tree.nodes, kind);+ let payload = crate::uia::snapshot::UiaSnapshot::from_nodes(&tree.nodes, kind)+ .with_read_while_minimized(crate::input::window_is_iconic(hwnd));(tree, payload, budget.outcome())});let started = std::time::Instant::now();@@ -3935,6 +3936,27 @@ impl Tool for ClickTool {&& count == 1&& crate::input::is_chromium_target_window(hwnd){+ // This route posts at the cached center without+ // `resolve_onscreen_point_with_scroll`, so check it here.+ if admitted+ .as_ref()+ .is_some_and(|element| element.read_while_minimized)+ {+ let message = format!(+ "Element [{idx}] resolves to ({cx},{cy}), which was read while \+ window 0x{hwnd:x} was minimized. Re-snapshot with get_window_state \+ before retrying."+ );+ return ToolResult::error(message.clone()).with_structured(json!({+ "status": "refused",+ "code": "element_not_visible",+ "effect": "refused",+ "refusal": {+ "code": "element_not_visible",+ "message": message,+ }+ }));+ }let posted = tokio::task::spawn_blocking({let admitted = admitted.clone();move || {diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rsindex 5f2e69a7ba..d58c62ab5c 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot.rs@@ -12,6 +12,7 @@ pub struct RetainedElement {pub center: (i32, i32),pub rect: Option<(i32, i32, i32, i32)>,pub msaa_role: Option<i32>,+ pub read_while_minimized: bool,}impl RetainedElement {@@ -72,6 +73,7 @@ impl Clone for RetainedElement {center: self.center,rect: self.rect,msaa_role: self.msaa_role,+ read_while_minimized: self.read_while_minimized,}}}@@ -111,10 +113,18 @@ impl UiaSnapshot {center: (node.center_x, node.center_y),rect: node.rect,msaa_role: node.msaa_role,+ read_while_minimized: false,}).collect(),}}++ pub fn with_read_while_minimized(mut self, minimized: bool) -> Self {+ for element in &mut self.elements {+ element.read_while_minimized = minimized;+ }+ self+ }}impl SnapshotPayload for UiaSnapshot {diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rsindex 276517bb69..68c4c542b7 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/snapshot_uaf_repro.rs@@ -114,6 +114,7 @@ fn snapshot_with(ptrs: Vec<usize>) -> UiaSnapshot {center: (0, 0),rect: None,msaa_role: None,+ read_while_minimized: false,}).collect(),}