Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): skip web content during Chrome consent scans (#5216)
injaneity3722b11a4fe4
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rsindex 1c5972a826..c6faae7192 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs@@ -86,17 +86,16 @@ whereoutcome = bounded.as_mut() => (None, outcome),}};+ let consent_timed_out = outcome.is_err();let outcome = match outcome {Ok(outcome) => outcome?,+ Err(_) if action.started() => consent.await.map_err(|mut error| {+ error+ .message+ .push_str(" (consent attempt deadline expired)");+ error+ })?,Err(_) => {- if action.started() {- consent.await.map_err(|mut error| {- error- .message- .push_str(" (consent attempt deadline expired)");- error- })?;- }return Err(refusal(BrowserRefusalCode::BrowserWrongTargetRefused,"browser consent did not settle within its bounded attempt",@@ -110,13 +109,19 @@ where"browser consent action did not produce a confirmed outcome",));}- Ok((- match result {- Some(result) => result,- None => claim.as_mut().await,- },- accepted,- ))+ let result = match result {+ Some(result) => result,+ None if consent_timed_out => tokio::time::timeout(Duration::from_secs(4), claim.as_mut())+ .await+ .map_err(|_| {+ refusal(+ BrowserRefusalCode::BrowserWrongTargetRefused,+ "browser reconnect did not settle after confirmed consent",+ )+ })?,+ None => claim.as_mut().await,+ };+ Ok((result, accepted))}async fn claim_with_delayed_consent<T, Claim, Consent, MakeConsent>(@@ -1615,7 +1620,7 @@ mod tests {let result = claim_with_optional_consent(&mut claim, &action, consent).await;assert!(dropped.load(Ordering::SeqCst));match outcome {- 0 => {+ 0 | 2 => {let (claim, displayed) = result.unwrap();assert_eq!(claim.is_ok(), claim_succeeds);assert!(displayed && settled.load(Ordering::SeqCst));@@ -1624,21 +1629,36 @@ mod tests {result.unwrap_err().code,BrowserRefusalCode::BrowserConsentRevoked),- _ => {- assert_eq!(- result.unwrap_err().code,- BrowserRefusalCode::BrowserWrongTargetRefused- );- assert!(- settled.load(Ordering::SeqCst),- "committed worker was detached on timeout"- );- }+ _ => unreachable!(),}}}}+ #[tokio::test]+ async fn late_confirmed_consent_allows_the_original_pending_claim_to_finish() {+ let action = BrowserConsentAction::default();+ let marker = action.clone();+ let (confirmed, awaiting_confirmation) = tokio::sync::oneshot::channel();+ let mut claim = Box::pin(async move {+ awaiting_confirmation.await.unwrap();+ tokio::time::sleep(Duration::from_millis(10)).await;+ Ok::<_, anyhow::Error>(7_u8)+ });+ let consent = async move {+ marker.perform(|| ());+ tokio::time::sleep(Duration::from_millis(4100)).await;+ confirmed.send(()).unwrap();+ Ok(BrowserConsentOutcome::Accepted)+ };++ let (result, displayed) = claim_with_optional_consent(&mut claim, &action, consent)+ .await+ .unwrap();+ assert_eq!(result.unwrap(), 7);+ assert!(displayed);+ }+#[tokio::test]async fn timed_out_consent_drains_before_pending_claim_is_dropped() {use std::sync::atomic::{AtomicBool, Ordering};diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rsindex cedbbe404d..681c636058 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs@@ -201,16 +201,42 @@ pub fn walk_tree_bounded()}+pub(crate) fn walk_native_chrome_bounded(+ pid: i32,+ window_id: u32,+ max_elements: usize,+) -> TreeWalkResult {+ walk_tree_with_web_content(+ pid,+ Some(window_id),+ None,+ DEFAULT_MAX_DEPTH,+ WalkBudget::nodes_only(max_elements),+ false,+ )+}+/// [`walk_tree_bounded`] under a caller's [`WalkBudget`]: the walk also stops/// when the budget's `timeout_ms` runs out, returning the partial tree. Each/// AX call is bounded by the per-element messaging timeout, so the walk/// overruns the deadline by at most one attribute read.pub fn walk_tree_budgeted(+ pid: i32,+ window_id: Option<u32>,+ query: Option<&str>,+ max_depth: usize,+ budget: WalkBudget,+) -> TreeWalkResult {+ walk_tree_with_web_content(pid, window_id, query, max_depth, budget, true)+}++fn walk_tree_with_web_content(pid: i32,window_id: Option<u32>,query: Option<&str>,max_depth: usize,mut budget: WalkBudget,+ include_web_content: bool,) -> TreeWalkResult {let mut nodes: Vec<AXNode> = Vec::new();let mut lines: Vec<(usize, String)> = Vec::new(); // (depth, line)@@ -297,6 +323,7 @@ pub fn walk_tree_budgeted(&mut index_counter,&mut budget,max_depth,+ include_web_content,);}@@ -412,6 +439,7 @@ unsafe fn walk_element(counter: &mut usize,budget: &mut WalkBudget,max_depth: usize,+ include_web_content: bool,) {if depth > max_depth {return;@@ -429,6 +457,9 @@ unsafe fn walk_element(let role = copy_string_attr(element, "AXRole").unwrap_or_else(|| "AXUnknown".into());let in_web_content = in_web_content || is_web_content_role(&role);+ if in_web_content && !include_web_content {+ return;+ }// Skip pure layout containers that have no interesting content.if role == "AXScrollArea" || role == "AXGroup" {@@ -447,6 +478,7 @@ unsafe fn walk_element(counter,budget,max_depth,+ include_web_content,);CFRelease(child as CFTypeRef);}@@ -513,6 +545,7 @@ unsafe fn walk_element(counter,budget,max_depth,+ include_web_content,);CFRelease(child as CFTypeRef);}@@ -648,6 +681,7 @@ unsafe fn walk_element(counter,budget,max_depth,+ include_web_content,);CFRelease(child as CFTypeRef);}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rsindex 57193da48f..82dac6171b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs@@ -9,13 +9,8 @@ use cua_driver_core::browser::{};use crate::ax::bindings::{kAXErrorSuccess, perform_action, AXUIElementRef};-use crate::ax::tree::{walk_tree_bounded, AXNode, TreeWalkResult, DEFAULT_MAX_DEPTH};+use crate::ax::tree::{walk_native_chrome_bounded, AXNode, TreeWalkResult};-// Large Chromium pages can put the browser-owned consent sheet after the-// ordinary 2,000-node snapshot cap. Keep this privileged scan bounded while-// allowing enough headroom to inspect Chrome's top-level sheet on pages such-// as Gmail. The matcher below still requires one exact AXSheet and one exact-// semantic Allow action before it will press anything.const CONSENT_MAX_ELEMENTS: usize = 5_000;fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefusal {@@ -214,17 +209,35 @@ impl Drop for ConsentTrees {fn read_consent_trees(pid: i32, window_id: u32) -> Result<ConsentTrees, BrowserRefusal> {let mut trees = ConsentTrees(Vec::new());for candidate in consent_surface_ids(crate::windows::all_windows(), pid, window_id) {- trees.push(walk_tree_bounded(+ trees.push(walk_native_chrome_bounded(pid,- Some(candidate),- None,+ candidate,CONSENT_MAX_ELEMENTS,- DEFAULT_MAX_DEPTH,))?;}Ok(trees)}+fn settled_after_scan(+ accepted_prompt: bool,+ prompt_present: bool,+ deadline_expired: bool,+ attempt: u8,+) -> Result<Option<BrowserConsentOutcome>, BrowserRefusal> {+ if accepted_prompt && !prompt_present {+ return Ok(Some(BrowserConsentOutcome::Accepted));+ }+ if deadline_expired {+ return Err(refusal(+ BrowserRefusalCode::BrowserWrongTargetRefused,+ format!(+ "Chrome remote-debugging consent did not settle for reconnect attempt {attempt}"+ ),+ ));+ }+ Ok(None)+}+/// Dismiss any exact Chrome-owned remote-debugging sheet before teardown./// Turning off the setting does not reliably close a sheet that an existing/// WebSocket connection already presented.@@ -309,20 +322,19 @@ pub async fn handle(format!("could not inspect the browser consent UI: {error}"),)})??;- if Instant::now() >= deadline {- return Err(refusal(- BrowserRefusalCode::BrowserWrongTargetRefused,- format!(- "Chrome remote-debugging consent did not settle for reconnect attempt {}",- request.attempt- ),- ));- }let prompt_present = trees.0.iter().any(|nodes| remote_debugging_sheet_present(nodes));saw_prompt |= prompt_present;+ if let Some(outcome) = settled_after_scan(+ accepted_prompt,+ prompt_present,+ Instant::now() >= deadline,+ request.attempt,+ )? {+ return Ok(outcome);+ }let mut candidates = Vec::new();for nodes in &trees.0 {if let Some(element) = exact_allow_button(nodes)? {@@ -358,9 +370,6 @@ pub async fn handle("multiple Chrome-owned remote-debugging consent sheets exposed semantic allow actions",));}- if accepted_prompt && !prompt_present {- return Ok(BrowserConsentOutcome::Accepted);- }if saw_prompt && !prompt_present {return Err(refusal(BrowserRefusalCode::BrowserConsentRevoked,@@ -375,6 +384,22 @@ pub async fn handle(mod tests {use super::*;+ #[test]+ fn confirmed_closed_sheet_wins_over_elapsed_deadline() {+ assert_eq!(+ settled_after_scan(true, false, true, 1).unwrap(),+ Some(BrowserConsentOutcome::Accepted)+ );+ assert_eq!(+ settled_after_scan(true, true, true, 1).unwrap_err().code,+ BrowserRefusalCode::BrowserWrongTargetRefused+ );+ assert_eq!(+ settled_after_scan(false, false, true, 1).unwrap_err().code,+ BrowserRefusalCode::BrowserWrongTargetRefused+ );+ }+fn node(role: &str, depth: usize, title: Option<&str>, actions: &[&str]) -> AXNode {AXNode {element_index: (!actions.is_empty()).then_some(0),