Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): skip web content during Chrome consent scans (#5216)

injaneity3722b11a4fe4
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs
index 1c5972a826..c6faae7192 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs
@@ -86,17 +86,16 @@ where
outcome = bounded.as_mut() => (None, outcome),
}
};
+ let consent_timed_out = outcome.is_err();
let outcome = match outcome {
Ok(outcome) => outcome?,
+ Err(_) if action.started() => consent.await.map_err(|mut error| {
+ error
+ .message
+ .push_str(" (consent attempt deadline expired)");
+ error
+ })?,
Err(_) => {
- if action.started() {
- consent.await.map_err(|mut error| {
- error
- .message
- .push_str(" (consent attempt deadline expired)");
- error
- })?;
- }
return Err(refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
"browser consent did not settle within its bounded attempt",
@@ -110,13 +109,19 @@ where
"browser consent action did not produce a confirmed outcome",
));
}
- Ok((
- match result {
- Some(result) => result,
- None => claim.as_mut().await,
- },
- accepted,
- ))
+ let result = match result {
+ Some(result) => result,
+ None if consent_timed_out => tokio::time::timeout(Duration::from_secs(4), claim.as_mut())
+ .await
+ .map_err(|_| {
+ refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ "browser reconnect did not settle after confirmed consent",
+ )
+ })?,
+ None => claim.as_mut().await,
+ };
+ Ok((result, accepted))
}
async fn claim_with_delayed_consent<T, Claim, Consent, MakeConsent>(
@@ -1615,7 +1620,7 @@ mod tests {
let result = claim_with_optional_consent(&mut claim, &action, consent).await;
assert!(dropped.load(Ordering::SeqCst));
match outcome {
- 0 => {
+ 0 | 2 => {
let (claim, displayed) = result.unwrap();
assert_eq!(claim.is_ok(), claim_succeeds);
assert!(displayed && settled.load(Ordering::SeqCst));
@@ -1624,21 +1629,36 @@ mod tests {
result.unwrap_err().code,
BrowserRefusalCode::BrowserConsentRevoked
),
- _ => {
- assert_eq!(
- result.unwrap_err().code,
- BrowserRefusalCode::BrowserWrongTargetRefused
- );
- assert!(
- settled.load(Ordering::SeqCst),
- "committed worker was detached on timeout"
- );
- }
+ _ => unreachable!(),
}
}
}
}
+ #[tokio::test]
+ async fn late_confirmed_consent_allows_the_original_pending_claim_to_finish() {
+ let action = BrowserConsentAction::default();
+ let marker = action.clone();
+ let (confirmed, awaiting_confirmation) = tokio::sync::oneshot::channel();
+ let mut claim = Box::pin(async move {
+ awaiting_confirmation.await.unwrap();
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ Ok::<_, anyhow::Error>(7_u8)
+ });
+ let consent = async move {
+ marker.perform(|| ());
+ tokio::time::sleep(Duration::from_millis(4100)).await;
+ confirmed.send(()).unwrap();
+ Ok(BrowserConsentOutcome::Accepted)
+ };
+
+ let (result, displayed) = claim_with_optional_consent(&mut claim, &action, consent)
+ .await
+ .unwrap();
+ assert_eq!(result.unwrap(), 7);
+ assert!(displayed);
+ }
+
#[tokio::test]
async fn timed_out_consent_drains_before_pending_claim_is_dropped() {
use std::sync::atomic::{AtomicBool, Ordering};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
index cedbbe404d..681c636058 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
@@ -201,16 +201,42 @@ pub fn walk_tree_bounded(
)
}
+pub(crate) fn walk_native_chrome_bounded(
+ pid: i32,
+ window_id: u32,
+ max_elements: usize,
+) -> TreeWalkResult {
+ walk_tree_with_web_content(
+ pid,
+ Some(window_id),
+ None,
+ DEFAULT_MAX_DEPTH,
+ WalkBudget::nodes_only(max_elements),
+ false,
+ )
+}
+
/// [`walk_tree_bounded`] under a caller's [`WalkBudget`]: the walk also stops
/// when the budget's `timeout_ms` runs out, returning the partial tree. Each
/// AX call is bounded by the per-element messaging timeout, so the walk
/// overruns the deadline by at most one attribute read.
pub fn walk_tree_budgeted(
+ pid: i32,
+ window_id: Option<u32>,
+ query: Option<&str>,
+ max_depth: usize,
+ budget: WalkBudget,
+) -> TreeWalkResult {
+ walk_tree_with_web_content(pid, window_id, query, max_depth, budget, true)
+}
+
+fn walk_tree_with_web_content(
pid: i32,
window_id: Option<u32>,
query: Option<&str>,
max_depth: usize,
mut budget: WalkBudget,
+ include_web_content: bool,
) -> TreeWalkResult {
let mut nodes: Vec<AXNode> = Vec::new();
let mut lines: Vec<(usize, String)> = Vec::new(); // (depth, line)
@@ -297,6 +323,7 @@ pub fn walk_tree_budgeted(
&mut index_counter,
&mut budget,
max_depth,
+ include_web_content,
);
}
@@ -412,6 +439,7 @@ unsafe fn walk_element(
counter: &mut usize,
budget: &mut WalkBudget,
max_depth: usize,
+ include_web_content: bool,
) {
if depth > max_depth {
return;
@@ -429,6 +457,9 @@ unsafe fn walk_element(
let role = copy_string_attr(element, "AXRole").unwrap_or_else(|| "AXUnknown".into());
let in_web_content = in_web_content || is_web_content_role(&role);
+ if in_web_content && !include_web_content {
+ return;
+ }
// Skip pure layout containers that have no interesting content.
if role == "AXScrollArea" || role == "AXGroup" {
@@ -447,6 +478,7 @@ unsafe fn walk_element(
counter,
budget,
max_depth,
+ include_web_content,
);
CFRelease(child as CFTypeRef);
}
@@ -513,6 +545,7 @@ unsafe fn walk_element(
counter,
budget,
max_depth,
+ include_web_content,
);
CFRelease(child as CFTypeRef);
}
@@ -648,6 +681,7 @@ unsafe fn walk_element(
counter,
budget,
max_depth,
+ include_web_content,
);
CFRelease(child as CFTypeRef);
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
index 57193da48f..82dac6171b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
@@ -9,13 +9,8 @@ use cua_driver_core::browser::{
};
use crate::ax::bindings::{kAXErrorSuccess, perform_action, AXUIElementRef};
-use crate::ax::tree::{walk_tree_bounded, AXNode, TreeWalkResult, DEFAULT_MAX_DEPTH};
+use crate::ax::tree::{walk_native_chrome_bounded, AXNode, TreeWalkResult};
-// Large Chromium pages can put the browser-owned consent sheet after the
-// ordinary 2,000-node snapshot cap. Keep this privileged scan bounded while
-// allowing enough headroom to inspect Chrome's top-level sheet on pages such
-// as Gmail. The matcher below still requires one exact AXSheet and one exact
-// semantic Allow action before it will press anything.
const CONSENT_MAX_ELEMENTS: usize = 5_000;
fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefusal {
@@ -214,17 +209,35 @@ impl Drop for ConsentTrees {
fn read_consent_trees(pid: i32, window_id: u32) -> Result<ConsentTrees, BrowserRefusal> {
let mut trees = ConsentTrees(Vec::new());
for candidate in consent_surface_ids(crate::windows::all_windows(), pid, window_id) {
- trees.push(walk_tree_bounded(
+ trees.push(walk_native_chrome_bounded(
pid,
- Some(candidate),
- None,
+ candidate,
CONSENT_MAX_ELEMENTS,
- DEFAULT_MAX_DEPTH,
))?;
}
Ok(trees)
}
+fn settled_after_scan(
+ accepted_prompt: bool,
+ prompt_present: bool,
+ deadline_expired: bool,
+ attempt: u8,
+) -> Result<Option<BrowserConsentOutcome>, BrowserRefusal> {
+ if accepted_prompt && !prompt_present {
+ return Ok(Some(BrowserConsentOutcome::Accepted));
+ }
+ if deadline_expired {
+ return Err(refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ format!(
+ "Chrome remote-debugging consent did not settle for reconnect attempt {attempt}"
+ ),
+ ));
+ }
+ Ok(None)
+}
+
/// Dismiss any exact Chrome-owned remote-debugging sheet before teardown.
/// Turning off the setting does not reliably close a sheet that an existing
/// WebSocket connection already presented.
@@ -309,20 +322,19 @@ pub async fn handle(
format!("could not inspect the browser consent UI: {error}"),
)
})??;
- if Instant::now() >= deadline {
- return Err(refusal(
- BrowserRefusalCode::BrowserWrongTargetRefused,
- format!(
- "Chrome remote-debugging consent did not settle for reconnect attempt {}",
- request.attempt
- ),
- ));
- }
let prompt_present = trees
.0
.iter()
.any(|nodes| remote_debugging_sheet_present(nodes));
saw_prompt |= prompt_present;
+ if let Some(outcome) = settled_after_scan(
+ accepted_prompt,
+ prompt_present,
+ Instant::now() >= deadline,
+ request.attempt,
+ )? {
+ return Ok(outcome);
+ }
let mut candidates = Vec::new();
for nodes in &trees.0 {
if let Some(element) = exact_allow_button(nodes)? {
@@ -358,9 +370,6 @@ pub async fn handle(
"multiple Chrome-owned remote-debugging consent sheets exposed semantic allow actions",
));
}
- if accepted_prompt && !prompt_present {
- return Ok(BrowserConsentOutcome::Accepted);
- }
if saw_prompt && !prompt_present {
return Err(refusal(
BrowserRefusalCode::BrowserConsentRevoked,
@@ -375,6 +384,22 @@ pub async fn handle(
mod tests {
use super::*;
+ #[test]
+ fn confirmed_closed_sheet_wins_over_elapsed_deadline() {
+ assert_eq!(
+ settled_after_scan(true, false, true, 1).unwrap(),
+ Some(BrowserConsentOutcome::Accepted)
+ );
+ assert_eq!(
+ settled_after_scan(true, true, true, 1).unwrap_err().code,
+ BrowserRefusalCode::BrowserWrongTargetRefused
+ );
+ assert_eq!(
+ settled_after_scan(false, false, true, 1).unwrap_err().code,
+ BrowserRefusalCode::BrowserWrongTargetRefused
+ );
+ }
+
fn node(role: &str, depth: usize, title: Option<&str>, actions: &[&str]) -> AXNode {
AXNode {
element_index: (!actions.is_empty()).then_some(0),