Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): harden existing browser attachment

trycua-release[bot]2ccb87c8a9cb
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
index fba6034f68..bad656150a 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
@@ -110,7 +110,20 @@ fn is_semantic_document_size_error(error: &anyhow::Error) -> bool {
.any(|needle| message.contains(needle))
}
+fn is_semantic_document_fallback_error(error: &anyhow::Error) -> bool {
+ if is_semantic_document_size_error(error) {
+ return true;
+ }
+ is_semantic_document_timeout_error(error)
+}
+
+fn is_semantic_document_timeout_error(error: &anyhow::Error) -> bool {
+ let message = error.to_string().to_ascii_lowercase();
+ message.contains("cdp dom.getdocument timed out after")
+}
+
const SEMANTIC_DOM_FALLBACK_DEPTHS: &[i64] = &[256, 128, 64, 32, 16, 8, 4, 2, 1];
+const SEMANTIC_DOM_TIMEOUT_FALLBACK_DEPTHS: &[i64] = &[8, 4, 2, 1];
const SEMANTIC_DOM_HYDRATION_DEPTH: i64 = 8;
const MAX_SEMANTIC_DOM_HYDRATION_CALLS: usize = 64;
const MAX_SEMANTIC_DOM_SCAN_NODES: usize = 50_000;
@@ -1674,9 +1687,14 @@ impl BrowserEngine {
.await
{
Ok(document) => Ok((document, true)),
- Err(error) if is_semantic_document_size_error(&error) => {
+ Err(error) if is_semantic_document_fallback_error(&error) => {
let mut last_size_error = error.to_string();
- for depth in SEMANTIC_DOM_FALLBACK_DEPTHS {
+ let fallback_depths = if is_semantic_document_timeout_error(&error) {
+ SEMANTIC_DOM_TIMEOUT_FALLBACK_DEPTHS
+ } else {
+ SEMANTIC_DOM_FALLBACK_DEPTHS
+ };
+ for depth in fallback_depths {
match conn
.call(
Some(cdp_session),
@@ -1691,7 +1709,7 @@ impl BrowserEngine {
.await?;
return Ok((document, false));
}
- Err(error) if is_semantic_document_size_error(&error) => {
+ Err(error) if is_semantic_document_fallback_error(&error) => {
last_size_error = error.to_string();
}
Err(error) => {
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs
index 26050ffe32..ebfaa0862d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs
@@ -1,8 +1,10 @@
//! Acting `browser_prepare` implementation for driver-owned Chromium profiles.
use std::fs::{self, OpenOptions};
+use std::future::Future;
use std::io::Write;
use std::path::{Path, PathBuf};
+use std::pin::Pin;
use std::process::{Child, Command, Stdio};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
@@ -28,6 +30,24 @@ use super::BrowserEngine;
const PROFILE_MARKER: &str = ".cua-driver-owned-profile.json";
const PROFILE_SCHEMA: &str = "cua-driver-browser-profile-v1";
+async fn claim_with_optional_consent<T, Claim, Consent>(
+ claim: &mut Pin<Box<Claim>>,
+ consent: Consent,
+) -> Result<(anyhow::Result<T>, bool), BrowserRefusal>
+where
+ Claim: Future<Output = anyhow::Result<T>>,
+ Consent: Future<Output = Result<BrowserConsentOutcome, BrowserRefusal>>,
+{
+ let mut consent = Box::pin(consent);
+ tokio::select! {
+ result = claim.as_mut() => Ok((result, false)),
+ outcome = consent.as_mut() => match outcome? {
+ BrowserConsentOutcome::Accepted => Ok((claim.as_mut().await, true)),
+ BrowserConsentOutcome::NotPresent => Ok((claim.as_mut().await, false)),
+ },
+ }
+}
+
fn with_setup_side_effects(
mut error: BrowserRefusal,
setup: &ExistingProfileSetupOutcome,
@@ -858,17 +878,18 @@ impl BrowserEngine {
if let Some(result) = initial {
(result, false)
} else {
- match self
- .platform
- .handle_existing_profile_consent(BrowserConsentRequest {
- pid: request.pid,
- window_id,
- attempt: 1,
- })
- .await
+ match claim_with_optional_consent(
+ &mut claim,
+ self.platform
+ .handle_existing_profile_consent(BrowserConsentRequest {
+ pid: request.pid,
+ window_id,
+ attempt: 1,
+ }),
+ )
+ .await
{
- Ok(BrowserConsentOutcome::Accepted) => (claim.await, true),
- Ok(BrowserConsentOutcome::NotPresent) => (claim.await, false),
+ Ok(result) => result,
Err(error) => {
// The connection future may hold the pool mutex while
// awaiting its WebSocket handshake. Cancel it before
@@ -973,6 +994,40 @@ impl BrowserEngine {
mod tests {
use super::*;
+ #[tokio::test]
+ async fn completed_claim_wins_while_optional_consent_is_absent() {
+ let mut claim = Box::pin(async {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ Ok::<_, anyhow::Error>(7_u8)
+ });
+ let consent = async {
+ tokio::time::sleep(Duration::from_secs(1)).await;
+ Err(refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ "no consent surface",
+ ))
+ };
+ let (result, displayed) = claim_with_optional_consent(&mut claim, consent)
+ .await
+ .expect("the completed claim should win the race");
+ assert_eq!(result.unwrap(), 7);
+ assert!(!displayed);
+ }
+
+ #[tokio::test]
+ async fn accepted_consent_waits_for_the_same_claim() {
+ let mut claim = Box::pin(async {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ Ok::<_, anyhow::Error>(9_u8)
+ });
+ let (result, displayed) =
+ claim_with_optional_consent(&mut claim, async { Ok(BrowserConsentOutcome::Accepted) })
+ .await
+ .expect("accepted consent should resume the existing claim");
+ assert_eq!(result.unwrap(), 9);
+ assert!(displayed);
+ }
+
#[test]
fn setup_side_effects_are_preserved_on_refusal() {
let error = with_setup_side_effects(
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
index 6efcaa7b0c..2bd6f483da 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
@@ -48,6 +48,7 @@ struct FixtureState {
completed_key_pairs: usize,
semantic_large_page: bool,
semantic_full_dom_fails: bool,
+ semantic_full_dom_times_out: bool,
semantic_truncated_dom: bool,
/// Every incoming CDP call: (sessionId, method, params).
calls: Vec<(Option<String>, String, Value)>,
@@ -68,6 +69,7 @@ impl Default for FixtureState {
completed_key_pairs: 0,
semantic_large_page: false,
semantic_full_dom_fails: false,
+ semantic_full_dom_times_out: false,
semantic_truncated_dom: false,
calls: Vec::new(),
}
@@ -440,7 +442,9 @@ fn fixture_handler(state: SharedState) -> MockHandler {
})),
"DOM.getDocument" if is_tab => {
let depth = call.params["depth"].as_i64().unwrap_or(-1);
- if st.semantic_full_dom_fails && (depth == -1 || depth > 8) {
+ if st.semantic_full_dom_times_out && (depth == -1 || depth > 8) {
+ MockReply::err(-32000, "CDP DOM.getDocument timed out after 20s")
+ } else if st.semantic_full_dom_fails && (depth == -1 || depth > 8) {
MockReply::err(-32000, "Object reference chain is too long")
} else if st.semantic_truncated_dom && depth == 8 {
MockReply::ok(truncated_semantic_document())
@@ -1392,6 +1396,29 @@ async fn semantic_snapshot_uses_bounded_dom_fallback_only_for_known_size_failure
.any(|(_, params)| params["depth"] == 8));
}
+#[tokio::test]
+async fn semantic_snapshot_uses_bounded_dom_fallback_for_full_tree_timeout() {
+ let f = fixture_with(|st| {
+ st.semantic_large_page = true;
+ st.semantic_full_dom_times_out = true;
+ })
+ .await;
+ let (target, tab) = bind(&f).await;
+ let snap = semantic_snapshot_with(&f, &target, &tab, json!({"query": "Reply"})).await;
+ assert_eq!(snap["status"], "ok", "{snap}");
+ assert_eq!(snap["snapshot"]["complete"], false, "{snap}");
+ let document_calls = recorded_calls(&f, "DOM.getDocument");
+ assert!(document_calls
+ .iter()
+ .any(|(_, params)| params["depth"] == -1));
+ assert!(document_calls
+ .iter()
+ .any(|(_, params)| params["depth"] == 8));
+ assert!(document_calls.iter().all(|(_, params)| {
+ params["depth"] == -1 || params["depth"].as_i64().is_some_and(|depth| depth <= 8)
+ }));
+}
+
#[tokio::test]
async fn semantic_snapshot_hydrates_truncated_fallback_branches() {
let f = fixture_with(|st| {
@@ -1560,6 +1587,8 @@ async fn trusted_click_refuses_when_standalone_background_posture_is_unavailable
.await;
assert_eq!(structured(&synthetic)["status"], "ok");
assert!(!recorded_calls(&f, "Runtime.callFunctionOn").is_empty());
+ assert!(recorded_calls(&f, "Page.bringToFront").is_empty());
+ assert!(recorded_calls(&f, "Target.activateTarget").is_empty());
}
#[tokio::test]
@@ -1700,6 +1729,8 @@ async fn typing_into_composed_shadow_input_uses_the_tab_session() {
!recorded_calls(&f, "Runtime.callFunctionOn").is_empty(),
"editability must be checked on the node itself"
);
+ assert!(recorded_calls(&f, "Page.bringToFront").is_empty());
+ assert!(recorded_calls(&f, "Target.activateTarget").is_empty());
}
#[tokio::test]
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
index f33219b4ec..13761935e6 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
@@ -65,9 +65,9 @@ fn setup_page_proven(
.iter()
.any(|title| field_equals(node, title))
});
- (exact_url || (trusted_navigation && !has_contradictory_address_bar))
- && exact_page
- && exact_heading
+ let exact_identity =
+ exact_url || (trusted_navigation && !has_contradictory_address_bar && exact_page);
+ exact_identity && exact_heading
}
fn exact_setup_checkbox<'a>(
@@ -479,6 +479,14 @@ mod tests {
Some(false)
);
+ let titleless = vec![nodes[0].clone(), nodes[2].clone(), nodes[3].clone()];
+ assert!(
+ exact_setup_checkbox(&titleless, descriptor(), false)
+ .unwrap()
+ .is_some(),
+ "an exact internal URL and heading prove products that omit the document title from AT-SPI"
+ );
+
let addressless = nodes[1..].to_vec();
assert!(
exact_setup_checkbox(&addressless, descriptor(), false)
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
index 0fe597da57..fe5b4822d4 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs
@@ -9,7 +9,14 @@ use cua_driver_core::browser::{
};
use crate::ax::bindings::{kAXErrorSuccess, perform_action, AXUIElementRef};
-use crate::ax::tree::{walk_tree, AXNode};
+use crate::ax::tree::{walk_tree_bounded, AXNode, DEFAULT_MAX_DEPTH};
+
+// Large Chromium pages can put the browser-owned consent sheet after the
+// ordinary 2,000-node snapshot cap. Keep this privileged scan bounded while
+// allowing enough headroom to inspect Chrome's top-level sheet on pages such
+// as Gmail. The matcher below still requires one exact AXSheet and one exact
+// semantic Allow action before it will press anything.
+const CONSENT_MAX_ELEMENTS: usize = 5_000;
fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefusal {
BrowserRefusal::new(code, message)
@@ -143,11 +150,21 @@ pub async fn handle(
})?;
let deadline = Instant::now() + Duration::from_secs(4);
let mut saw_prompt = false;
+ let mut accepted_prompt = false;
loop {
let trees = tokio::task::spawn_blocking(move || {
consent_surface_ids(crate::windows::all_windows(), pid, window_id)
.into_iter()
- .map(|candidate_window_id| walk_tree(pid, Some(candidate_window_id), None).nodes)
+ .map(|candidate_window_id| {
+ walk_tree_bounded(
+ pid,
+ Some(candidate_window_id),
+ None,
+ CONSENT_MAX_ELEMENTS,
+ DEFAULT_MAX_DEPTH,
+ )
+ .nodes
+ })
.collect::<Vec<_>>()
})
.await
@@ -192,7 +209,14 @@ pub async fn handle(
"the exact browser consent action became stale before AXPress",
));
}
- return Ok(BrowserConsentOutcome::Accepted);
+ // Chrome can queue more than one browser-owned consent sheet when
+ // an earlier connection attempt was interrupted. Do not report
+ // acceptance merely because AXPress returned success: keep
+ // inspecting the exact approved process until every matching
+ // sheet is gone, or the bounded deadline/ambiguity checks refuse.
+ accepted_prompt = true;
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ continue;
}
for nodes in &trees {
release_actionable_nodes(nodes);
@@ -203,6 +227,9 @@ pub async fn handle(
"multiple Chrome-owned remote-debugging consent sheets exposed semantic allow actions",
));
}
+ if accepted_prompt && !prompt_present {
+ return Ok(BrowserConsentOutcome::Accepted);
+ }
if saw_prompt && !prompt_present {
return Err(refusal(
BrowserRefusalCode::BrowserConsentRevoked,