Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): harden existing browser attachment
trycua-release[bot]2ccb87c8a9cb
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rsindex fba6034f68..bad656150a 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs@@ -110,7 +110,20 @@ fn is_semantic_document_size_error(error: &anyhow::Error) -> bool {.any(|needle| message.contains(needle))}+fn is_semantic_document_fallback_error(error: &anyhow::Error) -> bool {+ if is_semantic_document_size_error(error) {+ return true;+ }+ is_semantic_document_timeout_error(error)+}++fn is_semantic_document_timeout_error(error: &anyhow::Error) -> bool {+ let message = error.to_string().to_ascii_lowercase();+ message.contains("cdp dom.getdocument timed out after")+}+const SEMANTIC_DOM_FALLBACK_DEPTHS: &[i64] = &[256, 128, 64, 32, 16, 8, 4, 2, 1];+const SEMANTIC_DOM_TIMEOUT_FALLBACK_DEPTHS: &[i64] = &[8, 4, 2, 1];const SEMANTIC_DOM_HYDRATION_DEPTH: i64 = 8;const MAX_SEMANTIC_DOM_HYDRATION_CALLS: usize = 64;const MAX_SEMANTIC_DOM_SCAN_NODES: usize = 50_000;@@ -1674,9 +1687,14 @@ impl BrowserEngine {.await{Ok(document) => Ok((document, true)),- Err(error) if is_semantic_document_size_error(&error) => {+ Err(error) if is_semantic_document_fallback_error(&error) => {let mut last_size_error = error.to_string();- for depth in SEMANTIC_DOM_FALLBACK_DEPTHS {+ let fallback_depths = if is_semantic_document_timeout_error(&error) {+ SEMANTIC_DOM_TIMEOUT_FALLBACK_DEPTHS+ } else {+ SEMANTIC_DOM_FALLBACK_DEPTHS+ };+ for depth in fallback_depths {match conn.call(Some(cdp_session),@@ -1691,7 +1709,7 @@ impl BrowserEngine {.await?;return Ok((document, false));}- Err(error) if is_semantic_document_size_error(&error) => {+ Err(error) if is_semantic_document_fallback_error(&error) => {last_size_error = error.to_string();}Err(error) => {diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rsindex 26050ffe32..ebfaa0862d 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs@@ -1,8 +1,10 @@//! Acting `browser_prepare` implementation for driver-owned Chromium profiles.use std::fs::{self, OpenOptions};+use std::future::Future;use std::io::Write;use std::path::{Path, PathBuf};+use std::pin::Pin;use std::process::{Child, Command, Stdio};use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};@@ -28,6 +30,24 @@ use super::BrowserEngine;const PROFILE_MARKER: &str = ".cua-driver-owned-profile.json";const PROFILE_SCHEMA: &str = "cua-driver-browser-profile-v1";+async fn claim_with_optional_consent<T, Claim, Consent>(+ claim: &mut Pin<Box<Claim>>,+ consent: Consent,+) -> Result<(anyhow::Result<T>, bool), BrowserRefusal>+where+ Claim: Future<Output = anyhow::Result<T>>,+ Consent: Future<Output = Result<BrowserConsentOutcome, BrowserRefusal>>,+{+ let mut consent = Box::pin(consent);+ tokio::select! {+ result = claim.as_mut() => Ok((result, false)),+ outcome = consent.as_mut() => match outcome? {+ BrowserConsentOutcome::Accepted => Ok((claim.as_mut().await, true)),+ BrowserConsentOutcome::NotPresent => Ok((claim.as_mut().await, false)),+ },+ }+}+fn with_setup_side_effects(mut error: BrowserRefusal,setup: &ExistingProfileSetupOutcome,@@ -858,17 +878,18 @@ impl BrowserEngine {if let Some(result) = initial {(result, false)} else {- match self- .platform- .handle_existing_profile_consent(BrowserConsentRequest {- pid: request.pid,- window_id,- attempt: 1,- })- .await+ match claim_with_optional_consent(+ &mut claim,+ self.platform+ .handle_existing_profile_consent(BrowserConsentRequest {+ pid: request.pid,+ window_id,+ attempt: 1,+ }),+ )+ .await{- Ok(BrowserConsentOutcome::Accepted) => (claim.await, true),- Ok(BrowserConsentOutcome::NotPresent) => (claim.await, false),+ Ok(result) => result,Err(error) => {// The connection future may hold the pool mutex while// awaiting its WebSocket handshake. Cancel it before@@ -973,6 +994,40 @@ impl BrowserEngine {mod tests {use super::*;+ #[tokio::test]+ async fn completed_claim_wins_while_optional_consent_is_absent() {+ let mut claim = Box::pin(async {+ tokio::time::sleep(Duration::from_millis(10)).await;+ Ok::<_, anyhow::Error>(7_u8)+ });+ let consent = async {+ tokio::time::sleep(Duration::from_secs(1)).await;+ Err(refusal(+ BrowserRefusalCode::BrowserWrongTargetRefused,+ "no consent surface",+ ))+ };+ let (result, displayed) = claim_with_optional_consent(&mut claim, consent)+ .await+ .expect("the completed claim should win the race");+ assert_eq!(result.unwrap(), 7);+ assert!(!displayed);+ }++ #[tokio::test]+ async fn accepted_consent_waits_for_the_same_claim() {+ let mut claim = Box::pin(async {+ tokio::time::sleep(Duration::from_millis(10)).await;+ Ok::<_, anyhow::Error>(9_u8)+ });+ let (result, displayed) =+ claim_with_optional_consent(&mut claim, async { Ok(BrowserConsentOutcome::Accepted) })+ .await+ .expect("accepted consent should resume the existing claim");+ assert_eq!(result.unwrap(), 9);+ assert!(displayed);+ }+#[test]fn setup_side_effects_are_preserved_on_refusal() {let error = with_setup_side_effects(diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex 6efcaa7b0c..2bd6f483da 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -48,6 +48,7 @@ struct FixtureState {completed_key_pairs: usize,semantic_large_page: bool,semantic_full_dom_fails: bool,+ semantic_full_dom_times_out: bool,semantic_truncated_dom: bool,/// Every incoming CDP call: (sessionId, method, params).calls: Vec<(Option<String>, String, Value)>,@@ -68,6 +69,7 @@ impl Default for FixtureState {completed_key_pairs: 0,semantic_large_page: false,semantic_full_dom_fails: false,+ semantic_full_dom_times_out: false,semantic_truncated_dom: false,calls: Vec::new(),}@@ -440,7 +442,9 @@ fn fixture_handler(state: SharedState) -> MockHandler {})),"DOM.getDocument" if is_tab => {let depth = call.params["depth"].as_i64().unwrap_or(-1);- if st.semantic_full_dom_fails && (depth == -1 || depth > 8) {+ if st.semantic_full_dom_times_out && (depth == -1 || depth > 8) {+ MockReply::err(-32000, "CDP DOM.getDocument timed out after 20s")+ } else if st.semantic_full_dom_fails && (depth == -1 || depth > 8) {MockReply::err(-32000, "Object reference chain is too long")} else if st.semantic_truncated_dom && depth == 8 {MockReply::ok(truncated_semantic_document())@@ -1392,6 +1396,29 @@ async fn semantic_snapshot_uses_bounded_dom_fallback_only_for_known_size_failure.any(|(_, params)| params["depth"] == 8));}+#[tokio::test]+async fn semantic_snapshot_uses_bounded_dom_fallback_for_full_tree_timeout() {+ let f = fixture_with(|st| {+ st.semantic_large_page = true;+ st.semantic_full_dom_times_out = true;+ })+ .await;+ let (target, tab) = bind(&f).await;+ let snap = semantic_snapshot_with(&f, &target, &tab, json!({"query": "Reply"})).await;+ assert_eq!(snap["status"], "ok", "{snap}");+ assert_eq!(snap["snapshot"]["complete"], false, "{snap}");+ let document_calls = recorded_calls(&f, "DOM.getDocument");+ assert!(document_calls+ .iter()+ .any(|(_, params)| params["depth"] == -1));+ assert!(document_calls+ .iter()+ .any(|(_, params)| params["depth"] == 8));+ assert!(document_calls.iter().all(|(_, params)| {+ params["depth"] == -1 || params["depth"].as_i64().is_some_and(|depth| depth <= 8)+ }));+}+#[tokio::test]async fn semantic_snapshot_hydrates_truncated_fallback_branches() {let f = fixture_with(|st| {@@ -1560,6 +1587,8 @@ async fn trusted_click_refuses_when_standalone_background_posture_is_unavailable.await;assert_eq!(structured(&synthetic)["status"], "ok");assert!(!recorded_calls(&f, "Runtime.callFunctionOn").is_empty());+ assert!(recorded_calls(&f, "Page.bringToFront").is_empty());+ assert!(recorded_calls(&f, "Target.activateTarget").is_empty());}#[tokio::test]@@ -1700,6 +1729,8 @@ async fn typing_into_composed_shadow_input_uses_the_tab_session() {!recorded_calls(&f, "Runtime.callFunctionOn").is_empty(),"editability must be checked on the node itself");+ assert!(recorded_calls(&f, "Page.bringToFront").is_empty());+ assert!(recorded_calls(&f, "Target.activateTarget").is_empty());}#[tokio::test]diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rsindex f33219b4ec..13761935e6 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs@@ -65,9 +65,9 @@ fn setup_page_proven(.iter().any(|title| field_equals(node, title))});- (exact_url || (trusted_navigation && !has_contradictory_address_bar))- && exact_page- && exact_heading+ let exact_identity =+ exact_url || (trusted_navigation && !has_contradictory_address_bar && exact_page);+ exact_identity && exact_heading}fn exact_setup_checkbox<'a>(@@ -479,6 +479,14 @@ mod tests {Some(false));+ let titleless = vec![nodes[0].clone(), nodes[2].clone(), nodes[3].clone()];+ assert!(+ exact_setup_checkbox(&titleless, descriptor(), false)+ .unwrap()+ .is_some(),+ "an exact internal URL and heading prove products that omit the document title from AT-SPI"+ );+let addressless = nodes[1..].to_vec();assert!(exact_setup_checkbox(&addressless, descriptor(), false)diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rsindex 0fe597da57..fe5b4822d4 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/consent_ui.rs@@ -9,7 +9,14 @@ use cua_driver_core::browser::{};use crate::ax::bindings::{kAXErrorSuccess, perform_action, AXUIElementRef};-use crate::ax::tree::{walk_tree, AXNode};+use crate::ax::tree::{walk_tree_bounded, AXNode, DEFAULT_MAX_DEPTH};++// Large Chromium pages can put the browser-owned consent sheet after the+// ordinary 2,000-node snapshot cap. Keep this privileged scan bounded while+// allowing enough headroom to inspect Chrome's top-level sheet on pages such+// as Gmail. The matcher below still requires one exact AXSheet and one exact+// semantic Allow action before it will press anything.+const CONSENT_MAX_ELEMENTS: usize = 5_000;fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefusal {BrowserRefusal::new(code, message)@@ -143,11 +150,21 @@ pub async fn handle(})?;let deadline = Instant::now() + Duration::from_secs(4);let mut saw_prompt = false;+ let mut accepted_prompt = false;loop {let trees = tokio::task::spawn_blocking(move || {consent_surface_ids(crate::windows::all_windows(), pid, window_id).into_iter()- .map(|candidate_window_id| walk_tree(pid, Some(candidate_window_id), None).nodes)+ .map(|candidate_window_id| {+ walk_tree_bounded(+ pid,+ Some(candidate_window_id),+ None,+ CONSENT_MAX_ELEMENTS,+ DEFAULT_MAX_DEPTH,+ )+ .nodes+ }).collect::<Vec<_>>()}).await@@ -192,7 +209,14 @@ pub async fn handle("the exact browser consent action became stale before AXPress",));}- return Ok(BrowserConsentOutcome::Accepted);+ // Chrome can queue more than one browser-owned consent sheet when+ // an earlier connection attempt was interrupted. Do not report+ // acceptance merely because AXPress returned success: keep+ // inspecting the exact approved process until every matching+ // sheet is gone, or the bounded deadline/ambiguity checks refuse.+ accepted_prompt = true;+ tokio::time::sleep(Duration::from_millis(100)).await;+ continue;}for nodes in &trees {release_actionable_nodes(nodes);@@ -203,6 +227,9 @@ pub async fn handle("multiple Chrome-owned remote-debugging consent sheets exposed semantic allow actions",));}+ if accepted_prompt && !prompt_present {+ return Ok(BrowserConsentOutcome::Accepted);+ }if saw_prompt && !prompt_present {return Err(refusal(BrowserRefusalCode::BrowserConsentRevoked,