Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep browser attestation off the renderer while a dialog is open (#5524)
injaneityc29740b37ea5
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rsindex c401f20b43..acf7bba605 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs@@ -67,6 +67,7 @@ const EXISTING_PROFILE_METHODS: &[&str] = &["Target.activateTarget","Target.attachToTarget","Target.detachFromTarget",+ "Target.getTargetInfo","Target.getTargets","Target.setAutoAttach",];@@ -355,6 +356,19 @@ impl CdpConnection {session_id: Option<&str>,method: &str,params: Value,+ ) -> anyhow::Result<Value> {+ self.call_with_timeout(session_id, method, params, CALL_TIMEOUT)+ .await+ }++ /// [`Self::call`] with a caller-chosen reply deadline, for best-effort+ /// commands that must not hold up the caller when the renderer is blocked.+ pub async fn call_with_timeout(+ &self,+ session_id: Option<&str>,+ method: &str,+ params: Value,+ timeout: Duration,) -> anyhow::Result<Value> {let policy = self.method_policy();if !policy.allows(method) {@@ -389,10 +403,10 @@ impl CdpConnection {anyhow::bail!("CDP send failed during {method}: {e}");}- match tokio::time::timeout(CALL_TIMEOUT, rx).await {+ match tokio::time::timeout(timeout, rx).await {Err(_) => {self.demux.pending.lock().unwrap().remove(&id);- anyhow::bail!("CDP {method} timed out after {CALL_TIMEOUT:?}")+ anyhow::bail!("CDP {method} timed out after {timeout:?}")}Ok(Err(_)) => anyhow::bail!("CDP socket closed during {method}"),Ok(Ok(CallOutcome::Result(v))) => Ok(v),diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rsindex 73f9e92ef7..b1433a2ab4 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs@@ -73,6 +73,9 @@ pub const MAX_REFS_PER_SNAPSHOT: usize = 300;/// compromised or malformed endpoint before its response reaches consumers.const MAX_BROWSER_SCREENSHOT_BYTES: usize = 16 * 1024 * 1024;+/// Reply deadline for the best-effort `Page.enable` sent on a tab's first attach.+const DIALOG_SUBSCRIPTION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);+pub struct BrowserEngine {pub(crate) platform: Arc<dyn BrowserPlatform>,pub(crate) store: BrowserStore,@@ -1259,7 +1262,7 @@ impl BrowserEngine {format!("cannot attach to tab target {cdp_target_id}: {e}"),)})?;- attached+ let cdp_session = attached.get("sessionId").and_then(Value::as_str).map(str::to_owned)@@ -1268,7 +1271,39 @@ impl BrowserEngine {BrowserRefusalCode::BrowserTabNotFound,format!("attach to {cdp_target_id} returned no sessionId"),)- })+ })?;+ self.subscribe_dialog_events(conn, &cdp_session, cdp_target_id)+ .await;+ Ok(cdp_session)+ }++ /// Chromium reports a JavaScript dialog only to sessions that enabled the+ /// Page domain before it opened, so the first session attached to a tab+ /// subscribes on behalf of every later caller; tab sessions stay attached+ /// for the life of the connection. A renderer already blocked by a dialog+ /// nobody has seen cannot answer `Page.enable`, hence the short deadline:+ /// the subscription is best effort and `browser_dialog` retries it.+ async fn subscribe_dialog_events(+ &self,+ conn: &CdpConnection,+ cdp_session: &str,+ cdp_target_id: &str,+ ) {+ if conn.dialog_state(cdp_target_id).is_some() || conn.has_dialog_session(cdp_target_id) {+ return;+ }+ conn.register_dialog_session(cdp_session, cdp_target_id);+ let enabled = conn+ .call_with_timeout(+ Some(cdp_session),+ "Page.enable",+ json!({}),+ DIALOG_SUBSCRIPTION_TIMEOUT,+ )+ .await;+ if enabled.is_err() && conn.dialog_state(cdp_target_id).is_none() {+ conn.unregister_dialog_session(cdp_session, cdp_target_id);+ }}// ── Binding (get_browser_state, pid + window_id mode) ──────────────@@ -1644,7 +1679,9 @@ impl BrowserEngine {.as_deref().is_some_and(|context| context.capability_manifest().is_some()){- let live_url = self.live_top_level_url(&conn, &cdp_session).await?;+ let live_url = self+ .live_top_level_url(&conn, &cdp_session, &tab.cdp_target_id)+ .await?;// A browser mutation admitted for a delegated session must use// that exact session's capability manifest. Falling back to the// process compatibility manifest would let a missing task-local@@ -1669,19 +1706,38 @@ impl BrowserEngine {})}+ /// The URL of the tab's live top-level document.+ ///+ /// The frame tree is the renderer's answer and the usual proof. While a+ /// JavaScript dialog is open the renderer is blocked and `Page.getFrameTree`+ /// never replies, so the browser process's own record of the target is+ /// used instead: the document cannot navigate until the dialog is handled.async fn live_top_level_url(&self,conn: &CdpConnection,cdp_session: &str,+ cdp_target_id: &str,) -> Result<String, BrowserRefusal> {- let frame_tree = conn- .call(Some(cdp_session), "Page.getFrameTree", json!({}))- .await- .map_err(|error| {- route_err("could not prove the live top-level browser document", error)- })?;- frame_tree- .pointer("/frameTree/frame/url")+ let (method, params, session, pointer) = if conn.dialog_state(cdp_target_id).is_some() {+ (+ "Target.getTargetInfo",+ json!({ "targetId": cdp_target_id }),+ None,+ "/targetInfo/url",+ )+ } else {+ (+ "Page.getFrameTree",+ json!({}),+ Some(cdp_session),+ "/frameTree/frame/url",+ )+ };+ let reply = conn.call(session, method, params).await.map_err(|error| {+ route_err("could not prove the live top-level browser document", error)+ })?;+ reply+ .pointer(pointer).and_then(Value::as_str).filter(|url| !url.is_empty()).map(str::to_owned)@@ -1703,7 +1759,11 @@ impl BrowserEngine {.revalidate_for_mutation(session, target_id, Some(tab_id)).await?;let live_url = self- .live_top_level_url(&validated.conn, &validated.cdp_session)+ .live_top_level_url(+ &validated.conn,+ &validated.cdp_session,+ &validated.tab.cdp_target_id,+ ).await?;let live_origin = protected_live_origin_scope(&live_url)?;Ok((validated, live_origin))diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex 67410543f9..89eaf9223b 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -79,6 +79,9 @@ pub(super) struct FixtureState {omit_primary_target: bool,/// Make `Browser.getWindowForTarget` for T1 answer this CDP error.primary_window_error: Option<(i64, String)>,+ /// A page-owned JavaScript dialog is open in T1: `Page.enable` on a tab+ /// session announces it, and the renderer answers nothing else.+ dialog_open: bool,dom_click_changes_readback: bool,dom_click_readback_available: bool,dom_click_stall_after: Option<usize>,@@ -102,6 +105,7 @@ impl Default for FixtureState {sibling_tabs: Vec::new(),omit_primary_target: false,primary_window_error: None,+ dialog_open: false,main_loader: "L_MAIN_1".into(),iframe_loader: "L_IFRAME_1".into(),oopif_loader: "L_OOPIF_1".into(),@@ -506,6 +510,21 @@ fn fixture_handler(state: SharedState) -> MockHandler {st.tab_sessions += 1;MockReply::ok(json!({ "sessionId": format!("tab-sess-{}", st.tab_sessions) }))}+ "Page.enable" if is_tab => {+ let mut reply = MockReply::ok(json!({}));+ if st.dialog_open {+ reply = reply.with_events(vec![MockEvent {+ method: "Page.javascriptDialogOpening".into(),+ params: json!({ "type": "alert", "message": "hi", "url": st.main_url }),+ session_id: Some(sess.clone()),+ }]);+ }+ reply+ }+ "Page.getFrameTree" if is_tab && st.dialog_open => MockReply::err(+ -32000,+ "the renderer is blocked by a dialog and never answers",+ ),"Page.getFrameTree" if is_tab => MockReply::ok(json!({"frameTree": {"frame": {@@ -2760,6 +2779,34 @@ async fn protected_browser_scope_reproves_live_origin_and_omits_sensitive_url_te);}+#[tokio::test]+async fn an_open_dialog_moves_the_origin_proof_off_the_blocked_renderer() {+ let f = fixture_with(|st| st.dialog_open = true).await;+ let (target, tab) = bind(&f).await;+ let args = json!({ "target_id": target, "tab_id": tab, "session": SESSION, "x": 10, "y": 20 });++ let scope = browser_protected_resource_scope(&f.engine, &args, "browser_click")+ .await+ .unwrap()+ .unwrap();+ assert_eq!(scope["live_origin"], "https://fixture.test");+ assert!(+ recorded_calls(&f, "Page.getFrameTree").is_empty(),+ "a renderer blocked by a dialog must not be asked for its frame tree"+ );+ assert_eq!(recorded_calls(&f, "Target.getTargetInfo").len(), 1);++ browser_protected_resource_scope(&f.engine, &args, "browser_click")+ .await+ .unwrap()+ .unwrap();+ assert_eq!(+ recorded_calls(&f, "Page.enable").len(),+ 1,+ "the first tab session subscribes to dialog events for every later caller"+ );+}+#[tokio::test]async fn a_stale_binding_keeps_its_refusal_code_through_scope_attestation() {use crate::authorization::PermissionMode;