Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): clean up unresolved Chrome consent UI (#3468)

Francesco Bonaccie7295472e196
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
index 261139794e..603f2ae4ec 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs
@@ -20,8 +20,182 @@ use cua_driver_core::browser::types::{
EndpointOwnershipMethod, EndpointOwnershipProof, EndpointTransport, NativeOwnershipMethod,
NativeOwnershipProof, NativeWindowInfo, OwnedEndpoint, ProcessFingerprint, Rect,
};
+use serde::Serialize;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
+const CONSENT_CLEANUP_PASSES: usize = 20;
+
+#[derive(Clone, Debug, Serialize)]
+struct ConsentSurfaceEvidence {
+ window_id: u32,
+ pid: i32,
+ bounds: [f64; 4],
+}
+
+fn consent_surface_evidence(
+ windows: impl IntoIterator<Item = crate::windows::WindowInfo>,
+ pid: i32,
+) -> Vec<ConsentSurfaceEvidence> {
+ let mut surfaces = windows
+ .into_iter()
+ .filter(|window| {
+ window.pid == pid
+ && window.title == "Allow remote debugging?"
+ && window.is_on_screen
+ && window.bounds.width > 0.0
+ && window.bounds.height > 0.0
+ })
+ .map(|window| ConsentSurfaceEvidence {
+ window_id: window.window_id,
+ pid: window.pid,
+ bounds: [
+ window.bounds.x,
+ window.bounds.y,
+ window.bounds.width,
+ window.bounds.height,
+ ],
+ })
+ .collect::<Vec<_>>();
+ surfaces.sort_by_key(|surface| surface.window_id);
+ surfaces
+}
+
+fn visible_consent_surfaces(pid: i32) -> Vec<ConsentSurfaceEvidence> {
+ consent_surface_evidence(crate::windows::all_windows(), pid)
+}
+
+fn add_consent_cleanup_detail(
+ mut error: BrowserRefusal,
+ cleanup: serde_json::Value,
+) -> BrowserRefusal {
+ let prior = error.detail.take();
+ error.detail = Some(match prior {
+ Some(serde_json::Value::Object(mut detail)) => {
+ detail.insert("consent_cleanup".to_owned(), cleanup);
+ serde_json::Value::Object(detail)
+ }
+ Some(cause) => serde_json::json!({
+ "cause": cause,
+ "consent_cleanup": cleanup,
+ }),
+ None => serde_json::json!({ "consent_cleanup": cleanup }),
+ });
+ error
+}
+
+fn run_failed_prepare_consent_cleanup<D, R, O, P>(
+ error: BrowserRefusal,
+ max_passes: usize,
+ mut dismiss: D,
+ rollback: R,
+ mut observe: O,
+ mut pause: P,
+) -> BrowserRefusal
+where
+ D: FnMut() -> Result<bool, BrowserRefusal>,
+ R: FnOnce(BrowserRefusal) -> BrowserRefusal,
+ O: FnMut() -> Vec<ConsentSurfaceEvidence>,
+ P: FnMut(),
+{
+ let mut dismissed_via_ax = false;
+ let mut dismiss_error = None;
+ match dismiss() {
+ Ok(dismissed) => dismissed_via_ax |= dismissed,
+ Err(error) => dismiss_error = Some(error),
+ }
+
+ // The retained setup handle is the exact authorization to undo the
+ // setting change. Always run it even when consent UI is AX-ambiguous.
+ let error = rollback(error);
+ let mut retained_surfaces = Vec::new();
+ for pass in 0..max_passes {
+ match dismiss() {
+ Ok(dismissed) => {
+ dismissed_via_ax |= dismissed;
+ dismiss_error = None;
+ }
+ Err(error) => dismiss_error = Some(error),
+ }
+ retained_surfaces = observe();
+ if pass + 1 < max_passes {
+ pause();
+ }
+ }
+
+ let status = if retained_surfaces.is_empty() {
+ "dismissed"
+ } else {
+ "retained"
+ };
+ add_consent_cleanup_detail(
+ error,
+ serde_json::json!({
+ "status": status,
+ "dismissed_via_ax": dismissed_via_ax,
+ "retained_surfaces": retained_surfaces,
+ "dismiss_error": dismiss_error,
+ }),
+ )
+}
+
+fn run_committed_consent_cleanup<D, C, O, P>(
+ max_passes: usize,
+ mut dismiss: D,
+ cleanup_setup: C,
+ mut observe: O,
+ mut pause: P,
+) -> Result<bool, BrowserRefusal>
+where
+ D: FnMut() -> Result<bool, BrowserRefusal>,
+ C: FnOnce() -> Result<bool, BrowserRefusal>,
+ O: FnMut() -> Vec<ConsentSurfaceEvidence>,
+ P: FnMut(),
+{
+ let mut changed = false;
+ let mut dismiss_error = None;
+ match dismiss() {
+ Ok(dismissed) => changed |= dismissed,
+ Err(error) => dismiss_error = Some(error),
+ }
+ let cleanup_result = cleanup_setup();
+ let cleanup_error = cleanup_result.as_ref().err().cloned();
+ if let Ok(cleaned) = cleanup_result.as_ref() {
+ changed |= *cleaned;
+ }
+
+ let mut retained_surfaces = Vec::new();
+ for pass in 0..max_passes {
+ match dismiss() {
+ Ok(dismissed) => {
+ changed |= dismissed;
+ dismiss_error = None;
+ }
+ Err(error) => dismiss_error = Some(error),
+ }
+ retained_surfaces = observe();
+ if pass + 1 < max_passes {
+ pause();
+ }
+ }
+
+ if !retained_surfaces.is_empty() {
+ let mut error = cleanup_error.unwrap_or_else(|| {
+ refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ "remote-debugging cleanup retained browser consent UI",
+ )
+ });
+ error.detail = Some(serde_json::json!({
+ "status": "retained",
+ "retained_surfaces": retained_surfaces,
+ "dismiss_error": dismiss_error,
+ }));
+ return Err(error);
+ }
+
+ cleanup_result.map(|_| changed)
+}
+
#[derive(Clone)]
pub struct MacOsBrowserPlatform {
cursor_registry: Arc<crate::cursor::CursorRegistry>,
@@ -1112,10 +1286,13 @@ impl BrowserPlatform for MacOsBrowserPlatform {
"the approved browser window is outside the macOS window-id range",
)
})?;
- let dismissed_before = super::consent_ui::dismiss(pid, window_id)?;
- let closed_setup_page = super::setup_ui::disable(pid, window_id, descriptor)?;
- let dismissed_after = super::consent_ui::dismiss(pid, window_id)?;
- Ok(dismissed_before || closed_setup_page || dismissed_after)
+ run_committed_consent_cleanup(
+ CONSENT_CLEANUP_PASSES,
+ || super::consent_ui::dismiss(pid, window_id),
+ || super::setup_ui::disable(pid, window_id, descriptor),
+ || visible_consent_surfaces(pid),
+ || std::thread::sleep(Duration::from_millis(100)),
+ )
}
async fn abort_existing_profile_setup(
@@ -1129,14 +1306,23 @@ impl BrowserPlatform for MacOsBrowserPlatform {
let Ok(window_id) = u32::try_from(request.window_id) else {
return error;
};
- tokio::task::spawn_blocking(move || super::setup_ui::abort_pending(pid, window_id, error))
- .await
- .unwrap_or_else(|join_error| {
- refusal(
- BrowserRefusalCode::BrowserRouteUnavailable,
- format!("could not roll back exact browser setup: {join_error}"),
- )
- })
+ tokio::task::spawn_blocking(move || {
+ run_failed_prepare_consent_cleanup(
+ error,
+ CONSENT_CLEANUP_PASSES,
+ || super::consent_ui::dismiss(pid, window_id),
+ |error| super::setup_ui::abort_pending(pid, window_id, error),
+ || visible_consent_surfaces(pid),
+ || std::thread::sleep(Duration::from_millis(100)),
+ )
+ })
+ .await
+ .unwrap_or_else(|join_error| {
+ refusal(
+ BrowserRefusalCode::BrowserRouteUnavailable,
+ format!("could not roll back exact browser setup: {join_error}"),
+ )
+ })
}
async fn handle_existing_profile_consent(
@@ -1186,6 +1372,160 @@ impl BrowserPlatform for MacOsBrowserPlatform {
mod tests {
use super::*;
+ fn consent_surface(window_id: u32) -> ConsentSurfaceEvidence {
+ ConsentSurfaceEvidence {
+ window_id,
+ pid: 42,
+ bounds: [120.0, 120.0, 520.0, 260.0],
+ }
+ }
+
+ #[test]
+ fn consent_cleanup_evidence_requires_exact_pid_title_visibility_and_bounds() {
+ let exact = window(90, 42, "Allow remote debugging?");
+ let mut foreign = window(91, 7, "Allow remote debugging?");
+ foreign.app_name = "Google Chrome".to_owned();
+ let mut hidden = window(92, 42, "Allow remote debugging?");
+ hidden.is_on_screen = false;
+ let mut empty = window(93, 42, "Allow remote debugging?");
+ empty.bounds.width = 0.0;
+
+ let surfaces = consent_surface_evidence(
+ [exact, foreign, hidden, empty, window(94, 42, "Unrelated")],
+ 42,
+ );
+ assert_eq!(surfaces.len(), 1);
+ assert_eq!(surfaces[0].window_id, 90);
+ assert_eq!(surfaces[0].pid, 42);
+ }
+
+ #[test]
+ fn failed_prepare_cleanup_drains_stacked_and_late_consent_surfaces() {
+ let events = std::cell::RefCell::new(Vec::new());
+ let mut observations = vec![
+ vec![consent_surface(90), consent_surface(91)],
+ vec![consent_surface(92)],
+ Vec::new(),
+ Vec::new(),
+ ]
+ .into_iter();
+ let error = run_failed_prepare_consent_cleanup(
+ refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ "prepare failed",
+ ),
+ 4,
+ || {
+ events.borrow_mut().push("dismiss");
+ Ok(false)
+ },
+ |error| {
+ events.borrow_mut().push("rollback");
+ error
+ },
+ || observations.next().unwrap_or_default(),
+ || {},
+ );
+
+ assert_eq!(
+ *events.borrow(),
+ ["dismiss", "rollback", "dismiss", "dismiss", "dismiss", "dismiss"]
+ );
+ assert_eq!(
+ error.detail.as_ref().unwrap()["consent_cleanup"]["status"],
+ "dismissed"
+ );
+ assert_eq!(
+ error.detail.as_ref().unwrap()["consent_cleanup"]["retained_surfaces"],
+ serde_json::json!([])
+ );
+ }
+
+ #[test]
+ fn failed_prepare_cleanup_returns_bounded_evidence_for_retained_surfaces() {
+ let mut rollback_calls = 0;
+ let error = run_failed_prepare_consent_cleanup(
+ refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ "prepare failed",
+ ),
+ 3,
+ || {
+ Err(refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ "ambiguous cancel actions",
+ ))
+ },
+ |error| {
+ rollback_calls += 1;
+ error
+ },
+ || vec![consent_surface(90), consent_surface(91)],
+ || {},
+ );
+
+ assert_eq!(
+ rollback_calls, 1,
+ "ambiguity must not skip exact setup rollback"
+ );
+ assert_eq!(
+ error.detail.as_ref().unwrap()["consent_cleanup"]["status"],
+ "retained"
+ );
+ assert_eq!(
+ error.detail.as_ref().unwrap()["consent_cleanup"]["retained_surfaces"]
+ .as_array()
+ .unwrap()
+ .len(),
+ 2
+ );
+ assert_eq!(
+ error.detail.as_ref().unwrap()["consent_cleanup"]["dismiss_error"]["message"],
+ "ambiguous cancel actions"
+ );
+ }
+
+ #[test]
+ fn committed_cleanup_refuses_while_an_unresolved_consent_surface_is_retained() {
+ let result = run_committed_consent_cleanup(
+ 3,
+ || Ok(false),
+ || Ok(true),
+ || vec![consent_surface(90)],
+ || {},
+ );
+
+ let error = result.expect_err("retained consent UI must retain the core cleanup request");
+ assert_eq!(error.code, BrowserRefusalCode::BrowserWrongTargetRefused);
+ assert_eq!(error.detail.as_ref().unwrap()["status"], "retained");
+ assert_eq!(
+ error.detail.as_ref().unwrap()["retained_surfaces"][0]["window_id"],
+ 90
+ );
+ }
+
+ #[test]
+ fn committed_cleanup_observes_the_full_window_for_late_surfaces() {
+ let mut observations = vec![
+ Vec::new(),
+ Vec::new(),
+ Vec::new(),
+ vec![consent_surface(90)],
+ Vec::new(),
+ Vec::new(),
+ ]
+ .into_iter();
+ let result = run_committed_consent_cleanup(
+ 6,
+ || Ok(false),
+ || Ok(true),
+ || observations.next().unwrap_or_default(),
+ || {},
+ );
+
+ assert_eq!(result.unwrap(), true);
+ }
+
#[test]
fn isolated_browser_candidates_are_vendor_attested_system_installs() {
let candidates = isolated_browser_candidates();