Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): stabilize background browser tabs

trycua-release[bot]3348fc5778e5
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs
index ff4dc4a0a7..2d03e82c13 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs
@@ -1855,7 +1855,6 @@ fn run_multi_tab(spec: &BrowserSpec) {
*evidence = recording_evidence(fixture.driver.recording_dir());
let session = format!("standalone-multi-tab-{}", fixture.pid);
let _ = bind(&mut fixture, &session);
- let first_target = cdp_target_for_url(fixture.cdp_port, fixture.server.page_url());
let second_server = BrowserFixtureServer::start(&standalone_fixture_html());
let created = harness_cdp_call(
fixture.cdp_port,
@@ -1863,16 +1862,39 @@ fn run_multi_tab(spec: &BrowserSpec) {
serde_json::json!({
"url": second_server.page_url(),
"newWindow": false,
+ "background": true,
}),
);
assert!(created["targetId"].is_string(), "{created}");
wait_for_observed(&second_server, "WEB_HARNESS_MARKER_v1");
- let activated = harness_cdp_call(
- fixture.cdp_port,
- "Target.activateTarget",
- serde_json::json!({"targetId": first_target}),
- );
- assert!(activated.is_object(), "{activated}");
+
+ // Begin the background proof only after Chromium has honored the
+ // explicit background-tab creation posture and the first fixture is
+ // observably still active.
+ let setup_deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ let state = fixture.driver.call(
+ "get_browser_state",
+ serde_json::json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.window_id,
+ "session": session,
+ }),
+ );
+ let first_is_active = state.structured()["tabs"]
+ .as_array()
+ .and_then(|tabs| tabs.iter().find(|tab| tab["active"] == true))
+ .is_some_and(|tab| tab["url"] == fixture.server.page_url());
+ if first_is_active {
+ break;
+ }
+ assert!(
+ Instant::now() < setup_deadline,
+ "fixture setup did not make the first tab observably active: {}",
+ state.raw
+ );
+ thread::sleep(Duration::from_millis(100));
+ }
run_with_background_oracles(&mut fixture, |fixture| {
let deadline = Instant::now() + Duration::from_secs(5);
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
index 13761935e6..c50ad37e33 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_setup_ui.rs
@@ -7,7 +7,7 @@ use std::{
};
use cua_driver_core::browser::{
- BrowserRefusal, BrowserRefusalCode, BrowserSetupDescriptor,
+ BrowserProduct, BrowserRefusal, BrowserRefusalCode, BrowserSetupDescriptor,
EXISTING_PROFILE_SETUP_READY_TIMEOUT,
};
@@ -219,6 +219,8 @@ pub struct SetupUiHandle {
window_id: u64,
descriptor: &'static BrowserSetupDescriptor,
trusted_setup_navigation: bool,
+ enable_attempted: bool,
+ trusted_checkbox_fallback_attempted: bool,
pub opened_setup_page: bool,
pub enabled_remote_debugging: bool,
pub focused_setup_address_field: bool,
@@ -364,6 +366,8 @@ pub fn enable(
window_id,
descriptor,
trusted_setup_navigation: false,
+ enable_attempted: false,
+ trusted_checkbox_fallback_attempted: false,
opened_setup_page: false,
enabled_remote_debugging: false,
focused_setup_address_field: false,
@@ -376,6 +380,8 @@ pub fn enable(
window_id,
descriptor,
trusted_setup_navigation: true,
+ enable_attempted: false,
+ trusted_checkbox_fallback_attempted: false,
opened_setup_page: true,
enabled_remote_debugging: false,
focused_setup_address_field: true,
@@ -399,8 +405,13 @@ pub fn enable(
let tree = crate::atspi::walk_tree(pid, window_id, None);
match exact_setup_checkbox(&tree.nodes, descriptor, handle.trusted_setup_navigation) {
Ok(Some(node)) => match node.checked {
- Some(true) => return Ok(handle),
- Some(false) if !handle.enabled_remote_debugging => {
+ Some(true) => {
+ if handle.enable_attempted {
+ handle.enabled_remote_debugging = true;
+ }
+ return Ok(handle);
+ }
+ Some(false) if !handle.enable_attempted => {
let index = node.element_index.expect("actionable checkbox index");
if let Err(error) = crate::atspi::perform_action(pid, index) {
return Err(handle.abort(refusal(
@@ -408,7 +419,68 @@ pub fn enable(
format!("the exact checkbox action failed: {error}"),
)));
}
- handle.enabled_remote_debugging = true;
+ handle.enable_attempted = true;
+ }
+ Some(false)
+ if descriptor.product == BrowserProduct::MicrosoftEdge
+ && !handle.trusted_checkbox_fallback_attempted =>
+ {
+ handle.trusted_checkbox_fallback_attempted = true;
+ handle.foregrounded_window = true;
+ let trusted_navigation = handle.trusted_setup_navigation;
+ let clicked = with_target_foreground(pid, window_id, || {
+ std::thread::sleep(Duration::from_millis(60));
+ let tree = crate::atspi::walk_tree(pid, window_id, None);
+ let checkbox = exact_setup_checkbox(
+ &tree.nodes,
+ descriptor,
+ trusted_navigation,
+ )
+ .map_err(|error| anyhow::anyhow!(error.message))?
+ .ok_or_else(|| {
+ anyhow::anyhow!(
+ "the exact Microsoft Edge remote-debugging checkbox became stale before the trusted click"
+ )
+ })?;
+ if checkbox.checked == Some(true) {
+ return Ok(false);
+ }
+ if checkbox.checked != Some(false) {
+ anyhow::bail!(
+ "the exact Microsoft Edge remote-debugging checkbox had an unknown state before the trusted click"
+ );
+ }
+ let index = checkbox.element_index.expect("actionable checkbox index");
+ let (x, y, width, height) = crate::atspi::get_element_bounds(pid, index)?;
+ if width <= 1 || height <= 1 {
+ anyhow::bail!(
+ "the exact Microsoft Edge remote-debugging checkbox had empty screen bounds"
+ );
+ }
+ let center_x = x
+ .checked_add(i32::try_from(width / 2)?)
+ .ok_or_else(|| anyhow::anyhow!("checkbox center x overflowed"))?;
+ let center_y = y
+ .checked_add(i32::try_from(height / 2)?)
+ .ok_or_else(|| anyhow::anyhow!("checkbox center y overflowed"))?;
+ if std::env::var_os("WAYLAND_DISPLAY").is_some() {
+ crate::wayland::click_desktop(center_x, center_y, 1, 1)?;
+ } else {
+ crate::input::send_click_xtest_desktop(center_x, center_y, 1, 1)?;
+ }
+ Ok(true)
+ });
+ match clicked {
+ Ok(injected) => handle.injected_global_input |= injected,
+ Err(error) => {
+ return Err(handle.abort(refusal(
+ BrowserRefusalCode::BrowserWrongTargetRefused,
+ format!(
+ "could not toggle the exact Microsoft Edge remote-debugging checkbox: {error}"
+ ),
+ )))
+ }
+ }
}
Some(false) => {}
None => {