Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

test(cua-driver): attach before the background oracles on macOS, then check the same-Space refusal (#5063)

Francesco Bonacci88602d15650a
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
index 59806451eb..888d432462 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
@@ -349,6 +349,9 @@ struct BrowserFixture {
cdp_port: u16,
pid: u32,
window_id: u64,
+ /// Sessions attached by `prepare_same_space_attachment` before the
+ /// background oracles started (macOS only).
+ attached_before_observation: Vec<String>,
}
fn browser_specs() -> Vec<BrowserSpec> {
@@ -1290,6 +1293,7 @@ fn launch_browser_with_driver(
cdp_port,
pid,
window_id,
+ attached_before_observation: Vec::new(),
}
}
@@ -1326,6 +1330,7 @@ fn launch_unprepared_browser(spec: &BrowserSpec, label: &str) -> BrowserFixture
cdp_port: 0,
pid,
window_id,
+ attached_before_observation: Vec::new(),
}
}
@@ -1434,11 +1439,30 @@ fn semantic_ref_by_name(snapshot: &ToolResponse, name: &str, action: &str) -> St
.to_owned()
}
-fn bind(fixture: &mut BrowserFixture, session: &str) -> (String, String, ToolResponse) {
+/// On macOS, attaching to an existing Chrome or Edge profile shows Chrome's
+/// "Allow remote debugging?" prompt, which brings the browser forward. When
+/// the browser shares the user's Space, the driver therefore attaches only
+/// while the browser is already in front (or with
+/// `delivery_mode:"foreground"`), and refuses a background attachment with
+/// `background_unavailable`. Attach here, before the background oracles
+/// start, while the test-owned browser is in front. `bind` then checks the
+/// refusal under the oracles and reuses this attachment.
+fn prepare_same_space_attachment(
+ fixture: &mut BrowserFixture,
+ session: &str,
+) -> Option<ToolResponse> {
+ if !cfg!(target_os = "macos") {
+ return None;
+ }
let started = fixture
.driver
.call("start_session", serde_json::json!({ "session": session }));
assert!(!started.is_error(), "start_session failed: {}", started.raw);
+ let raised = fixture.driver.call(
+ "bring_to_front",
+ serde_json::json!({ "pid": fixture.pid, "window_id": fixture.window_id }),
+ );
+ assert!(!raised.is_error(), "bring_to_front failed: {}", raised.raw);
let prepared = fixture.driver.call(
"browser_prepare",
serde_json::json!({
@@ -1455,6 +1479,64 @@ fn bind(fixture: &mut BrowserFixture, session: &str) -> (String, String, ToolRes
"{}",
prepared.raw
);
+ fixture.attached_before_observation.push(session.to_owned());
+ Some(prepared)
+}
+
+/// A background existing-profile attachment while another app is in front
+/// on the same Space must refuse without visible effects (macOS).
+fn assert_same_space_background_refusal(prepared: &ToolResponse) {
+ assert_eq!(
+ prepared.structured()["refusal"]["code"],
+ "background_unavailable",
+ "{}",
+ prepared.raw
+ );
+ assert_eq!(
+ prepared.structured()["refusal"]["detail"]["visible_effects"],
+ false,
+ "{}",
+ prepared.raw
+ );
+ assert_eq!(
+ prepared.structured()["refusal"]["detail"]["alternative_delivery_mode"],
+ "foreground",
+ "{}",
+ prepared.raw
+ );
+}
+
+fn bind(fixture: &mut BrowserFixture, session: &str) -> (String, String, ToolResponse) {
+ let attached_before = fixture
+ .attached_before_observation
+ .iter()
+ .any(|attached| attached == session);
+ if !attached_before {
+ let started = fixture
+ .driver
+ .call("start_session", serde_json::json!({ "session": session }));
+ assert!(!started.is_error(), "start_session failed: {}", started.raw);
+ }
+ let prepared = fixture.driver.call(
+ "browser_prepare",
+ serde_json::json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.window_id,
+ "session": session,
+ "strategy": {"kind": "existing_profile"},
+ }),
+ );
+ if attached_before {
+ assert_same_space_background_refusal(&prepared);
+ } else {
+ assert_eq!(prepared.structured()["status"], "ok", "{}", prepared.raw);
+ assert_eq!(
+ prepared.structured()["action"],
+ "attached_existing_profile",
+ "{}",
+ prepared.raw
+ );
+ }
// A newly mapped Wayland toplevel can briefly appear with a protocol-local
// id before the compositor publishes its stable pid/geometry identity.
// Model the real client preamble: re-list windows and bind only the id the
@@ -1768,8 +1850,9 @@ fn run_roundtrip(spec: &BrowserSpec) {
execute_case(case(&spec.name, "browser_tool_roundtrip"), |evidence| {
let mut fixture = launch_browser(spec, &scenario);
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-roundtrip-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-roundtrip-{}", fixture.pid);
let (target, tab, snapshot) = bind(fixture, &session);
let click_ref = ref_by_label(&snapshot, "id=btn-increment");
let click = fixture.driver.call(
@@ -1822,8 +1905,9 @@ fn run_trust_gated_dom_click(spec: &BrowserSpec) {
let mut fixture =
launch_browser_with_html(spec, &scenario, standalone_trust_gated_click_html());
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-trust-gated-dom-click-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-trust-gated-dom-click-{}", fixture.pid);
let (target, tab, snapshot) = bind(fixture, &session);
let click_ref = ref_by_label(&snapshot, "id=standalone-trust-gated");
let click = fixture.driver.call(
@@ -1884,8 +1968,9 @@ fn run_semantic_state(spec: &BrowserSpec) {
let mut fixture =
launch_browser_with_html(spec, &scenario, standalone_semantic_fixture_html());
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-semantic-state-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-semantic-state-{}", fixture.pid);
let (target, tab, _) = bind(fixture, &session);
let snapshot = fixture.driver.call(
"get_browser_state",
@@ -1992,8 +2077,9 @@ fn run_background_type(spec: &BrowserSpec) {
execute_case(case(&spec.name, "background_type"), |evidence| {
let mut fixture = launch_browser(spec, &scenario);
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-background-type-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-background-type-{}", fixture.pid);
let (target, tab, snapshot) = bind(fixture, &session);
let input_ref = ref_by_label(&snapshot, "id=txt-input");
let typed = fixture.driver.call(
@@ -2358,8 +2444,9 @@ fn run_trusted_click(spec: &BrowserSpec) {
execute_case(case, |evidence| {
let mut fixture = launch_browser(spec, &scenario);
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-trusted-click-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-trusted-click-{}", fixture.pid);
let (target, tab, snapshot) = bind(fixture, &session);
let click_ref = ref_by_label(&snapshot, "id=btn-increment");
let click = fixture.driver.call(
@@ -2612,12 +2699,15 @@ fn run_existing_profile_attach(spec: &BrowserSpec) {
|evidence| {
let mut fixture = launch_browser(spec, &scenario);
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-existing-profile-{}", fixture.pid);
+ let attached_before = prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-existing-profile-{}", fixture.pid);
- let started = fixture
- .driver
- .call("start_session", serde_json::json!({ "session": session }));
- assert!(!started.is_error(), "start_session failed: {}", started.raw);
+ if attached_before.is_none() {
+ let started = fixture
+ .driver
+ .call("start_session", serde_json::json!({ "session": session }));
+ assert!(!started.is_error(), "start_session failed: {}", started.raw);
+ }
fixture.driver.start_behavior_recording();
let prepared = fixture.driver.call(
@@ -2629,50 +2719,62 @@ fn run_existing_profile_attach(spec: &BrowserSpec) {
"strategy": {"kind": "existing_profile"},
}),
);
- assert_eq!(prepared.structured()["status"], "ok", "{}", prepared.raw);
- assert_eq!(
- prepared.structured()["action"],
- "attached_existing_profile",
- "{}",
- prepared.raw
- );
- assert_eq!(
- prepared.structured()["attachment"]["kind"],
- "existing_profile"
- );
- assert_eq!(
- prepared.structured()["attachment"]["capabilities_invalidated"],
- true
- );
- assert_eq!(
- prepared.structured()["attachment"]["next_action"],
- "get_browser_state"
- );
- assert_eq!(
- prepared.structured()["side_effects"]["launched_browser"],
- false
- );
- assert_eq!(
- prepared.structured()["side_effects"]["created_profile"],
- false
- );
- assert_eq!(
- prepared.structured()["side_effects"]["copied_profile_data"],
- false
- );
+ // On macOS the attachment was made before observation, while
+ // the browser was in front; check its result, and that a
+ // background attachment now refuses without visible effects.
+ let prepared = match &attached_before {
+ Some(attached) => {
+ assert_same_space_background_refusal(&prepared);
+ attached
+ }
+ None => &prepared,
+ };
+ {
+ assert_eq!(prepared.structured()["status"], "ok", "{}", prepared.raw);
+ assert_eq!(
+ prepared.structured()["action"],
+ "attached_existing_profile",
+ "{}",
+ prepared.raw
+ );
+ assert_eq!(
+ prepared.structured()["attachment"]["kind"],
+ "existing_profile"
+ );
+ assert_eq!(
+ prepared.structured()["attachment"]["capabilities_invalidated"],
+ true
+ );
+ assert_eq!(
+ prepared.structured()["attachment"]["next_action"],
+ "get_browser_state"
+ );
+ assert_eq!(
+ prepared.structured()["side_effects"]["launched_browser"],
+ false
+ );
+ assert_eq!(
+ prepared.structured()["side_effects"]["created_profile"],
+ false
+ );
+ assert_eq!(
+ prepared.structured()["side_effects"]["copied_profile_data"],
+ false
+ );
- let public_result = prepared.raw.to_string();
- assert!(!public_result.contains("ws://"), "{}", prepared.raw);
- assert!(
- !public_result.contains("webSocketDebuggerUrl"),
- "{}",
- prepared.raw
- );
- assert!(
- !public_result.contains(&fixture._profile.path().display().to_string()),
- "{}",
- prepared.raw
- );
+ let public_result = prepared.raw.to_string();
+ assert!(!public_result.contains("ws://"), "{}", prepared.raw);
+ assert!(
+ !public_result.contains("webSocketDebuggerUrl"),
+ "{}",
+ prepared.raw
+ );
+ assert!(
+ !public_result.contains(&fixture._profile.path().display().to_string()),
+ "{}",
+ prepared.raw
+ );
+ }
let state = fixture.driver.call(
"get_browser_state",
@@ -3054,8 +3156,9 @@ fn run_stale_ref(spec: &BrowserSpec) {
execute_case(case(&spec.name, "browser_ref_stale"), |evidence| {
let mut fixture = launch_browser(spec, &scenario);
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-stale-ref-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-stale-ref-{}", fixture.pid);
let (target, tab, first) = bind(fixture, &session);
let stale_ref = ref_by_label(&first, "id=btn-increment");
let newer = fixture.driver.call(
@@ -3105,8 +3208,9 @@ fn run_frame_roundtrip(spec: &BrowserSpec) {
wait_for_text(&fixture.server, "standalone-frame-state", "iframe=ready");
wait_for_observed(&child_server, "WEB_HARNESS_MARKER_v1");
+ let session = format!("standalone-frame-roundtrip-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-frame-roundtrip-{}", fixture.pid);
let (target, tab, snapshot) = bind(fixture, &session);
assert_eq!(
snapshot.structured()["oopif"]["status"],
@@ -4398,8 +4502,9 @@ fn run_dialog_background_refusal(spec: &BrowserSpec) {
let mut fixture =
launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-dialog-refusal-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-dialog-refusal-{}", fixture.pid);
let (target_id, tab_id, _) = bind(fixture, &session);
let refused = fixture.driver.call(
"browser_dialog",
@@ -4568,8 +4673,9 @@ fn run_upload(spec: &BrowserSpec) {
let mut fixture =
launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-upload-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-upload-{}", fixture.pid);
let (target, tab, _) = bind(fixture, &session);
let snapshot = fixture.driver.call(
"get_browser_state",
@@ -4621,8 +4727,9 @@ fn run_pointer_actions(spec: &BrowserSpec) {
let mut fixture =
launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-pointer-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-pointer-{}", fixture.pid);
let (target, tab, _) = bind(fixture, &session);
let snapshot = fixture.driver.call(
"get_browser_state",
@@ -4747,8 +4854,9 @@ fn run_download(spec: &BrowserSpec) {
let mut fixture =
launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-download-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-download-{}", fixture.pid);
let (target, tab, snapshot) = bind(fixture, &session);
// Sandboxed browser packages such as Ubuntu's Chromium snap use a
// private /tmp namespace. A temporary directory under HOME remains
@@ -4886,8 +4994,9 @@ fn run_type_replace(spec: &BrowserSpec) {
execute_case(case(&spec.name, "browser_type_replace"), |evidence| {
let mut fixture = launch_browser(spec, &scenario);
*evidence = recording_evidence(fixture.driver.recording_dir());
+ let session = format!("standalone-type-replace-{}", fixture.pid);
+ prepare_same_space_attachment(&mut fixture, &session);
run_with_background_oracles(&mut fixture, |fixture| {
- let session = format!("standalone-type-replace-{}", fixture.pid);
let (target, tab, snapshot) = bind(fixture, &session);
let input_ref = ref_by_label(&snapshot, "id=txt-input");
let number_input_ref = ref_by_label(&snapshot, "id=number-input");