Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): refuse background submits into a Chromium address bar (#4856)

Francesco Bonacci3532ba53f947
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index 3ac8d59627..3df1b9e6d6 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -288,6 +288,23 @@ impl Tool for TypeTextTool {
return super::background_refusal_result(pid, wid, &refusal);
}
+ // A Chromium browser's own text fields (the address bar) take an AX
+ // insert as display text only: Return, background or foreground, does
+ // not submit it, and background keystrokes never reach the browser.
+ let chromium_native_field = !delivery_mode.is_foreground()
+ && !used_pixel_focus
+ && is_chromium_identity(&apps::bundle_id_for_pid(pid).unwrap_or_default())
+ && matches!(
+ classify_target_web_area(pid, element_ptr, window_id),
+ WebAreaClassification::NonWebContent
+ );
+ if let Some(refusal) = chromium_native_field_submit_refusal(chromium_native_field, &text) {
+ return match window_id {
+ Some(wid) => super::background_refusal_result(pid, wid, &refusal),
+ None => ToolResult::error(refusal.reason.clone()),
+ };
+ }
+
if let (Some((element, _)), Some(wid)) = (element_guard.as_ref(), window_id) {
let center_guard = element.clone();
if let Ok((Some((screen_x, screen_y)), target_rect)) =
@@ -466,6 +483,11 @@ impl Tool for TypeTextTool {
.to_string(),
)
};
+ let note = if chromium_native_field && path == PATH_AX {
+ format!("{note}{CHROMIUM_NATIVE_FIELD_NOTE}")
+ } else {
+ note
+ };
ToolResult::text(format!(
"{mark} {char_count} char(s){detail}.{note}{}",
changes.result_suffix()
@@ -743,6 +765,31 @@ fn electron_background_ax_refusal(
})
}
+const CHROMIUM_NATIVE_FIELD_NOTE: &str = " This is a Chromium browser's own field (address \
+ bar): it shows accessibility-inserted text but does not submit it on Return. To \
+ navigate, re-type the URL ending in \\n with delivery_mode:\"foreground\", or open it \
+ with launch_app {bundle_id, urls:[...]}.";
+
+/// Refuse a background AX insert that is meant to submit (it contains a line
+/// break) into a Chromium browser's native field. The insert would land as
+/// display text, the line break would not submit it, and the caller would see
+/// a success for a navigation that never happened (bench CDB-G04: nine
+/// address-bar attempts in one trial, none of which loaded the page).
+fn chromium_native_field_submit_refusal(
+ chromium_native_field: bool,
+ text: &str,
+) -> Option<BackgroundRefusal> {
+ if !chromium_native_field || !text.contains(['\n', '\r']) {
+ return None;
+ }
+ Some(BackgroundRefusal {
+ code: "browser_field_needs_keystrokes",
+ reason: "A Chromium browser's own field (address bar) does not submit accessibility-inserted text, and background keystrokes do not reach the browser, so nothing was typed. Re-send the same type_text with delivery_mode:\"foreground\" (the driver activates the browser, keeps the field focus, types and presses Return), or open a URL with launch_app {bundle_id, urls:[...]}."
+ .to_owned(),
+ advice: Some("foreground"),
+ })
+}
+
const DELIVERY_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
const DELIVERY_DRAIN_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10);
@@ -1560,6 +1607,19 @@ fn type_text_blocking(
#[cfg(test)]
mod tests {
+ #[test]
+ fn chromium_native_field_refuses_only_background_submits() {
+ let refusal = super::chromium_native_field_submit_refusal(true, "file:///tmp/report.log\n")
+ .expect("a submit into the address bar is refused");
+ assert_eq!(refusal.code, "browser_field_needs_keystrokes");
+ assert_eq!(refusal.advice, Some("foreground"));
+ assert!(refusal.reason.contains("delivery_mode:\"foreground\""));
+ assert!(super::chromium_native_field_submit_refusal(true, "a\rb").is_some());
+ // Plain text still inserts (with a note); other surfaces are untouched.
+ assert!(super::chromium_native_field_submit_refusal(true, "example.com").is_none());
+ assert!(super::chromium_native_field_submit_refusal(false, "example.com\n").is_none());
+ }
+
#[test]
fn browser_identities_take_the_activating_rung() {
for id in [