Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): return from a trusted browser click when its handler opens a dialog (#6562)

injaneityf440b5a51bf3
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
index 1a3ea0ce84..adc456073b 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/pointer-cursor-and-browser.mdx
@@ -580,7 +580,7 @@ Exact browser input over CDP: clicks, typing, pointer, dialogs, files and downlo
### `browser_click`
-Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode="foreground" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route="dom_event" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If the handler opens a page-owned JavaScript dialog, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.
+Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode="foreground" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route="dom_event" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If a click's handler opens a page-owned JavaScript dialog, on either route, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.
Effect: mutating. Platforms: macOS, Linux, Windows.
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
index e589d3cbf2..d8de6c17b8 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
@@ -1156,8 +1156,8 @@ impl BrowserClickTool {
input_route=\"dom_event\" (synthetic pointer and mouse \
down/up/click sequence at the element's center, ref required) is used only when explicitly requested; \
it proves dispatch, not control activation, because trust-gated \
- controls may ignore synthetic events. If the handler opens a \
- page-owned JavaScript dialog, the call returns at once with a \
+ controls may ignore synthetic events. If a click's handler opens a \
+ page-owned JavaScript dialog, on either route, the call returns at once with a \
dialog block (present, kind, dialog_id) to pass to browser_dialog. \
Refused while a page-owned dialog is open on the tab, and for \
heuristic bindings."
@@ -1363,11 +1363,11 @@ impl Tool for BrowserClickTool {
let conn = &validated.conn;
let cdp = cdp_session.as_str();
+ let cdp_target_id = validated.tab.cdp_target_id.as_str();
// dom_event route: explicit opt-in only.
if route == "dom_event" {
let backend = backend_node_id.expect("checked above");
- let cdp_target_id = validated.tab.cdp_target_id.as_str();
let resolved = match conn
.call(
Some(cdp),
@@ -1439,17 +1439,9 @@ impl Tool for BrowserClickTool {
"functionDeclaration": DOM_EVENT_CLICK_SEQUENCE,
}),
);
- tokio::pin!(click);
- let outcome = loop {
- tokio::select! {
- replied = &mut click => break replied,
- _ = tokio::time::sleep(std::time::Duration::from_millis(25)) => {
- if conn.dialog_state(cdp_target_id).is_some() {
- break Ok(json!({}));
- }
- }
- }
- };
+ let outcome = call_until_dialog(conn, cdp_target_id, click)
+ .await
+ .unwrap_or(Ok(json!({})));
return match (outcome, conn.dialog_state(cdp_target_id)) {
(Ok(_), None) => {
let observation = observe_dom_click(conn, cdp, &object_id, before).await;
@@ -1576,33 +1568,49 @@ impl Tool for BrowserClickTool {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
let mut delivery_error = None;
+ let mut opened = None;
for (event_type, click_count) in [("mousePressed", 1), ("mouseReleased", 1)] {
- if let Err(error) = conn
- .call(
+ let press = conn.call(
+ Some(cdp),
+ "Input.dispatchMouseEvent",
+ json!({
+ "type": event_type,
+ "x": x,
+ "y": y,
+ "button": "left",
+ "clickCount": click_count,
+ }),
+ );
+ match call_until_dialog(conn, cdp_target_id, press).await {
+ Ok(Ok(_)) => {}
+ Ok(Err(error)) => {
+ delivery_error = Some(error);
+ break;
+ }
+ // Still send the release after a mousedown dialog, so the
+ // page is not left with the button held.
+ Err(open) => opened = Some(open),
+ }
+ }
+ let opened = opened.or_else(|| conn.dialog_state(cdp_target_id));
+ let unfocus = json!({ "enabled": false });
+ let cleanup_error = if opened.is_some() {
+ // The blocked renderer applies this once the dialog closes, and
+ // only then replies, so send it without waiting for the reply.
+ let _ = conn
+ .call_with_timeout(
Some(cdp),
- "Input.dispatchMouseEvent",
- json!({
- "type": event_type,
- "x": x,
- "y": y,
- "button": "left",
- "clickCount": click_count,
- }),
+ "Emulation.setFocusEmulationEnabled",
+ unfocus,
+ std::time::Duration::from_millis(50),
)
+ .await;
+ None
+ } else {
+ conn.call(Some(cdp), "Emulation.setFocusEmulationEnabled", unfocus)
.await
- {
- delivery_error = Some(error);
- break;
- }
- }
- let cleanup_error = conn
- .call(
- Some(cdp),
- "Emulation.setFocusEmulationEnabled",
- json!({ "enabled": false }),
- )
- .await
- .err();
+ .err()
+ };
if let Some(error) = delivery_error {
// Trusted input is the contract; we never silently fall back to
// synthetic events. Focus emulation has already been unwound.
@@ -1625,7 +1633,7 @@ impl Tool for BrowserClickTool {
.with_detail(json!({ "delivery": "unknown", "retryable": false }))
.to_tool_result();
}
- ToolResult::text(format!("clicked ({x:.0}, {y:.0}) in {tab_id}")).with_structured(json!({
+ let mut structured = json!({
"status": "ok",
"route": "trusted",
"delivery_mode": if foreground { "foreground" } else { "background" },
@@ -1635,7 +1643,44 @@ impl Tool for BrowserClickTool {
"frame": frame_kind,
"x": x,
"y": y,
- }))
+ });
+ let Some(open) = opened else {
+ return ToolResult::text(format!("clicked ({x:.0}, {y:.0}) in {tab_id}"))
+ .with_structured(structured);
+ };
+ structured["dialog"] = dialog_block(&open);
+ structured["escalation"] = json!({
+ "recommended": "page",
+ "reason": "the click opened a page-owned JavaScript dialog; resolve it with browser_dialog, then refresh page state",
+ });
+ ToolResult::text(format!(
+ "clicked ({x:.0}, {y:.0}) in {tab_id}; its handler opened a {} dialog that is \
+ blocking the page. Inspect and resolve it with browser_dialog (dialog_id {}) \
+ before any other action on this tab",
+ open.kind,
+ dialog_capability_id(open.generation)
+ ))
+ .with_structured(structured)
+ }
+}
+
+/// Await `call` unless a page-owned dialog opens on the tab first. The dialog
+/// blocks the renderer, so the call's reply would not come until it closes.
+async fn call_until_dialog(
+ conn: &super::cdp_ws::CdpConnection,
+ cdp_target_id: &str,
+ call: impl std::future::Future<Output = anyhow::Result<Value>>,
+) -> Result<anyhow::Result<Value>, super::cdp_ws::CdpDialogState> {
+ tokio::pin!(call);
+ loop {
+ tokio::select! {
+ replied = &mut call => return Ok(replied),
+ _ = tokio::time::sleep(std::time::Duration::from_millis(25)) => {
+ if let Some(open) = conn.dialog_state(cdp_target_id) {
+ return Err(open);
+ }
+ }
+ }
}
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
index 1b542a4d6e..116c89407f 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
@@ -64,6 +64,11 @@ pub(super) struct FixtureState {
/// A synthetic `this.click()` opens a confirm() dialog: the mock emits
/// `Page.javascriptDialogOpening` and never answers the call.
click_opens_dialog: bool,
+ /// The trusted mouse event (`mousePressed` or `mouseReleased`) that opens
+ /// a confirm() dialog. Like Chromium, the renderer then answers no input
+ /// or emulation command until it closes.
+ trusted_click_opens_dialog: Option<&'static str>,
+ renderer_blocked: bool,
/// Tab sessions that enabled the Page domain. Chromium reports a dialog
/// to each of them, not to the session whose call opened it.
page_sessions: Vec<String>,
@@ -125,6 +130,8 @@ impl Default for FixtureState {
fail_key_down_after: None,
completed_key_pairs: 0,
click_opens_dialog: false,
+ trusted_click_opens_dialog: None,
+ renderer_blocked: false,
page_sessions: Vec::new(),
semantic_large_page: false,
semantic_html_style: String::new(),
@@ -743,6 +750,28 @@ fn fixture_handler(state: SharedState) -> MockHandler {
MockReply::ok(json!({}))
}
}
+ "Input.dispatchMouseEvent" | "Emulation.setFocusEmulationEnabled"
+ if st.renderer_blocked =>
+ {
+ MockReply::hang()
+ }
+ "Input.dispatchMouseEvent"
+ if st
+ .trusted_click_opens_dialog
+ .is_some_and(|opener| call.params["type"] == opener) =>
+ {
+ st.renderer_blocked = true;
+ let events = st
+ .page_sessions
+ .iter()
+ .map(|page_session| MockEvent {
+ method: "Page.javascriptDialogOpening".into(),
+ session_id: Some(page_session.clone()),
+ params: json!({ "type": "confirm", "message": "Delete?", "url": st.main_url }),
+ })
+ .collect();
+ MockReply::hang().with_events(events)
+ }
"DOM.focus"
| "Emulation.setFocusEmulationEnabled"
| "Input.dispatchMouseEvent"
@@ -818,6 +847,7 @@ fn fixture_handler(state: SharedState) -> MockHandler {
MockReply::ok(json!({ "result": { "value": true } }))
}
"Page.handleJavaScriptDialog" if is_tab => {
+ st.renderer_blocked = false;
MockReply::ok(json!({})).with_events(vec![MockEvent {
method: "Page.javascriptDialogClosed".into(),
session_id: Some(sess.clone()),
@@ -3984,3 +4014,60 @@ async fn dom_event_click_refuses_while_a_dialog_is_already_open() {
assert_eq!(t["refusal"]["code"], "browser_action_unavailable", "{t}");
assert!(recorded_calls(&f, "Input.dispatchMouseEvent").is_empty());
}
+
+#[tokio::test]
+async fn trusted_coordinate_click_returns_when_its_handler_opens_a_dialog() {
+ for opener in ["mouseReleased", "mousePressed"] {
+ trusted_click_returns_when_a_dialog_opens_on(opener).await;
+ }
+}
+
+async fn trusted_click_returns_when_a_dialog_opens_on(opener: &'static str) {
+ let f = fixture_with(|state| state.trusted_click_opens_dialog = Some(opener)).await;
+ let (target, tab) = bind(&f).await;
+
+ let started = std::time::Instant::now();
+ let clicked = BrowserClickTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target, "tab_id": tab, "session": SESSION, "x": 10, "y": 20,
+ "delivery_mode": "foreground"
+ }))
+ .await;
+ assert!(
+ started.elapsed() < std::time::Duration::from_secs(5),
+ "the click must not wait out the CDP call timeout ({opener}): {:?}",
+ started.elapsed()
+ );
+ let s = structured(&clicked);
+ assert_eq!(s["status"], "ok", "{s}");
+ assert_eq!(s["route"], "trusted", "{s}");
+ assert_eq!(s["dialog"]["present"], true, "{s}");
+ assert_eq!(s["dialog"]["kind"], "confirm", "{s}");
+ let dialog_id = s["dialog"]["dialog_id"]
+ .as_str()
+ .expect("dialog id")
+ .to_owned();
+ let mouse: Vec<_> = recorded_calls(&f, "Input.dispatchMouseEvent")
+ .into_iter()
+ .map(|(_, p)| p["type"].clone())
+ .collect();
+ assert_eq!(
+ mouse,
+ [json!("mousePressed"), json!("mouseReleased")],
+ "the press must always be released ({opener})"
+ );
+ let focus_emulation = recorded_calls(&f, "Emulation.setFocusEmulationEnabled");
+ assert_eq!(
+ focus_emulation.last().map(|(_, p)| p["enabled"].clone()),
+ Some(json!(false)),
+ "focus emulation must still be switched off: {focus_emulation:?}"
+ );
+
+ let accepted = BrowserDialogTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target, "tab_id": tab, "action": "accept",
+ "dialog_id": dialog_id, "session": SESSION, "delivery_mode": "foreground"
+ }))
+ .await;
+ assert_eq!(structured(&accepted)["status"], "ok");
+}
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
index 94b9831fe9..52c28a5312 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
@@ -2610,7 +2610,7 @@
"read_only": false
},
{
- "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If the handler opens a page-owned JavaScript dialog, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.",
+ "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If a click's handler opens a page-owned JavaScript dialog, on either route, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.",
"destructive": false,
"idempotent": false,
"input_schema": {
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index 07ed2e3d8f..af215b766e 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -2338,7 +2338,7 @@
"read_only": false
},
{
- "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If the handler opens a page-owned JavaScript dialog, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.",
+ "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If a click's handler opens a page-owned JavaScript dialog, on either route, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.",
"destructive": false,
"idempotent": false,
"input_schema": {
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
index 4a367d6e67..0146c10047 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
@@ -2304,7 +2304,7 @@
"read_only": false
},
{
- "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If the handler opens a page-owned JavaScript dialog, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.",
+ "description": "Click a page element (by ref) or viewport coordinates in an exactly-bound tab. Default route is trusted hardware-like input (Input.dispatchMouseEvent), and refuses where that route cannot preserve standalone-browser background posture unless delivery_mode=\"foreground\" accepts that the browser window may activate (including Linux Chromium configurations whose trusted route cannot stay in the background; for example a browser inside a sandbox). input_route=\"dom_event\" (synthetic pointer and mouse down/up/click sequence at the element's center, ref required) is used only when explicitly requested; it proves dispatch, not control activation, because trust-gated controls may ignore synthetic events. If a click's handler opens a page-owned JavaScript dialog, on either route, the call returns at once with a dialog block (present, kind, dialog_id) to pass to browser_dialog. Refused while a page-owned dialog is open on the tab, and for heuristic bindings.",
"destructive": false,
"idempotent": false,
"input_schema": {