Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): add browser_select for native HTML select (#5232)

injaneity3df424fafcd6
diff --git a/.github/scripts/cua-driver-mcp-compat/expected-tools.json b/.github/scripts/cua-driver-mcp-compat/expected-tools.json
index 163c008b8e..3b4d96cfbf 100644
--- a/.github/scripts/cua-driver-mcp-compat/expected-tools.json
+++ b/.github/scripts/cua-driver-mcp-compat/expected-tools.json
@@ -13,6 +13,7 @@
"browser_navigate",
"browser_pointer",
"browser_prepare",
+ "browser_select",
"browser_set_input_files",
"browser_type",
"check_for_update",
diff --git a/.github/scripts/tests/test_cua_driver_release_wiring.py b/.github/scripts/tests/test_cua_driver_release_wiring.py
index 674a321886..afdae142a0 100644
--- a/.github/scripts/tests/test_cua_driver_release_wiring.py
+++ b/.github/scripts/tests/test_cua_driver_release_wiring.py
@@ -1105,7 +1105,7 @@ fi
self.assertEqual(
len(expected["baseTools"]), len(set(expected["baseTools"]))
)
- self.assertEqual(len(expected["baseTools"]), 60)
+ self.assertEqual(len(expected["baseTools"]), 61)
self.assertEqual(
expected["outputSchemaCountByPlatform"],
{"darwin": 35, "linux": 39, "win32": 35},
diff --git a/docs/content/docs/cua-driver/reference/index.mdx b/docs/content/docs/cua-driver/reference/index.mdx
index b3918d8f26..5cc98b33a9 100644
--- a/docs/content/docs/cua-driver/reference/index.mdx
+++ b/docs/content/docs/cua-driver/reference/index.mdx
@@ -40,7 +40,7 @@ description: "Every cua-driver command and every MCP tool on macOS, Linux and Wi
| [Value and clipboard tools](/cua-driver/reference/mcp-tools/values-and-clipboard) | Set element values and read or write the clipboard. 3 tools. |
| [Batch and script tools](/cua-driver/reference/mcp-tools/batch) | Run several action tools, or a script that drives them, in one call. 2 tools. |
| [Page tools](/cua-driver/reference/mcp-tools/page) | Read and act on web pages, and prepare and navigate browsers. 4 tools. |
-| [Browser input tools](/cua-driver/reference/mcp-tools/browser-input) | Exact browser input over CDP: clicks, typing, pointer, dialogs, files and downloads. 6 tools. |
+| [Browser input tools](/cua-driver/reference/mcp-tools/browser-input) | Exact browser input over CDP: clicks, typing, pointer, dialogs, files and downloads. 7 tools. |
| [Session tools](/cua-driver/reference/mcp-tools/sessions) | Start, escalate, inspect and end lifecycle sessions. 6 tools. |
| [Agent cursor tools](/cua-driver/reference/mcp-tools/agent-cursor) | Show, animate and style the agent cursor. 4 tools. |
| [Recording tools](/cua-driver/reference/mcp-tools/recording) | Record and replay trajectories. 5 tools. |
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/browser-input.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/browser-input.mdx
index 504260a021..b3ab49e0a2 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/browser-input.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/browser-input.mdx
@@ -14,6 +14,7 @@ description: "Exact browser input over CDP: clicks, typing, pointer, dialogs, fi
| --- | --- | --- |
| [`browser_click`](#browser_click) | Click a page element (by ref) or viewport coordinates in an exactly-bound tab. | macOS, Linux, Windows |
| [`browser_type`](#browser_type) | Type text into an exactly-bound tab via the Input domain. | macOS, Linux, Windows |
+| [`browser_select`](#browser_select) | Select one option of a native HTML <select> ref on an exactly-bound tab without opening its popup. | macOS, Linux, Windows |
| [`browser_pointer`](#browser_pointer) | Perform hover, right-click, double-click, scroll, or drag in an exactly-bound browser tab. | macOS, Linux, Windows |
| [`browser_dialog`](#browser_dialog) | Inspect or resolve a page-owned JavaScript alert, confirm, prompt, or beforeunload dialog on one exactly-bound tab. | macOS, Linux, Windows |
| [`browser_set_input_files`](#browser_set_input_files) | Assign one or more explicit absolute local files to an exact live <input type=file> ref through CDP. | macOS, Linux, Windows |
@@ -66,6 +67,27 @@ Effect: mutating. Platforms: macOS, Linux, Windows.
{"target_id":"<target_id>","tab_id":"<tab_id>","ref":"<ref>","text":"<text>"}
```
+## `browser_select`
+
+Select one option of a native HTML &lt;select&gt; ref on an exactly-bound tab without opening its popup. Pass exactly one of label or value; exactly one option must equal it. Refuses non-native, multiple-selection, and disabled selects, disabled options, and zero or ambiguous matches before changing anything. A change dispatches bubbling input and change events; they are synthetic, so verify the page accepted the selection. Selecting the option that is already selected dispatches no events.
+
+Effect: mutating. Platforms: macOS, Linux, Windows.
+
+| Parameter | Type | Default | Description |
+| --- | --- | --- | --- |
+| <span id="browser_select--label"></span>`label` | `string` | | Exact visible label of the option to select. |
+| <span id="browser_select--ref"></span>`ref` | `string` | required | Page element ref in the p&lt;snapshot&gt;:&lt;index&gt; namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab. |
+| <span id="browser_select--session"></span>`session` | `string` | | For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session. |
+| <span id="browser_select--tab_id"></span>`tab_id` | `string` | required | Opaque tab id from get_browser_state (session-scoped). |
+| <span id="browser_select--target_id"></span>`target_id` | `string` | required | Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id). |
+| <span id="browser_select--value"></span>`value` | `string` | | Exact value attribute of the option to select. |
+
+**Example arguments**
+
+```json
+{"target_id":"<target_id>","tab_id":"<tab_id>","ref":"<ref>"}
+```
+
## `browser_pointer`
Perform hover, right-click, double-click, scroll, or drag in an exactly-bound browser tab. Semantic refs must declare pointer for hover, right-click, double-click, and drag; scroll accepts a scroll or pointer capability. The trusted route uses CDP Input events and refuses if standalone background posture cannot be preserved. The explicit dom_event route requires a page ref and synthesizes full-background DOM events. Never activates or brings a tab to the foreground.
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx
index dd7a300e52..ebfb3d7a33 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx
@@ -171,6 +171,7 @@ Exact browser input over CDP: clicks, typing, pointer, dialogs, files and downlo
| --- | --- | --- |
| <span id="browser_click"></span>[`browser_click`](/cua-driver/reference/mcp-tools/browser-input#browser_click) | Click a page element (by ref) or viewport coordinates in an exactly-bound tab. | macOS, Linux, Windows |
| <span id="browser_type"></span>[`browser_type`](/cua-driver/reference/mcp-tools/browser-input#browser_type) | Type text into an exactly-bound tab via the Input domain. | macOS, Linux, Windows |
+| <span id="browser_select"></span>[`browser_select`](/cua-driver/reference/mcp-tools/browser-input#browser_select) | Select one option of a native HTML &lt;select&gt; ref on an exactly-bound tab without opening its popup. | macOS, Linux, Windows |
| <span id="browser_pointer"></span>[`browser_pointer`](/cua-driver/reference/mcp-tools/browser-input#browser_pointer) | Perform hover, right-click, double-click, scroll, or drag in an exactly-bound browser tab. | macOS, Linux, Windows |
| <span id="browser_dialog"></span>[`browser_dialog`](/cua-driver/reference/mcp-tools/browser-input#browser_dialog) | Inspect or resolve a page-owned JavaScript alert, confirm, prompt, or beforeunload dialog on one exactly-bound tab. | macOS, Linux, Windows |
| <span id="browser_set_input_files"></span>[`browser_set_input_files`](/cua-driver/reference/mcp-tools/browser-input#browser_set_input_files) | Assign one or more explicit absolute local files to an exact live &lt;input type=file&gt; ref through CDP. | macOS, Linux, Windows |
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs
index d8c79dd0b2..a5ce1cc99c 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/authorization.rs
@@ -352,6 +352,7 @@ const BROWSER_BOUND_INPUT_OPERATIONS: &[&str] = &[
"browser_navigate",
"browser_click",
"browser_type",
+ "browser_select",
"browser_pointer",
];
const BROWSER_BOUND_INPUT_SCOPE_KEYS: &[&str] = &[
@@ -925,6 +926,7 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment {
| "browser_navigate"
| "browser_click"
| "browser_type"
+ | "browser_select"
| "browser_pointer"
| "history_status"
| "history_query"
@@ -1944,6 +1946,10 @@ mod tests {
ids("browser_click", serde_json::json!({})),
vec!["browser_bound_input"]
);
+ assert_eq!(
+ ids("browser_select", serde_json::json!({})),
+ vec!["browser_bound_input"]
+ );
assert_eq!(
ids("kill_app", serde_json::json!({})),
vec!["process_control"]
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
index 5a041f0d46..15162d9e7c 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
@@ -36,6 +36,7 @@ pub fn register_browser_tools(engine: &Arc<BrowserEngine>, registry: &mut ToolRe
registry.register(Box::new(BrowserNavigateTool::new(engine.clone())));
registry.register(Box::new(BrowserClickTool::new(engine.clone())));
registry.register(Box::new(BrowserTypeTool::new(engine.clone())));
+ registry.register(Box::new(BrowserSelectTool::new(engine.clone())));
registry.register(Box::new(BrowserDialogTool::new(engine.clone())));
registry.register(Box::new(BrowserSetInputFilesTool::new(engine.clone())));
registry.register(Box::new(BrowserDownloadTool::new(engine.clone())));
@@ -2226,6 +2227,226 @@ impl Tool for BrowserTypeTool {
}
}
+const SELECT_NATIVE_OPTION: &str = "function(field, wanted) { \
+ const view = this.ownerDocument && this.ownerDocument.defaultView; \
+ if (!view || !(this instanceof view.HTMLSelectElement)) \
+ return { refused: 'the ref is not a native HTML select element' }; \
+ if (this.multiple) return { refused: 'multiple-selection select elements are not supported' }; \
+ if (this.matches(':disabled')) return { refused: 'the select element is disabled' }; \
+ const matches = Array.from(this.options).filter((option) => option[field] === wanted); \
+ if (matches.length !== 1) \
+ return { refused: `${matches.length} options match the requested ${field}; exactly one must` }; \
+ const option = matches[0]; \
+ if (option.matches(':disabled')) return { refused: 'the matching option is disabled' }; \
+ const changed = !option.selected; \
+ if (changed) { \
+ option.selected = true; \
+ this.dispatchEvent(new view.Event('input', { bubbles: true, composed: true })); \
+ this.dispatchEvent(new view.Event('change', { bubbles: true })); \
+ } \
+ const current = this.options[this.selectedIndex]; \
+ return { changed, value: this.value, label: current ? current.label : null }; \
+}";
+
+pub struct BrowserSelectTool {
+ def: ToolDef,
+ engine: Arc<BrowserEngine>,
+}
+
+impl BrowserSelectTool {
+ pub fn new(engine: Arc<BrowserEngine>) -> Self {
+ let def = ToolDef {
+ name: "browser_select".into(),
+ description: "Select one option of a native HTML <select> ref on an exactly-bound \
+ tab without opening its popup. Pass exactly one of label or value; exactly one \
+ option must equal it. Refuses non-native, multiple-selection, and disabled \
+ selects, disabled options, and zero or ambiguous matches before changing \
+ anything. A change dispatches bubbling input and change events; they are \
+ synthetic, so verify the page accepted the selection. Selecting the option \
+ that is already selected dispatches no events."
+ .into(),
+ input_schema: json!({
+ "type": "object",
+ "properties": {
+ "target_id": schema_target_id(),
+ "tab_id": schema_tab_id(),
+ "session": schema_session(),
+ "ref": schema_ref(),
+ "label": {
+ "type": "string",
+ "description": "Exact visible label of the option to select."
+ },
+ "value": {
+ "type": "string",
+ "description": "Exact value attribute of the option to select."
+ },
+ },
+ "required": ["target_id", "tab_id", "ref"],
+ "additionalProperties": false
+ }),
+ read_only: false,
+ destructive: false,
+ idempotent: false,
+ open_world: true,
+ };
+ Self { def, engine }
+ }
+}
+
+#[async_trait]
+impl Tool for BrowserSelectTool {
+ fn def(&self) -> &ToolDef {
+ &self.def
+ }
+
+ async fn protected_resource_ownership(
+ &self,
+ adapter_id: &str,
+ args: &Value,
+ ) -> ProtectedResourceOwnership {
+ if adapter_id == "browser_bound_input" {
+ browser_resource_ownership(&self.engine, args)
+ } else {
+ ProtectedResourceOwnership::UserOwned
+ }
+ }
+
+ async fn protected_resource_scope(
+ &self,
+ adapter_id: &str,
+ args: &Value,
+ ) -> Result<Option<Value>, String> {
+ if adapter_id == "browser_bound_input" {
+ browser_protected_resource_scope(&self.engine, args, "browser_select").await
+ } else {
+ Ok(None)
+ }
+ }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ let (target_id, tab_id, ext_ref) = match (
+ args.require_str("target_id"),
+ args.require_str("tab_id"),
+ args.require_str("ref"),
+ ) {
+ (Ok(target), Ok(tab), Ok(ext_ref)) => (target, tab, ext_ref),
+ (Err(error), _, _) | (_, Err(error), _) | (_, _, Err(error)) => return error,
+ };
+ let (field, wanted) = match (args.opt_str("label"), args.opt_str("value")) {
+ (Some(label), None) => ("label", label),
+ (None, Some(value)) => ("value", value),
+ _ => return ToolResult::error("pass exactly one of label or value"),
+ };
+ let session = match require_explicit_session(&args) {
+ Ok(session) => session,
+ Err(error) => return error,
+ };
+ let _mutation = match self
+ .engine
+ .lock_mutation(&session, &target_id, &tab_id)
+ .await
+ {
+ Ok(guard) => guard,
+ Err(refusal) => return refusal.to_tool_result(),
+ };
+ let validated = match self
+ .engine
+ .revalidate_for_mutation(&session, &target_id, Some(&tab_id))
+ .await
+ {
+ Ok(validated) => validated,
+ Err(refusal) => return refusal.to_tool_result(),
+ };
+ let entry = match self
+ .engine
+ .store
+ .resolve_ref(&session, &target_id, &tab_id, &ext_ref)
+ {
+ Ok(entry) => entry,
+ Err(refusal) => return refusal.to_tool_result(),
+ };
+ let cdp_session = match self
+ .engine
+ .frame_session_for_mutation(&session, &target_id, &tab_id, &validated, &entry.frame)
+ .await
+ {
+ Ok(session) => session,
+ Err(refusal) => return refusal.to_tool_result(),
+ };
+ let conn = &validated.conn;
+ let object_id = match conn
+ .call(
+ Some(&cdp_session),
+ "DOM.resolveNode",
+ json!({ "backendNodeId": entry.backend_node_id }),
+ )
+ .await
+ .ok()
+ .and_then(|resolved| resolved["object"]["objectId"].as_str().map(str::to_owned))
+ {
+ Some(object_id) => object_id,
+ None => {
+ return BrowserRefusal::new(
+ BrowserRefusalCode::BrowserRefStale,
+ "the ref's node no longer resolves in the live page",
+ )
+ .to_tool_result()
+ }
+ };
+ let outcome = match conn
+ .call(
+ Some(&cdp_session),
+ "Runtime.callFunctionOn",
+ json!({
+ "objectId": object_id,
+ "functionDeclaration": SELECT_NATIVE_OPTION,
+ "arguments": [{ "value": field }, { "value": wanted }],
+ "returnByValue": true,
+ }),
+ )
+ .await
+ {
+ Ok(reply) => reply["result"]["value"].clone(),
+ Err(error) => {
+ return BrowserRefusal::new(
+ BrowserRefusalCode::BrowserActionUnavailable,
+ format!("the select script could not run on the ref: {error}"),
+ )
+ .to_tool_result()
+ }
+ };
+ if let Some(reason) = outcome["refused"].as_str() {
+ return BrowserRefusal::new(BrowserRefusalCode::BrowserActionUnavailable, reason)
+ .to_tool_result();
+ }
+ let Some(changed) = outcome["changed"].as_bool() else {
+ return BrowserRefusal::new(
+ BrowserRefusalCode::BrowserActionUnavailable,
+ "the select script failed before reporting a selection",
+ )
+ .to_tool_result();
+ };
+ ToolResult::text(if changed {
+ format!(
+ "selected {} in {tab_id} and dispatched synthetic input and change events",
+ outcome["label"]
+ )
+ } else {
+ format!("{} was already selected in {tab_id}", outcome["label"])
+ })
+ .with_structured(json!({
+ "status": "ok",
+ "target_id": target_id,
+ "tab_id": tab_id,
+ "ref": ext_ref,
+ "frame": entry.frame.kind.as_str(),
+ "changed": changed,
+ "value": outcome["value"],
+ "label": outcome["label"],
+ }))
+ }
+}
+
// ── browser_dialog ──────────────────────────────────────────────────────────
pub struct BrowserDialogTool {
@@ -2950,6 +3171,10 @@ mod tests {
BrowserTypeTool::new(e.clone()).def().clone(),
"browser_type",
),
+ (
+ BrowserSelectTool::new(e.clone()).def().clone(),
+ "browser_select",
+ ),
(
BrowserDialogTool::new(e.clone()).def().clone(),
"browser_dialog",
@@ -2978,6 +3203,7 @@ mod tests {
BrowserNavigateTool::new(e.clone()).def().clone(),
BrowserClickTool::new(e.clone()).def().clone(),
BrowserTypeTool::new(e.clone()).def().clone(),
+ BrowserSelectTool::new(e.clone()).def().clone(),
BrowserDialogTool::new(e.clone()).def().clone(),
BrowserSetInputFilesTool::new(e.clone()).def().clone(),
BrowserDownloadTool::new(e.clone()).def().clone(),
@@ -3002,6 +3228,7 @@ mod tests {
"browser_navigate",
"browser_click",
"browser_type",
+ "browser_select",
"browser_dialog",
"browser_set_input_files",
"browser_download",
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
index a036c3a4d6..67e81ea828 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
@@ -29,7 +29,7 @@ use super::pointer::BrowserPointerTool;
use super::refusal::BrowserRefusal;
use super::tools::{
browser_protected_resource_scope, BrowserClickTool, BrowserNavigateTool, BrowserPrepareTool,
- BrowserTypeTool, GetBrowserStateTool,
+ BrowserSelectTool, BrowserTypeTool, GetBrowserStateTool,
};
use super::types::{
BrowserClassification, BrowserEngineFamily, BrowserProcessRole, BrowserProduct,
@@ -66,6 +66,7 @@ pub(super) struct FixtureState {
viewport_css_width: f64,
viewport_css_height: f64,
tab_visible: bool,
+ select_outcome: Value,
/// Extra page targets Chromium reports but cannot map to any browser
/// window (`Browser.getWindowForTarget` → `-32000 Browser window not
/// found`): an extension side panel, an offscreen document.
@@ -105,6 +106,7 @@ impl Default for FixtureState {
viewport_css_width: 800.0,
viewport_css_height: 600.0,
tab_visible: true,
+ select_outcome: Value::Null,
calls: Vec::new(),
}
}
@@ -694,6 +696,13 @@ fn fixture_handler(state: SharedState) -> MockHandler {
"DOM.resolveNode" => MockReply::ok(json!({
"object": { "objectId": format!("obj-{}", call.params["backendNodeId"]) }
})),
+ "Runtime.callFunctionOn"
+ if call.params["functionDeclaration"]
+ .as_str()
+ .is_some_and(|declaration| declaration.contains("HTMLSelectElement")) =>
+ {
+ MockReply::ok(json!({ "result": { "value": st.select_outcome.clone() } }))
+ }
"Runtime.callFunctionOn" => MockReply::ok(json!({ "result": { "value": true } })),
other => MockReply::method_not_found(other),
}
@@ -3068,6 +3077,135 @@ async fn keystrokes_use_char_events_for_text_delivery() {
assert!(recorded_calls(&f, "Target.activateTarget").is_empty());
}
+async fn select_with(f: &Fixture, frame: &str, element: &str, matcher: Value) -> Value {
+ let (target, tab) = bind(f).await;
+ let snap = snapshot(f, &target, &tab).await;
+ let mut args = json!({
+ "target_id": target,
+ "tab_id": tab,
+ "ref": ref_of(&snap, frame, element),
+ "session": SESSION
+ });
+ args.as_object_mut()
+ .unwrap()
+ .extend(matcher.as_object().unwrap().clone());
+ let result = BrowserSelectTool::new(f.engine.clone()).invoke(args).await;
+ result
+ .structured_content
+ .clone()
+ .unwrap_or_else(|| json!({ "error": result.is_error }))
+}
+
+fn select_scripts(f: &Fixture) -> Vec<(Option<String>, Value)> {
+ recorded_calls(f, "Runtime.callFunctionOn")
+ .into_iter()
+ .filter(|(_, params)| {
+ params["functionDeclaration"]
+ .as_str()
+ .is_some_and(|declaration| declaration.contains("HTMLSelectElement"))
+ })
+ .collect()
+}
+
+#[tokio::test]
+async fn select_passes_the_matcher_as_cdp_arguments_and_reports_the_readback() {
+ for (field, wanted) in [("label", "Q3 report"), ("value", "q3-report")] {
+ let f = fixture_with(|st| {
+ st.select_outcome =
+ json!({ "changed": true, "value": "q3-report", "label": "Q3 report" });
+ })
+ .await;
+ let s = select_with(&f, "main", "Shadow Input", json!({ field: wanted })).await;
+ assert_eq!(s["status"], "ok", "{s}");
+ assert_eq!(s["changed"], true);
+ assert_eq!(s["value"], "q3-report");
+ assert_eq!(s["label"], "Q3 report");
+ assert_eq!(s["frame"], "main");
+
+ let scripts = select_scripts(&f);
+ assert_eq!(scripts.len(), 1);
+ let (session, params) = &scripts[0];
+ assert!(session.as_deref().unwrap().starts_with("tab-sess-"));
+ assert_eq!(params["objectId"], "obj-20");
+ assert_eq!(params["returnByValue"], true);
+ assert_eq!(
+ params["arguments"],
+ json!([{ "value": field }, { "value": wanted }])
+ );
+ assert!(!params["functionDeclaration"]
+ .as_str()
+ .unwrap()
+ .contains(wanted));
+ assert!(recorded_calls(&f, "Runtime.evaluate")
+ .iter()
+ .all(|(_, params)| params["expression"] == "document.visibilityState === 'visible'"));
+ }
+}
+
+#[tokio::test]
+async fn select_requires_exactly_one_of_label_or_value_before_touching_the_page() {
+ for matcher in [json!({}), json!({ "label": "Beta", "value": "b" })] {
+ let f = fixture().await;
+ let s = select_with(&f, "main", "Shadow Input", matcher).await;
+ assert_eq!(s, json!({ "error": true }));
+ assert!(select_scripts(&f).is_empty());
+ }
+}
+
+#[tokio::test]
+async fn select_reports_live_refusals_and_script_failures_as_refused() {
+ for (outcome, message) in [
+ (
+ json!({ "refused": "the ref is not a native HTML select element" }),
+ "the ref is not a native HTML select element",
+ ),
+ (
+ json!({ "refused": "2 options match the requested label; exactly one must" }),
+ "2 options match the requested label; exactly one must",
+ ),
+ (
+ json!({ "refused": "the matching option is disabled" }),
+ "the matching option is disabled",
+ ),
+ (
+ Value::Null,
+ "the select script failed before reporting a selection",
+ ),
+ ] {
+ let f = fixture_with(|st| st.select_outcome = outcome).await;
+ let s = select_with(&f, "main", "Shadow Input", json!({ "label": "Beta" })).await;
+ assert_eq!(s["status"], "refused", "{s}");
+ assert_eq!(s["refusal"]["code"], "browser_action_unavailable");
+ assert_eq!(s["refusal"]["message"], message);
+ }
+}
+
+#[tokio::test]
+async fn select_of_the_current_option_reports_no_change() {
+ let f = fixture_with(|st| {
+ st.select_outcome = json!({ "changed": false, "value": "a", "label": "Alpha" });
+ })
+ .await;
+ let s = select_with(&f, "main", "Shadow Input", json!({ "value": "a" })).await;
+ assert_eq!(s["status"], "ok", "{s}");
+ assert_eq!(s["changed"], false);
+ assert_eq!(s["label"], "Alpha");
+}
+
+#[tokio::test]
+async fn select_in_an_oopif_routes_to_the_child_session() {
+ let f = fixture_with(|st| {
+ st.select_outcome = json!({ "changed": true, "value": "b", "label": "Beta" });
+ })
+ .await;
+ let s = select_with(&f, "oopif", "ad-input", json!({ "label": "Beta" })).await;
+ assert_eq!(s["status"], "ok", "{s}");
+ assert_eq!(s["frame"], "oopif");
+ let scripts = select_scripts(&f);
+ assert_eq!(scripts.len(), 1);
+ assert!(scripts[0].0.as_deref().unwrap().starts_with("oopif-sess-"));
+}
+
// ---- window-less page targets (trycua/cua#3540) ------------------------------------------
//
// Chromium exposes page targets that live in no browser window — an extension
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 7ff2e42c9d..677392578a 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -837,6 +837,7 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {
"browser_navigate" => &["browser.navigate"],
"browser_click" => &["browser.input.click"],
"browser_type" => &["browser.input.type"],
+ "browser_select" => &["browser.input.select"],
"browser_dialog" => &["browser.dialog"],
"browser_set_input_files" => &["browser.input.files"],
"browser_download" => &["browser.download"],
@@ -6392,6 +6393,7 @@ mod capability_tests {
"browser_navigate",
"browser_click",
"browser_type",
+ "browser_select",
"browser_dialog",
"browser_set_input_files",
"browser_download",
@@ -6483,6 +6485,7 @@ mod capability_tests {
"browser.navigate",
"browser.input.click",
"browser.input.type",
+ "browser.input.select",
"browser.input.files",
"browser.dialog",
"browser.download",
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
index 3c2ba4fa3f..037e67246e 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
@@ -2669,6 +2669,48 @@
"name": "browser_type",
"read_only": false
},
+ {
+ "description": "Select one option of a native HTML <select> ref on an exactly-bound tab without opening its popup. Pass exactly one of label or value; exactly one option must equal it. Refuses non-native, multiple-selection, and disabled selects, disabled options, and zero or ambiguous matches before changing anything. A change dispatches bubbling input and change events; they are synthetic, so verify the page accepted the selection. Selecting the option that is already selected dispatches no events.",
+ "destructive": false,
+ "idempotent": false,
+ "input_schema": {
+ "additionalProperties": false,
+ "properties": {
+ "label": {
+ "description": "Exact visible label of the option to select.",
+ "type": "string"
+ },
+ "ref": {
+ "description": "Page element ref in the p<snapshot>:<index> namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.",
+ "type": "string"
+ },
+ "session": {
+ "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+ "type": "string"
+ },
+ "tab_id": {
+ "description": "Opaque tab id from get_browser_state (session-scoped).",
+ "type": "string"
+ },
+ "target_id": {
+ "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
+ "type": "string"
+ },
+ "value": {
+ "description": "Exact value attribute of the option to select.",
+ "type": "string"
+ }
+ },
+ "required": [
+ "target_id",
+ "tab_id",
+ "ref"
+ ],
+ "type": "object"
+ },
+ "name": "browser_select",
+ "read_only": false
+ },
{
"description": "Inspect or resolve a page-owned JavaScript alert, confirm, prompt, or beforeunload dialog on one exactly-bound tab. This never handles browser permission UI, extension UI, native dialogs, or file pickers. Inspect returns an opaque dialog_id; accept/dismiss require that exact current id. Resolution defaults to background delivery; Linux callers must explicitly request foreground delivery because Chromium's native modal cannot be resolved there without changing foreground posture.",
"destructive": false,
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index 6e4e8eb389..455f4422f8 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -2312,6 +2312,48 @@
"name": "browser_type",
"read_only": false
},
+ {
+ "description": "Select one option of a native HTML <select> ref on an exactly-bound tab without opening its popup. Pass exactly one of label or value; exactly one option must equal it. Refuses non-native, multiple-selection, and disabled selects, disabled options, and zero or ambiguous matches before changing anything. A change dispatches bubbling input and change events; they are synthetic, so verify the page accepted the selection. Selecting the option that is already selected dispatches no events.",
+ "destructive": false,
+ "idempotent": false,
+ "input_schema": {
+ "additionalProperties": false,
+ "properties": {
+ "label": {
+ "description": "Exact visible label of the option to select.",
+ "type": "string"
+ },
+ "ref": {
+ "description": "Page element ref in the p<snapshot>:<index> namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.",
+ "type": "string"
+ },
+ "session": {
+ "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+ "type": "string"
+ },
+ "tab_id": {
+ "description": "Opaque tab id from get_browser_state (session-scoped).",
+ "type": "string"
+ },
+ "target_id": {
+ "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
+ "type": "string"
+ },
+ "value": {
+ "description": "Exact value attribute of the option to select.",
+ "type": "string"
+ }
+ },
+ "required": [
+ "target_id",
+ "tab_id",
+ "ref"
+ ],
+ "type": "object"
+ },
+ "name": "browser_select",
+ "read_only": false
+ },
{
"description": "Inspect or resolve a page-owned JavaScript alert, confirm, prompt, or beforeunload dialog on one exactly-bound tab. This never handles browser permission UI, extension UI, native dialogs, or file pickers. Inspect returns an opaque dialog_id; accept/dismiss require that exact current id. Resolution defaults to background delivery; Linux callers must explicitly request foreground delivery because Chromium's native modal cannot be resolved there without changing foreground posture.",
"destructive": false,
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
index 6d2e3b3c6c..b7660e12a2 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
@@ -2364,6 +2364,48 @@
"name": "browser_type",
"read_only": false
},
+ {
+ "description": "Select one option of a native HTML <select> ref on an exactly-bound tab without opening its popup. Pass exactly one of label or value; exactly one option must equal it. Refuses non-native, multiple-selection, and disabled selects, disabled options, and zero or ambiguous matches before changing anything. A change dispatches bubbling input and change events; they are synthetic, so verify the page accepted the selection. Selecting the option that is already selected dispatches no events.",
+ "destructive": false,
+ "idempotent": false,
+ "input_schema": {
+ "additionalProperties": false,
+ "properties": {
+ "label": {
+ "description": "Exact visible label of the option to select.",
+ "type": "string"
+ },
+ "ref": {
+ "description": "Page element ref in the p<snapshot>:<index> namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.",
+ "type": "string"
+ },
+ "session": {
+ "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. Omit it to use the authenticated transport's implicit lifecycle session. Browser targets, tabs, and refs belong to the resolved lifecycle session.",
+ "type": "string"
+ },
+ "tab_id": {
+ "description": "Opaque tab id from get_browser_state (session-scoped).",
+ "type": "string"
+ },
+ "target_id": {
+ "description": "Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).",
+ "type": "string"
+ },
+ "value": {
+ "description": "Exact value attribute of the option to select.",
+ "type": "string"
+ }
+ },
+ "required": [
+ "target_id",
+ "tab_id",
+ "ref"
+ ],
+ "type": "object"
+ },
+ "name": "browser_select",
+ "read_only": false
+ },
{
"description": "Inspect or resolve a page-owned JavaScript alert, confirm, prompt, or beforeunload dialog on one exactly-bound tab. This never handles browser permission UI, extension UI, native dialogs, or file pickers. Inspect returns an opaque dialog_id; accept/dismiss require that exact current id. Resolution defaults to background delivery; Linux callers must explicitly request foreground delivery because Chromium's native modal cannot be resolved there without changing foreground posture.",
"destructive": false,
diff --git a/scripts/docs-generators/cua-driver.ts b/scripts/docs-generators/cua-driver.ts
index 922beb2b64..3e4f8e629a 100644
--- a/scripts/docs-generators/cua-driver.ts
+++ b/scripts/docs-generators/cua-driver.ts
@@ -325,7 +325,7 @@ export const MCP_CATEGORIES: Array<McpCategory & { tools: string[] }> = [
slug: 'browser-input',
title: 'Browser input tools',
summary: 'Exact browser input over CDP: clicks, typing, pointer, dialogs, files and downloads',
- tools: ['browser_click', 'browser_type', 'browser_pointer', 'browser_dialog', 'browser_set_input_files', 'browser_download'],
+ tools: ['browser_click', 'browser_type', 'browser_select', 'browser_pointer', 'browser_dialog', 'browser_set_input_files', 'browser_download'],
},
{
slug: 'sessions',