Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): browser_type can replace a field's content, not only append (#2624)
Manfred + TARSeb0b5a0c72cd
diff --git a/docs/content/docs/how-to-guides/driver/drive-a-web-page.mdx b/docs/content/docs/how-to-guides/driver/drive-a-web-page.mdxindex a3b2078377..142e9933f4 100644--- a/docs/content/docs/how-to-guides/driver/drive-a-web-page.mdx+++ b/docs/content/docs/how-to-guides/driver/drive-a-web-page.mdx@@ -331,6 +331,20 @@ browser_type({The default `insert_text` mode is efficient for ordinary text. Use`mode: "keystrokes"` when the page depends on per-character keyboard events.+Both modes insert at the current selection. To replace a pre-filled value,+pass `replace: true`. An empty `text` with `replace: true` clears the field+through normal input events:++```jsonc+browser_type({+ "target_id": "<target_id>",+ "tab_id": "<tab_id>",+ "ref": "p2:4",+ "text": "new value",+ "replace": true,+ "session": "research-1"+})+```## Use extended pointer actionsdiff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdxindex 9b831098dc..003fff4eec 100644--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx@@ -810,12 +810,13 @@ Click a page element (by ref) or viewport coordinates in an exactly-bound tab. D### `browser_type`-Type text into an exactly-bound tab via the Input domain. mode="insert_text" (default) uses Input.insertText; mode="keystrokes" dispatches per-character key events. Pass a ref to an editable element from the latest snapshot. A ref is required; heuristic bindings are refused.+Type text into an exactly-bound tab via the Input domain. mode="insert_text" (default) uses Input.insertText; mode="keystrokes" dispatches per-character key events. Both insert at the caret, so typing into a field that already holds text appends to it; pass replace=true to set the field instead, or to clear it by typing an empty string. Pass a ref to an editable element from the latest snapshot. A ref is required; heuristic bindings are refused.**Arguments:**- `mode` (string, optional): insert_text (default): bulk Input.insertText. keystrokes: per-character Input.dispatchKeyEvent.- `ref` (string, required): Page element ref in the p<snapshot>:<index> namespace from get_browser_state. Refs are invalidated by navigation and by newer snapshots of the same tab.+- `replace` (boolean, optional): false (default): insert at the caret, appending to whatever the field already holds. true: select the element's whole content first so the text replaces it — with an empty text this clears the field. Replacement goes through the selection, so beforeinput/input still fire and framework state stays consistent.- `session` (string, optional): Stable caller-declared session id. Browser targets, tabs, and refs are scoped to this session.- `tab_id` (string, required): Opaque tab id from get_browser_state (session-scoped).- `target_id` (string, required): Opaque browser target id minted by get_browser_state (session-scoped; never a CDP id).diff --git a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.mdindex 2f932b8324..9f7f873274 100644--- a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md+++ b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md@@ -297,8 +297,11 @@ cua-driver browser_type \````insert_text` is the default bulk insertion route. Use `keystrokes` only when-the page requires per-character key events. Inspect the live schema when in-doubt:+the page requires per-character key events. Both modes insert at the current+selection. When a field already contains text, pass `"replace":true` to select+its complete value first. Passing an empty `text` with `replace:true` clears+the field while preserving normal input events. Inspect the live schema when+in doubt:```bashcua-driver describe browser_typediff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex bb2be1bd9c..fce9fc8561 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -15,6 +15,7 @@ use crate::tool::{ProtectedResourceOwnership, Tool, ToolDef, ToolRegistry};use crate::tool_args::ArgsExt;use super::approval::MCP_HOST_APPROVAL_ARG;+use super::cdp_ws::CdpConnection;use super::download::BrowserDownloadTool;use super::engine::{BrowserEngine, BrowserTabScreenshot};use super::platform::{@@ -1331,6 +1332,145 @@ const FOCUS_EMULATION_READY_CHECK: &str = "function() { \return document.hasFocus() && active === this; \}";+// Select the element's whole content so the next insertion replaces it instead+// of appending. Returns the number of characters selected, or -1 when the node+// is not a shape we know how to select. Selection is the only clearing path+// that keeps input semantics intact: assigning `value` directly would skip the+// beforeinput/input events that frameworks bind their state to.+const SELECT_ALL_IN_ELEMENT: &str = "function() { \+ if (this.tagName === 'INPUT' || this.tagName === 'TEXTAREA') { \+ const value = this.value || ''; \+ try { this.setSelectionRange(0, value.length); } catch (e) { \+ try { this.select(); } catch (_) { return -1; } \+ } \+ if (this.selectionStart !== 0 || this.selectionEnd !== value.length) return -1; \+ return Array.from(value).length; \+ } \+ if (this.isContentEditable) { \+ const root = this.getRootNode(); \+ const owner = ('getSelection' in root) ? root : this.ownerDocument; \+ const selection = owner.getSelection(); \+ if (!selection) return -1; \+ const range = this.ownerDocument.createRange(); \+ range.selectNodeContents(this); \+ selection.removeAllRanges(); \+ selection.addRange(range); \+ return Array.from(this.textContent || '').length; \+ } \+ return -1; \+}";++/// Put the element's entire content into the selection.+///+/// `browser_type` delivers through `Input.insertText` and the trusted key path,+/// and both insert at the caret. Without an explicit selection there is no way+/// to *set* a field that already holds text — a caller who tries ends up with+/// the old and the new value concatenated. Both delivery paths replace the+/// current selection, so selecting everything first turns "insert" into "set".+async fn select_all_in_element(+ conn: &CdpConnection,+ cdp: &str,+ object_id: &str,+) -> Result<usize, String> {+ let selected = conn+ .call(+ Some(cdp),+ "Runtime.callFunctionOn",+ json!({+ "objectId": object_id,+ "functionDeclaration": SELECT_ALL_IN_ELEMENT,+ "returnByValue": true,+ }),+ )+ .await+ .map_err(|error| error.to_string())?;+ match selected["result"]["value"].as_i64() {+ Some(n) if n >= 0 => Ok(n as usize),+ _ => Err("the ref is not a text input, textarea, or contenteditable \+ element, so its content cannot be selected for replacement"+ .into()),+ }+}++/// Establish Chromium's trusted-input focus state for an inactive tab.+///+/// Selection alone is not durable in a fully occluded window: without focus+/// emulation Chromium can accept `Input.insertText` while discarding the+/// selection, silently turning replacement back into append.+async fn enter_focus_emulation(+ conn: &CdpConnection,+ cdp: &str,+ backend_node_id: i64,+ object_id: &str,+) -> Result<(), String> {+ conn.call(+ Some(cdp),+ "Emulation.setFocusEmulationEnabled",+ json!({ "enabled": true }),+ )+ .await+ .map_err(|error| error.to_string())?;++ let mut focus_error = None;+ for _ in 0..20 {+ if let Err(error) = conn+ .call(+ Some(cdp),+ "DOM.focus",+ json!({ "backendNodeId": backend_node_id }),+ )+ .await+ {+ focus_error = Some(error.to_string());+ break;+ }+ let ready = conn+ .call(+ Some(cdp),+ "Runtime.callFunctionOn",+ json!({+ "objectId": object_id,+ "functionDeclaration": FOCUS_EMULATION_READY_CHECK,+ "returnByValue": true,+ }),+ )+ .await;+ if matches!(+ ready,+ Ok(ref value) if value["result"]["value"].as_bool() == Some(true)+ ) {+ tokio::time::sleep(std::time::Duration::from_millis(100)).await;+ if let Err(error) = conn+ .call(+ Some(cdp),+ "DOM.focus",+ json!({ "backendNodeId": backend_node_id }),+ )+ .await+ {+ focus_error = Some(error.to_string());+ break;+ }+ return Ok(());+ }+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;+ }++ let _ = conn+ .call(+ Some(cdp),+ "Emulation.setFocusEmulationEnabled",+ json!({ "enabled": false }),+ )+ .await;+ Err(format!(+ "the exact editable ref did not become focus-ready under CDP emulation{}",+ focus_error+ .map(|error| format!(": {error}"))+ .unwrap_or_default()+ ))+}+pub struct BrowserTypeTool {def: ToolDef,engine: Arc<BrowserEngine>,@@ -1342,9 +1482,11 @@ impl BrowserTypeTool {name: "browser_type".into(),description: "Type text into an exactly-bound tab via the Input domain. \mode=\"insert_text\" (default) uses Input.insertText; \- mode=\"keystrokes\" dispatches per-character key events. Pass a ref \- to an editable element from the latest snapshot. A ref is required; \- heuristic bindings are refused."+ mode=\"keystrokes\" dispatches per-character key events. Both insert \+ at the caret, so typing into a field that already holds text appends \+ to it; pass replace=true to set the field instead, or to clear it by \+ typing an empty string. Pass a ref to an editable element from the \+ latest snapshot. A ref is required; heuristic bindings are refused.".into(),input_schema: json!({"type": "object",@@ -1360,6 +1502,15 @@ impl BrowserTypeTool {"description": "insert_text (default): bulk Input.insertText. \keystrokes: per-character Input.dispatchKeyEvent."},+ "replace": {+ "type": "boolean",+ "description": "false (default): insert at the caret, appending \+ to whatever the field already holds. true: select the \+ element's whole content first so the text replaces it — \+ with an empty text this clears the field. Replacement goes \+ through the selection, so beforeinput/input still fire and \+ framework state stays consistent."+ },},"required": ["target_id", "tab_id", "ref", "text"],"additionalProperties": true@@ -1422,6 +1573,7 @@ impl Tool for BrowserTypeTool {"mode must be \"insert_text\" or \"keystrokes\", got {mode:?}"));}+ let replace = args.opt_bool("replace").unwrap_or(false);let _mutation = match self.engine@@ -1568,11 +1720,93 @@ impl Tool for BrowserTypeTool {}let requested_chars = text.chars().count();+ let mut replaced_chars = 0usize;let (typed, delivered_chars) = if mode == "insert_text" {- match conn- .call(Some(cdp), "Input.insertText", json!({ "text": text }))- .await- {+ if replace {+ if let Err(detail) =+ enter_focus_emulation(conn, cdp, entry.backend_node_id, &object_id).await+ {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserInputTrustUnavailable,+ format!(+ "the inactive tab could not prepare trusted replacement input: {detail}"+ ),+ )+ .to_tool_result();+ }+ match select_all_in_element(conn, cdp, &object_id).await {+ Ok(n) => replaced_chars = n,+ Err(detail) => {+ let _ = conn+ .call(+ Some(cdp),+ "Emulation.setFocusEmulationEnabled",+ json!({ "enabled": false }),+ )+ .await;+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ format!("replace=true could not select the ref's content: {detail}"),+ )+ .to_tool_result();+ }+ }+ }+ // An empty insertText is a no-op, so "clear the field" needs an+ // explicit deletion of the selection we just made. Delete keeps the+ // same event path as typing; it is not a value assignment.+ let mut call = if replace && text.is_empty() {+ if replaced_chars == 0 {+ Ok(json!({}))+ } else {+ match conn+ .call(+ Some(cdp),+ "Input.dispatchKeyEvent",+ json!({+ "type": "keyDown",+ "key": "Delete",+ "code": "Delete",+ "windowsVirtualKeyCode": 46,+ "nativeVirtualKeyCode": 46,+ }),+ )+ .await+ {+ Ok(_) => {+ conn.call(+ Some(cdp),+ "Input.dispatchKeyEvent",+ json!({+ "type": "keyUp",+ "key": "Delete",+ "code": "Delete",+ "windowsVirtualKeyCode": 46,+ "nativeVirtualKeyCode": 46,+ }),+ )+ .await+ }+ Err(error) => Err(error),+ }+ }+ } else {+ conn.call(Some(cdp), "Input.insertText", json!({ "text": text }))+ .await+ };+ if replace {+ if let Err(error) = conn+ .call(+ Some(cdp),+ "Emulation.setFocusEmulationEnabled",+ json!({ "enabled": false }),+ )+ .await+ {+ call = Err(error);+ }+ }+ match call {Ok(_) => (Ok(()), requested_chars),Err(error) => (Err(error), 0),}@@ -1672,8 +1906,56 @@ impl Tool for BrowserTypeTool {).to_tool_result();}+ // Select AFTER the last DOM.focus above: re-focusing a node drops+ // the selection again, so selecting any earlier would silently+ // degrade replace=true back to append.+ if replace {+ match select_all_in_element(conn, cdp, &object_id).await {+ Ok(n) => replaced_chars = n,+ Err(detail) => {+ // Leave focus emulation the way we found it, exactly as+ // the other early returns in this branch do.+ let _ = conn+ .call(+ Some(cdp),+ "Emulation.setFocusEmulationEnabled",+ json!({ "enabled": false }),+ )+ .await;+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ format!("replace=true could not select the ref's content: {detail}"),+ )+ .to_tool_result();+ }+ }+ }let mut result = Ok(());let mut delivered = 0;+ // No characters to type means the selection has to go away by+ // itself; the loop below would leave the old content selected but+ // present, and "cleared" would be a false report.+ if replace && text.is_empty() && replaced_chars > 0 {+ for phase in ["keyDown", "keyUp"] {+ if let Err(error) = conn+ .call(+ Some(cdp),+ "Input.dispatchKeyEvent",+ json!({+ "type": phase,+ "key": "Delete",+ "code": "Delete",+ "windowsVirtualKeyCode": 46,+ "nativeVirtualKeyCode": 46,+ }),+ )+ .await+ {+ result = Err(error);+ break;+ }+ }+ }for ch in text.chars() {let (key, key_text) = if ch == '\n' {("Enter".to_string(), "\r".to_string())@@ -1730,10 +2012,14 @@ impl Tool for BrowserTypeTool {};match typed {- Ok(()) => ToolResult::text(format!(- "typed {} char(s) into {tab_id}",- requested_chars- ))+ Ok(()) => ToolResult::text(if replace {+ format!(+ "typed {requested_chars} char(s) into {tab_id}, replacing \+ {replaced_chars} char(s)"+ )+ } else {+ format!("typed {requested_chars} char(s) into {tab_id}")+ }).with_structured(json!({"status": "ok","target_id": target_id,@@ -1744,6 +2030,11 @@ impl Tool for BrowserTypeTool {"chars": requested_chars,"requested_chars": requested_chars,"delivered_chars": delivered_chars,+ // Report what was displaced, not just what was sent: a caller+ // that asked to replace needs to distinguish "set an empty+ // field" from "overwrote something" without re-reading the page.+ "replace": replace,+ "replaced_chars": replaced_chars,})),Err(e) => BrowserRefusal::new(BrowserRefusalCode::BrowserInputIncomplete,diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rsindex d213e571b6..bde023927c 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs@@ -3728,6 +3728,106 @@ fn settle_between_browser_rows() {thread::sleep(Duration::from_secs(2));}+fn run_type_replace(spec: &BrowserSpec) {+ let scenario = format!(+ "{}-{}-standalone-type-replace",+ std::env::consts::OS,+ spec.name+ );+ execute_case(case(&spec.name, "browser_type_replace"), |evidence| {+ let mut fixture = launch_browser(spec, &scenario);+ *evidence = recording_evidence(fixture.driver.recording_dir());+ run_with_background_oracles(&mut fixture, |fixture| {+ let session = format!("standalone-type-replace-{}", fixture.pid);+ let (target, tab, snapshot) = bind(fixture, &session);+ let input_ref = ref_by_label(&snapshot, "id=txt-input");+ let number_input_ref = ref_by_label(&snapshot, "id=number-input");++ let mut type_text = |text: &str, replace: Option<bool>, mode: Option<&str>| {+ let mut args = serde_json::json!({+ "target_id": target,+ "tab_id": tab,+ "ref": input_ref,+ "text": text,+ "session": session,+ });+ if let Some(replace) = replace {+ args["replace"] = serde_json::json!(replace);+ }+ if let Some(mode) = mode {+ args["mode"] = serde_json::json!(mode);+ }+ let response = fixture.driver.call("browser_type", args);+ assert_eq!(response.structured()["status"], "ok", "{}", response.raw);+ response+ };++ // The default must keep appending. This is the control: without it+ // a passing replace case could simply mean the tool always sets.+ type_text("first", None, None);+ wait_for_value(&fixture.server, "txt-input", "first");+ type_text("second", None, None);+ wait_for_value(&fixture.server, "txt-input", "firstsecond");++ // replace=true sets the field instead of extending it, and reports+ // how much it displaced so the caller need not re-read the page.+ let replaced = type_text("third🙂", Some(true), None);+ wait_for_value(&fixture.server, "txt-input", "third🙂");+ assert_eq!(replaced.structured()["replace"], true, "{}", replaced.raw);+ assert_eq!(+ replaced.structured()["replaced_chars"],+ 11,+ "replaced_chars must count the displaced text: {}",+ replaced.raw+ );++ // The trusted keystroke path replaces through the same selection.+ let replaced_unicode = type_text("fourth", Some(true), Some("keystrokes"));+ wait_for_value(&fixture.server, "txt-input", "fourth");+ assert_eq!(+ replaced_unicode.structured()["replaced_chars"],+ 6,+ "replaced_chars must count Unicode scalar values like requested_chars: {}",+ replaced_unicode.raw+ );++ // Empty text with replace=true is the only way to clear a field.+ let cleared = type_text("", Some(true), None);+ wait_for_value(&fixture.server, "txt-input", "");+ assert_eq!(+ cleared.structured()["replaced_chars"],+ 6,+ "clearing must report what it removed: {}",+ cleared.raw+ );++ // Input types without a real selection API must fail closed. If+ // this silently proceeded, Input.insertText would append and turn+ // 42 into 427 while reporting a successful replacement.+ let unsupported = fixture.driver.call(+ "browser_type",+ serde_json::json!({+ "target_id": target,+ "tab_id": tab,+ "ref": number_input_ref,+ "text": "7",+ "replace": true,+ "session": session,+ }),+ );+ assert_eq!(+ unsupported.structured()["refusal"]["code"],+ "browser_action_unavailable",+ "{}",+ unsupported.raw+ );+ wait_for_value(&fixture.server, "number-input", "42");++ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())+ })+ });+}+fn run_browser_scenario(run: fn(&BrowserSpec)) {let _guard = STANDALONE_BROWSER_TEST_LOCK.lock()@@ -3774,6 +3874,7 @@ macro_rules! standalone_browser_test {standalone_browser_test!(standalone_browser_roundtrip, run_roundtrip);standalone_browser_test!(standalone_browser_semantic_state, run_semantic_state);standalone_browser_test!(standalone_browser_background_type, run_background_type);+standalone_browser_test!(standalone_browser_type_replace, run_type_replace);#[cfg(target_os = "macos")]standalone_browser_test!(standalone_browser_native_omnibox_select_all,diff --git a/libs/cua-driver/tests/fixtures/shared/web/index.html b/libs/cua-driver/tests/fixtures/shared/web/index.htmlindex 77807c8eb8..6439929efb 100644--- a/libs/cua-driver/tests/fixtures/shared/web/index.html+++ b/libs/cua-driver/tests/fixtures/shared/web/index.html@@ -72,6 +72,8 @@aria-label="txt-input" placeholder="type here"autocapitalize="off" autocorrect="off" spellcheck="false" /><span class="mirror" id="lbl-input-mirror" data-cua-id="lbl-input-mirror">mirror=</span>+ <input type="number" id="number-input" data-cua-id="number-input"+ aria-label="number-input" value="42" /></div></fieldset>diff --git a/scripts/ci/run-rust-standalone-browser-e2e.sh b/scripts/ci/run-rust-standalone-browser-e2e.shindex 1afb7f35f8..cf2dca15dc 100755--- a/scripts/ci/run-rust-standalone-browser-e2e.sh+++ b/scripts/ci/run-rust-standalone-browser-e2e.sh@@ -107,6 +107,7 @@ if [[ "${HOST_OS}" == Linux && "${CUA_E2E_WAYLAND_SESSION:-}" == generic ]]; theelsetests=(standalone_browser_background_type+ standalone_browser_type_replacestandalone_browser_dialogs)if [[ "${HOST_OS}" == Linux ]]; thendiff --git a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1 b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1index e3570488f1..ad0008f15d 100644--- a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1+++ b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1@@ -119,6 +119,7 @@ if (-not (Test-Path $env:CUA_TEST_DRIVER_BIN)) {$tests = @("standalone_browser_background_type",+ "standalone_browser_type_replace","standalone_browser_dialogs","standalone_browser_download","standalone_browser_existing_profile",