Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): wait for generic type_text completion (#2463)
Francesco Bonacci3858c7f2aafc
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rsindex 0e6afdd757..448ba712c4 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs@@ -15,6 +15,8 @@ use std::thread;use std::time::{Duration, Instant};use base64::Engine as _;+#[cfg(target_os = "macos")]+use cua_driver_testkit::ax::element_index_containing;use cua_driver_testkit::e2e::{append_json_line, execute_case, recording_evidence, write_environment_from_env, CaseSpec,Delivery, DriverRoute, EnvironmentRecord, Evidence, Observation, OracleKind, RefusalCode,@@ -47,6 +49,18 @@ fn standalone_fixture_html() -> String {)}+#[cfg(target_os = "macos")]+fn standalone_generic_type_text_html() -> String {+ standalone_fixture_html().replace(+ r#"<main class="harness-grid">"#,+ r#"<div id="generic-long-editor" data-cua-id="generic-long-editor"+ aria-label="generic-long-editor" contenteditable="true"+ autocapitalize="off" autocorrect="off" spellcheck="false"+ style="min-height:2em;border:1px solid #888;padding:4px"></div>+<main class="harness-grid">"#,+ )+}+fn standalone_browser_completeness_html() -> String {standalone_fixture_html().replace("</body>",@@ -1253,6 +1267,30 @@ fn foreground_page_case(browser: &str, action: &str) -> CaseSpec {)}+#[cfg(target_os = "macos")]+fn generic_type_text_case(browser: &str) -> CaseSpec {+ CaseSpec::delivered(+ format!(+ "{}-{browser}-standalone-generic-type-text-completion",+ std::env::consts::OS+ ),+ browser,+ "standalone-chromium",+ "generic_type_text_completion",+ Targeting::Px,+ Delivery::Background,+ Scope::Window,+ DriverRoute::MacosCgEventPid,+ vec![+ OracleKind::FixtureState,+ OracleKind::Focus,+ OracleKind::ZOrder,+ OracleKind::NoLeakedInput,+ OracleKind::Cursor,+ ],+ )+}+fn run_with_background_oracles(fixture: &mut BrowserFixture,action: impl FnOnce(&mut BrowserFixture) -> Observation,@@ -1481,6 +1519,114 @@ fn run_background_type(spec: &BrowserSpec) {});}+#[cfg(target_os = "macos")]+fn run_generic_type_text_completion(spec: &BrowserSpec) {+ let scenario = format!(+ "{}-{}-standalone-generic-type-text-completion",+ std::env::consts::OS,+ spec.name+ );+ execute_case(generic_type_text_case(&spec.name), |evidence| {+ let mut fixture =+ launch_browser_with_html(spec, &scenario, standalone_generic_type_text_html());+ *evidence = recording_evidence(fixture.driver.recording_dir());+ run_with_background_oracles(&mut fixture, |fixture| {+ let state = fixture.driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "capture_mode": "vision",+ }),+ );+ assert!(!state.is_error(), "native browser snapshot: {}", state.raw);+ let index = element_index_containing(state.tree_text(), "generic-long-editor")+ .expect("generic long editor must be present in the native AX tree");+ let elements = state.structured()["elements"]+ .as_array()+ .expect("native snapshot elements");+ let editor_frame = elements+ .iter()+ .find(|element| element["element_index"].as_u64() == Some(index))+ .and_then(|element| element["frame"].as_object())+ .expect("generic long editor frame");+ let window_frame = elements+ .iter()+ .find(|element| element["role"].as_str() == Some("AXWindow"))+ .and_then(|element| element["frame"].as_object())+ .expect("browser AXWindow frame");+ let window_x = window_frame["x"].as_f64().expect("window x");+ let window_y = window_frame["y"].as_f64().expect("window y");+ let window_width = window_frame["w"].as_f64().expect("window width");+ let scale = state.structured()["screenshot_width"]+ .as_f64()+ .expect("screenshot width")+ / window_width;+ let x = (editor_frame["x"].as_f64().expect("editor x")+ + editor_frame["w"].as_f64().expect("editor width") / 2.0+ - window_x)+ * scale;+ let y = (editor_frame["y"].as_f64().expect("editor y")+ + editor_frame["h"].as_f64().expect("editor height") / 2.0+ - window_y)+ * scale;++ let payload = format!("BEGIN-{}-END", "0123456789abcdef".repeat(52));+ let requested_chars = payload.chars().count();+ let typed = fixture.driver.call(+ "type_text",+ serde_json::json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "x": x,+ "y": y,+ "text": payload,+ "delay_ms": 0,+ }),+ );+ assert!(+ !typed.is_error(),+ "generic long type_text failed: {}",+ typed.raw+ );+ assert_eq!(+ typed.structured()["requested_chars"].as_u64(),+ Some(requested_chars as u64),+ "{}",+ typed.raw+ );+ assert_eq!(+ typed.structured()["delivered_chars"].as_u64(),+ Some(requested_chars as u64),+ "{}",+ typed.raw+ );++ // Read the DOM synchronously after type_text returns. No fixture+ // polling is allowed here: the tool's completion is the behavior+ // under test, and the distinctive END sentinel proves the queued+ // renderer tail was consumed before it returned.+ let ws_url = cdp_page_websocket_for_url(fixture.cdp_port, fixture.server.page_url());+ let value = harness_cdp_call_at_url(+ &ws_url,+ "Runtime.evaluate",+ serde_json::json!({+ "expression": "document.getElementById('generic-long-editor').innerText",+ "returnByValue": true,+ }),+ )["result"]["value"]+ .as_str()+ .expect("generic long editor DOM value")+ .to_owned();+ assert_eq!(value.chars().count(), requested_chars);+ assert!(value.ends_with("-END"), "missing END sentinel: {value:?}");+ assert_eq!(value, payload);++ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())+ })+ });+}+fn run_trusted_click(spec: &BrowserSpec) {let scenario = format!("{}-{}-standalone-trusted-click",@@ -3310,6 +3456,11 @@ macro_rules! standalone_browser_test {standalone_browser_test!(standalone_browser_roundtrip, run_roundtrip);standalone_browser_test!(standalone_browser_semantic_state, run_semantic_state);standalone_browser_test!(standalone_browser_background_type, run_background_type);+#[cfg(target_os = "macos")]+standalone_browser_test!(+ standalone_browser_generic_type_text_completion,+ run_generic_type_text_completion+);standalone_browser_test!(standalone_browser_trusted_click, run_trusted_click);standalone_browser_test!(standalone_browser_prepare_isolated,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rsindex eddd97ce79..514fb9ff52 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs@@ -305,7 +305,29 @@ impl Tool for TypeTextTool {let changes = super::finish_window_observation(snapshot, &args).await;match result {- Ok(Ok((detail, path, verified))) => {+ Ok(Ok(outcome)) if outcome.delivered_chars.is_some_and(|n| n < char_count) => {+ let delivered_chars = outcome.delivered_chars.unwrap_or_default();+ ToolResult::error(format!(+ "type_text incomplete: delivered {delivered_chars} of {char_count} character(s){}; retry only the remaining suffix",+ outcome.detail+ ))+ .with_structured(serde_json::json!({+ "code": "type_text_incomplete",+ "path": outcome.path,+ "effect": "partial",+ "requested_chars": char_count,+ "delivered_chars": delivered_chars,+ "retryable": true,+ "retry_from_character": delivered_chars,+ }))+ }+ Ok(Ok(outcome)) => {+ let TypeTextOutcome {+ detail,+ path,+ verified,+ delivered_chars,+ } = outcome;// SURFACE-AWARE VERIFICATION. On any web-content surface —// Chromium/WebKit/Electron — the AX layer accepts a write and// echoes it straight back through `AXValue` while the renderer/DOM@@ -363,9 +385,13 @@ impl Tool for TypeTextTool {let mut s = serde_json::json!({"path": path,"characters": char_count,+ "requested_chars": char_count,"verified": verified,"effect": if verified { "confirmed" } else { "unverifiable" },});+ if let Some(delivered_chars) = delivered_chars {+ s["delivered_chars"] = serde_json::json!(delivered_chars);+ }if ax_echo_surface {// Web-content AX echo. A real browser TAB → the `page` tool// (drives the DOM via CDP) is the reliable rung; an embedded@@ -425,6 +451,26 @@ const PATH_AX: &str = "ax";const PATH_KEY_EVENTS: &str = "key_events";const PATH_KEY_EVENTS_FG: &str = "key_events_fg";+const DELIVERY_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);+const DELIVERY_DRAIN_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10);++struct TypeTextOutcome {+ detail: String,+ path: &'static str,+ verified: bool,+ /// Exact when AX exposed the target value. `None` means delivery could not+ /// be observed, so the existing unverifiable contract remains in force.+ delivered_chars: Option<usize>,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum TypedProgress {+ Complete,+ Partial(usize),+ Unchanged,+ Unverifiable,+}+fn foreground_settle_ms(pid: i32, frontmost_pid: Option<i32>) -> u64 {if frontmost_pid == Some(pid) {20@@ -439,18 +485,42 @@ fn foreground_settle_ms(pid: i32, frontmost_pid: Option<i32>) -> u64 {/// the keystrokes. Returns whether we can *positively confirm* the text landed:/// - unreadable `after` (`None`) → unverifiable → `false` (Catalyst case; the/// agent must confirm via screenshot).-/// - `after` contains the text, or grew vs `before` → `true`.+/// - `after` contains the complete text → `true`./// - empty input text → trivially `true`.////// Apps that normalize input (smart quotes, autocomplete) may fail the/// substring/length test even though something landed — we report `false`/// (unverified) rather than erroring, so the agent can still confirm.fn verify_typed(before: Option<&str>, after: Option<&str>, text: &str) -> bool {+ matches!(typed_progress(before, after, text), TypedProgress::Complete)+}++/// Classify an observable insertion without mistaking a prefix for complete+/// delivery. A positive length delta is an exact delivered-character count for+/// insert-at-cursor typing; it is capped at the request size defensively.+fn typed_progress(before: Option<&str>, after: Option<&str>, text: &str) -> TypedProgress {if text.is_empty() {- return true;+ return TypedProgress::Complete;+ }+ let Some(after) = after else {+ return TypedProgress::Unverifiable;+ };+ if after.contains(text) {+ return TypedProgress::Complete;+ }+ let Some(before) = before else {+ return TypedProgress::Unverifiable;+ };+ let delivered = after+ .chars()+ .count()+ .saturating_sub(before.chars().count())+ .min(text.chars().count());+ if delivered == 0 {+ TypedProgress::Unchanged+ } else {+ TypedProgress::Partial(delivered)}- let Some(after) = after else { return false };- after.contains(text) || before.is_some_and(|b| after.chars().count() > b.chars().count())}/// Read the focused/target field's `AXValue`, for before/after read-back.@@ -537,7 +607,7 @@ fn cgevent_type_verified(before: Option<&str>,element_ptr_and_idx: Option<(usize, Option<usize>)>,settle_ms: u64,-) -> anyhow::Result<bool> {+) -> anyhow::Result<(bool, Option<usize>)> {// Focus the target element first so the keystrokes land in IT. Critical in// foreground mode: a freshly-fronted window's keyboard focus may be on the// search box or nowhere, so without this the text goes into the void (or the@@ -556,8 +626,45 @@ fn cgevent_type_verified(std::thread::sleep(std::time::Duration::from_millis(settle_ms));}crate::input::keyboard::type_text_with_delay(pid, text, delay_ms)?;- let after = read_axvalue(pid, element_ptr_and_idx);- Ok(verify_typed(before, after.as_deref(), text))++ // CGEvent posting is asynchronous with respect to the renderer. In+ // particular, Chromium can acknowledge the posting process while a long+ // tail remains queued. Poll the AX value until the complete payload is+ // visible instead of treating any growth as success. If the deadline+ // expires after observable growth, surface the exact partial count.+ let deadline = std::time::Instant::now() + DELIVERY_DRAIN_TIMEOUT;+ Ok(await_typed_delivery(before, text, deadline, || {+ read_axvalue(pid, element_ptr_and_idx)+ }))+}++fn await_typed_delivery(+ before: Option<&str>,+ text: &str,+ deadline: std::time::Instant,+ mut read_value: impl FnMut() -> Option<String>,+) -> (bool, Option<usize>) {+ let mut best_partial = None;+ loop {+ let after = read_value();+ match typed_progress(before, after.as_deref(), text) {+ TypedProgress::Complete => return (true, Some(text.chars().count())),+ TypedProgress::Partial(delivered) => {+ best_partial =+ Some(best_partial.map_or(delivered, |best: usize| best.max(delivered)));+ }+ TypedProgress::Unverifiable => return (false, None),+ TypedProgress::Unchanged => {+ // A readable unchanged value is an observed zero-character+ // delivery, not an unverifiable success.+ best_partial.get_or_insert(0);+ }+ }+ if std::time::Instant::now() >= deadline {+ return (false, best_partial);+ }+ std::thread::sleep(DELIVERY_DRAIN_POLL_INTERVAL);+ }}/// Best-effort-background ladder for `type_text`.@@ -578,7 +685,7 @@ fn type_text_blocking(is_terminal_target: bool,delivery_mode: super::DeliveryMode,window_id: Option<u32>,-) -> anyhow::Result<(String, &'static str, bool)> {+) -> anyhow::Result<TypeTextOutcome> {// Original field value before any rung drives read-back verification only.// An unreadable value is not evidence that the field is empty.let before = read_axvalue(pid, element_ptr_and_idx);@@ -604,22 +711,22 @@ fn type_text_blocking(foreground_settle_ms,)};- let (verified, fronted) = match window_id {+ let ((verified, delivered_chars), fronted) = match window_id {Some(wid) => {// Front → type → restore. The closure returns the read-back// result; with_foreground_assist returns whether it actually// fronted (Ok(false) when the fronting SPIs are unavailable —// the keystrokes still ran, just as background input).- let mut typed_verified = false;+ let mut typed_delivery = (false, None);let fronted = crate::input::skylight::with_foreground_assist(pid as libc::pid_t,wid,|| {- typed_verified = do_type()?;+ typed_delivery = do_type()?;Ok(())},)?;- (typed_verified, fronted)+ (typed_delivery, fronted)}// No window to front — best-effort background keystrokes instead.None => (do_type()?, false),@@ -627,15 +734,16 @@ fn type_text_blocking(// Only claim the `_fg` path when a front actually happened; when no// foregrounding occurred (no window, or SPIs unavailable) these were// background keystrokes and `path` must say so honestly.- return Ok((- format!(" via foreground keystrokes ({delay_ms}ms delay)"),- if fronted {+ return Ok(TypeTextOutcome {+ detail: format!(" via foreground keystrokes ({delay_ms}ms delay)"),+ path: if fronted {PATH_KEY_EVENTS_FG} else {PATH_KEY_EVENTS},+ delivered_chars,verified,- ));+ });}// --- Background rung 0: terminal emulator → CGEvent only (AX is dropped). ---@@ -644,7 +752,7 @@ fn type_text_blocking("type_text: pid {pid} is a terminal emulator; skipping AX value-set, \using CGEvent key-event synthesis");- let verified = cgevent_type_verified(+ let (verified, delivered_chars) = cgevent_type_verified(pid,text,delay_ms,@@ -652,11 +760,12 @@ fn type_text_blocking(element_ptr_and_idx,/*settle_ms=*/ 0,)?;- return Ok((- format!(" via CGEvent (terminal emulator, {delay_ms}ms delay)"),- PATH_KEY_EVENTS,+ return Ok(TypeTextOutcome {+ detail: format!(" via CGEvent (terminal emulator, {delay_ms}ms delay)"),+ path: PATH_KEY_EVENTS,verified,- ));+ delivered_chars,+ });}// --- Background rung 1: AX SelectedText write (element or focused). ---@@ -684,7 +793,12 @@ fn type_text_blocking(}if ax_landed {let idx_str = idx_opt.map(|i| format!(" [{i}]")).unwrap_or_default();- return Ok((format!(" into{idx_str} {role} \"{title}\""), PATH_AX, true));+ return Ok(TypeTextOutcome {+ detail: format!(" into{idx_str} {role} \"{title}\""),+ path: PATH_AX,+ verified: true,+ delivered_chars: Some(text.chars().count()),+ });}tracing::debug!("AX write did not land for {role} \"{title}\" (err={err}); \@@ -697,7 +811,7 @@ fn type_text_blocking(// --- Background rung 2: CGEvent keystrokes with read-back. ---// Never clear here: a partial AX write is rare, and clearing would violate// insert-at-cursor semantics.- let verified = cgevent_type_verified(+ let (verified, delivered_chars) = cgevent_type_verified(pid,text,delay_ms,@@ -705,11 +819,12 @@ fn type_text_blocking(element_ptr_and_idx,/*settle_ms=*/ 0,)?;- Ok((- format!(" via CGEvent ({delay_ms}ms delay)"),- PATH_KEY_EVENTS,+ Ok(TypeTextOutcome {+ detail: format!(" via CGEvent ({delay_ms}ms delay)"),+ path: PATH_KEY_EVENTS,verified,- ))+ delivered_chars,+ })}#[cfg(test)]@@ -757,10 +872,54 @@ mod tests {}#[test]- fn verify_typed_contains_or_grew_is_verified() {+ fn verify_typed_contains_full_request_is_verified() {assert!(verify_typed(Some(""), Some("hi"), "hi")); // containsassert!(verify_typed(Some("ab"), Some("ab hi"), "hi")); // contains, appended- assert!(verify_typed(Some("ab"), Some("abXY"), "??")); // grew vs before+ }++ #[test]+ fn observable_prefix_is_partial_not_verified() {+ assert_eq!(+ typed_progress(Some(""), Some("BEGINpayload"), "BEGINpayloadEND"),+ TypedProgress::Partial(12)+ );+ assert!(!verify_typed(+ Some(""),+ Some("BEGINpayload"),+ "BEGINpayloadEND"+ ));+ }++ #[test]+ fn delivery_waits_through_a_partial_readback_until_complete() {+ let text = "BEGIN-payload-END";+ let mut values = std::collections::VecDeque::from([+ Some("BEGIN-payload".to_owned()),+ Some(text.to_owned()),+ ]);+ let mut reads = 0;+ let delivery = await_typed_delivery(+ Some(""),+ text,+ std::time::Instant::now() + std::time::Duration::from_secs(1),+ || {+ reads += 1;+ values.pop_front().flatten()+ },+ );+ assert_eq!(delivery, (true, Some(text.chars().count())));+ assert_eq!(reads, 2, "completion must wait past the prefix readback");+ }++ #[test]+ fn drained_prefix_reports_the_delivered_character_count() {+ let delivery = await_typed_delivery(+ Some(""),+ "BEGIN-payload-END",+ std::time::Instant::now(),+ || Some("BEGIN".to_owned()),+ );+ assert_eq!(delivery, (false, Some(5)));}#[test]