Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep a full budget for each dom_event click readback (#6519)
injaneity21efdf32ff36
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rsindex d144ea4a00..7794e8664d 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs@@ -37,6 +37,8 @@ pub(crate) struct MockEvent {pub(crate) struct MockReply {pub events: Vec<MockEvent>,pub suppress_response: bool,+ /// Hold the reply this long before sending it (a slow but live page).+ pub delay: Option<std::time::Duration>,pub result: Result<Value, (i64, String)>,}@@ -46,6 +48,7 @@ impl MockReply {events: Vec::new(),result: Ok(result),suppress_response: false,+ delay: None,}}@@ -55,6 +58,7 @@ impl MockReply {events: Vec::new(),result: Ok(json!({})),suppress_response: true,+ delay: None,}}@@ -63,6 +67,7 @@ impl MockReply {events: Vec::new(),result: Err((code, message.to_owned())),suppress_response: false,+ delay: None,}}@@ -142,6 +147,9 @@ impl MockCdpServer {if reply.suppress_response {continue;}+ if let Some(delay) = reply.delay {+ tokio::time::sleep(delay).await;+ }let mut response = match reply.result {Ok(result) => json!({ "id": id, "result": result }),Err((code, message)) => {diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex 4724a62cb2..e589d3cbf2 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -139,13 +139,12 @@ async fn observe_dom_click(readback_available: true,};}- if let Some(after) = read_dom_click_state(- conn,- cdp,- object_id,- DOM_CLICK_READBACK_DEADLINE.saturating_sub(started.elapsed()),- )- .await+ // The window only decides whether to start another read; each read+ // keeps a full budget. Clipping an in-flight read to the window's+ // remainder reported a slow-but-answering page as "readback+ // unavailable" instead of "state unchanged".+ if let Some(after) =+ read_dom_click_state(conn, cdp, object_id, DOM_CLICK_READBACK_DEADLINE).await{if dom_click_state_changed(&before, &after) {return DomClickObservation {diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex 935a8bf9ed..1b542a4d6e 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -97,6 +97,8 @@ pub(super) struct FixtureState {dom_click_generation: u64,dom_click_focused: bool,dom_click_reads_before_failure: Option<usize>,+ /// Answer every post-snapshot semantic readback after this delay.+ dom_click_readback_delay: Option<std::time::Duration>,/// Every incoming CDP call: (sessionId, method, params).calls: Vec<(Option<String>, String, Value)>,}@@ -140,6 +142,7 @@ impl Default for FixtureState {dom_click_generation: 0,dom_click_focused: false,dom_click_reads_before_failure: None,+ dom_click_readback_delay: None,calls: Vec::new(),}}@@ -783,7 +786,9 @@ fn fixture_handler(state: SharedState) -> MockHandler {let value = json!({"innerText": format!("counter={}", st.dom_click_generation)});- MockReply::ok(json!({ "result": { "value": value } }))+ let mut reply = MockReply::ok(json!({ "result": { "value": value } }));+ reply.delay = st.dom_click_readback_delay;+ reply}"Runtime.callFunctionOn" => {st.dom_click_focused = true;@@ -3259,6 +3264,40 @@ async fn dom_event_click_bounds_stalled_readbacks() {}}+#[tokio::test]+async fn dom_event_click_keeps_readback_when_reads_are_slow() {+ // Each read answers in 150 ms, so the third post-click read starts with+ // only ~50 ms of the 400 ms window left. It must still be waited for:+ // a slow page whose state did not change is "unchanged", not+ // "readback unavailable".+ let f = fixture_with(|state| {+ state.dom_click_changes_readback = false;+ state.dom_click_readback_delay = Some(std::time::Duration::from_millis(150));+ })+ .await;+ let (target, tab) = bind(&f).await;+ let snap = snapshot(&f, &target, &tab).await;+ let main_ref = ref_of(&snap, "main", "main-btn");+ let result = BrowserClickTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "ref": main_ref,+ "input_route": "dom_event",+ "session": SESSION,+ }))+ .await;+ assert!(result.content.iter().any(|content| matches!(+ content,+ Content::Text { text, .. }+ if text.contains("target state remained unchanged; application effect is unverifiable")+ )));+ let public = result.action_record.unwrap().public_result().unwrap();+ let public = serde_json::to_value(public).unwrap();+ assert_eq!(public["effect"], "unverifiable", "{public}");+ assert_eq!(public["evidence"][0]["kind"], "value_readback", "{public}");+}+#[tokio::test]async fn dom_event_click_is_unverifiable_when_readback_disappears() {let f = fixture_with(|state| {