Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): await synthetic smooth-scroll readback (#4602)
Iflaq khurshidf92b01906fc4
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rsindex bc741d9ba2..cd4fea9f64 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs@@ -442,7 +442,29 @@ impl BrowserPointerTool {json!([]),),PointerAction::Scroll => (- "function(dx,dy) { const o={deltaX:dx,deltaY:dy,bubbles:true,cancelable:true,composed:true,view:this.ownerDocument.defaultView}; this.dispatchEvent(new WheelEvent('wheel',o)); let n=this; while (n && n !== this.ownerDocument.documentElement) { const s=this.ownerDocument.defaultView.getComputedStyle(n); if (/(auto|scroll)/.test(s.overflow+s.overflowX+s.overflowY)) break; n=n.parentElement; } const target=n || this.ownerDocument.scrollingElement || this.ownerDocument.documentElement; const beforeX=target.scrollLeft; const beforeY=target.scrollTop; target.scrollBy(dx,dy); const changed=target.scrollLeft !== beforeX || target.scrollTop !== beforeY; if (changed) target.dispatchEvent(new Event('scroll')); return changed; }",+ r#"async function(dx,dy) {+ const view=this.ownerDocument.defaultView;+ const o={deltaX:dx,deltaY:dy,bubbles:true,cancelable:true,composed:true,view};+ this.dispatchEvent(new WheelEvent('wheel',o));+ let n=this;+ while (n && n !== this.ownerDocument.documentElement) {+ const s=view.getComputedStyle(n);+ if (/(auto|scroll)/.test(s.overflow+s.overflowX+s.overflowY)) break;+ n=n.parentElement;+ }+ const target=n || this.ownerDocument.scrollingElement || this.ownerDocument.documentElement;+ const beforeX=target.scrollLeft;+ const beforeY=target.scrollTop;+ target.scrollBy(dx,dy);+ // Smooth scrolling can start after this JavaScript task returns.+ const deadline=view.performance.now()+250;+ while (target.scrollLeft === beforeX && target.scrollTop === beforeY) {+ if (view.performance.now() >= deadline) return false;+ await new Promise(resolve => view.setTimeout(resolve,25));+ }+ target.dispatchEvent(new Event('scroll'));+ return true;+ }"#,json!([{ "value": request.delta_x }, { "value": request.delta_y }]),),PointerAction::Drag => {@@ -486,6 +508,7 @@ impl BrowserPointerTool {"functionDeclaration": function,"arguments": arguments,"returnByValue": true,+ "awaitPromise": request.action == PointerAction::Scroll,}),).awaitdiff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex d5f50d2a6e..9b3ab7c082 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -2895,6 +2895,36 @@ async fn foreground_delivery_mode_accepts_window_activation_for_trusted_click()assert_eq!(public["delivery"]["mode"], "foreground", "{public}");}+#[tokio::test]+async fn synthetic_scroll_awaits_page_readback_without_changing_routes() {+ let f = fixture().await;+ let (target, tab) = bind(&f).await;+ let snap = snapshot(&f, &target, &tab).await;+ let main_ref = ref_of(&snap, "main", "main-btn");+ let result = BrowserPointerTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target, "tab_id": tab, "ref": main_ref,+ "action": "scroll", "input_route": "dom_event",+ "delta_y": 240, "session": SESSION+ }))+ .await;+ assert_eq!(structured(&result)["status"], "ok");+ let calls = recorded_calls(&f, "Runtime.callFunctionOn");+ let scroll_calls: Vec<_> = calls+ .iter()+ .filter(|call| {+ call.1["functionDeclaration"]+ .as_str()+ .is_some_and(|s| s.contains("target.scrollBy"))+ })+ .collect();+ assert_eq!(scroll_calls.len(), 1, "scroll must not be retried");+ assert_eq!(scroll_calls[0].1["awaitPromise"], true);+ assert!(recorded_calls(&f, "Input.dispatchMouseEvent").is_empty());+ assert!(recorded_calls(&f, "Page.bringToFront").is_empty());+ assert!(recorded_calls(&f, "Target.activateTarget").is_empty());+}+#[tokio::test]async fn trusted_click_refuses_when_standalone_background_posture_is_unavailable() {let f = fixture_with_platform(|_| {}, true).await;diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rsindex 3813c3f665..eee8482076 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs@@ -4760,8 +4760,11 @@ fn run_upload(spec: &BrowserSpec) {fn run_pointer_actions(spec: &BrowserSpec) {let scenario = format!("{}-{}-standalone-pointer", std::env::consts::OS, spec.name);execute_case(case(&spec.name, "browser_pointer_actions"), |evidence| {- let mut fixture =- launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());+ let html = standalone_browser_completeness_html().replace(+ "</head>",+ "<style>#scroll-tall { scroll-behavior: smooth; }</style></head>",+ );+ let mut fixture = launch_browser_with_html(spec, &scenario, html);*evidence = recording_evidence(fixture.driver.recording_dir());let session = format!("standalone-pointer-{}", fixture.pid);prepare_same_space_attachment(&mut fixture, &session);@@ -4829,12 +4832,41 @@ fn run_pointer_actions(spec: &BrowserSpec) {"last_action=double_click",);+ let scroll_ref = semantic_ref_by_name(&snapshot, "scroll-tall", "scroll");+ let exhausted = fixture.driver.call(+ "browser_pointer",+ serde_json::json!({+ "target_id": target,+ "tab_id": tab,+ "ref": scroll_ref,+ "action": "scroll",+ "input_route": "dom_event",+ "delta_y": -240,+ "session": session,+ }),+ );+ assert_eq!(+ exhausted.action_effect(),+ Some("refused"),+ "{}",+ exhausted.raw+ );+ assert!(+ exhausted.text().contains("browser_action_unavailable"),+ "{}",+ exhausted.raw+ );+ assert_eq!(+ fixture.server.text("lbl-scroll-offset").as_deref(),+ Some("scroll_offset=0")+ );+let scrolled = fixture.driver.call("browser_pointer",serde_json::json!({"target_id": target,"tab_id": tab,- "ref": semantic_ref_by_name(&snapshot, "scroll-tall", "scroll"),+ "ref": scroll_ref,"action": "scroll","input_route": "dom_event","delta_y": 240,