Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): refuse unconfirmed macOS Electron scrolls (#5251)
injaneityfca488d88b30
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rsindex 5d9ead77a0..39d99e2dd3 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs@@ -30,8 +30,6 @@ pub struct RevealOutcome {pub container_role: String,/// How far the content moved, in points, in the requested direction.pub moved: f64,- /// Nothing lay beyond the edge: the container is already at its end.- pub at_end: bool,}/// `[x, y, w, h]` in screen points.@@ -165,7 +163,7 @@ pub fn pick_clamped(/// inner scroller is the page's AXWebArea. The groups between the hit/// element and that ancestor that clip content past their edge are tried/// first, innermost out; the scroll-role container is the last resort. The-/// first one that moves wins, and the end is reported only when none moved.+/// first one that moves wins.////// # Safety///@@ -180,24 +178,21 @@ pub unsafe fn reveal_scroll(let hit = element_at_screen_position(pid, point.0, point.1)?;let candidates = scroll_candidates(hit, direction);CFRelease(hit as CFTypeRef);- let mut at_end = None;let mut candidates = candidates.into_iter();while let Some((container, role)) = candidates.next() {let outcome = reveal_in(container, &role, window, direction, distance);CFRelease(container as CFTypeRef);match outcome {- Some(outcome) if !outcome.at_end => {+ Some(outcome) => {for (rest, _) in candidates {CFRelease(rest as CFTypeRef);}return Some(outcome);}- // Keep the outermost end: the scroll-role container comes last.- Some(outcome) => at_end = Some(outcome),None => {}}}- at_end+ None}/// The elements at or above `element` to try, innermost first and retained:@@ -357,11 +352,10 @@ unsafe fn reveal_in((moved > 0.5).then(|| RevealOutcome {container_role: role.to_owned(),moved,- at_end: false,})} else if let Some(pick) = pick_clamped(viewport, direction, distance, &nodes) {// A clamped sliver: it scrolled when it now shows at a real size- // inside the viewport and a visible element moved (or left). Its old+ // inside the viewport or a visible reference moved (or left). Its old// position is estimated from the row count, so the distance is// approximate.let reference = nodes@@ -374,21 +368,8 @@ unsafe fn reveal_in(let reference_after = reference.and_then(|at| element_screen_rect(elements[at]));let shown =after.is_some_and(|rect| real_size(rect) && intersect(rect, viewport).is_some());- match content_moved(reference_before, reference_after) {- // Nothing visible moved: the container is at its end that way.- Some(false) => {- return finish(- elements,- Some(RevealOutcome {- container_role: role.to_owned(),- moved: 0.0,- at_end: true,- }),- )- }- Some(true) => {}- None if shown => {}- None => return finish(elements, None),+ if content_moved(reference_before, reference_after) != Some(true) && !shown {+ return finish(elements, None);}after.filter(|rect| real_size(*rect)).map(|after| {let past = (pick.rows as f64 - 1.0) * pick.pitch;@@ -402,15 +383,10 @@ unsafe fn reveal_in(RevealOutcome {container_role: role.to_owned(),moved: moved.max(1.0),- at_end: false,}})} else {- Some(RevealOutcome {- container_role: role.to_owned(),- moved: 0.0,- at_end: true,- })+ None};finish(elements, outcome)}@@ -487,7 +463,7 @@ mod tests {}#[test]- fn at_the_end_or_with_only_containers_beyond_nothing_is_picked() {+ fn without_candidates_beyond_the_viewport_nothing_is_picked() {let visible = vec![row(120.0), row(300.0), row(550.0)];assert_eq!(pick_reveal_target(VIEWPORT, "down", 120.0, &visible), None);// A page-sized wrapper reaching past the edge is not a target.@@ -542,7 +518,7 @@ mod tests {}#[test]- fn a_reveal_that_moves_nothing_visible_is_the_end() {+ fn a_reveal_that_moves_nothing_visible_is_not_confirmed_movement() {let row = [518.0, 301.0, 600.0, 41.0];// Seen live at the top of the page: the reveal changed nothing.assert_eq!(content_moved(Some(row), Some(row)), Some(false));diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rsindex 3a2554aaf7..87a6f05c67 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs@@ -60,8 +60,8 @@ fn clamp_amount(requested: u64) -> usize {const ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE: &str = "Background scroll is unavailable for \this Electron/Chromium window on macOS: it drops background wheel events and keystrokes, \- and no scroll container under the target moved through accessibility, so nothing \- scrolled. Retry with delivery_mode:\"foreground\" (and the same window_id, element_token \+ and background scroll movement could not be established through accessibility. \+ Retry with delivery_mode:\"foreground\" (and the same window_id, element_token \or x,y); the driver briefly fronts that window, scrolls, and restores the prior frontmost \app.";@@ -749,17 +749,6 @@ async fn electron_reveal_scroll(}).await;match outcome {- Ok(Some(outcome)) if outcome.at_end => ToolResult::text(format!(- "Nothing to scroll {direction}: the {} under the target is already at its end \- (checked through accessibility; this Electron/Chromium window drops background \- wheel events).",- outcome.container_role- ))- .with_structured(serde_json::json!({- "path": "ax",- "verified": false,- "effect": "unverifiable",- })),Ok(Some(outcome)) => ToolResult::text(format!("✅ Scrolled {direction} about {:.0} pt in the background through accessibility \(revealed the content just past the {}'s edge; this Electron/Chromium window drops \