Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): refuse unconfirmed macOS Electron scrolls (#5251)

injaneityfca488d88b30
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
index 5d9ead77a0..39d99e2dd3 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
@@ -30,8 +30,6 @@ pub struct RevealOutcome {
pub container_role: String,
/// How far the content moved, in points, in the requested direction.
pub moved: f64,
- /// Nothing lay beyond the edge: the container is already at its end.
- pub at_end: bool,
}
/// `[x, y, w, h]` in screen points.
@@ -165,7 +163,7 @@ pub fn pick_clamped(
/// inner scroller is the page's AXWebArea. The groups between the hit
/// element and that ancestor that clip content past their edge are tried
/// first, innermost out; the scroll-role container is the last resort. The
-/// first one that moves wins, and the end is reported only when none moved.
+/// first one that moves wins.
///
/// # Safety
///
@@ -180,24 +178,21 @@ pub unsafe fn reveal_scroll(
let hit = element_at_screen_position(pid, point.0, point.1)?;
let candidates = scroll_candidates(hit, direction);
CFRelease(hit as CFTypeRef);
- let mut at_end = None;
let mut candidates = candidates.into_iter();
while let Some((container, role)) = candidates.next() {
let outcome = reveal_in(container, &role, window, direction, distance);
CFRelease(container as CFTypeRef);
match outcome {
- Some(outcome) if !outcome.at_end => {
+ Some(outcome) => {
for (rest, _) in candidates {
CFRelease(rest as CFTypeRef);
}
return Some(outcome);
}
- // Keep the outermost end: the scroll-role container comes last.
- Some(outcome) => at_end = Some(outcome),
None => {}
}
}
- at_end
+ None
}
/// The elements at or above `element` to try, innermost first and retained:
@@ -357,11 +352,10 @@ unsafe fn reveal_in(
(moved > 0.5).then(|| RevealOutcome {
container_role: role.to_owned(),
moved,
- at_end: false,
})
} else if let Some(pick) = pick_clamped(viewport, direction, distance, &nodes) {
// A clamped sliver: it scrolled when it now shows at a real size
- // inside the viewport and a visible element moved (or left). Its old
+ // inside the viewport or a visible reference moved (or left). Its old
// position is estimated from the row count, so the distance is
// approximate.
let reference = nodes
@@ -374,21 +368,8 @@ unsafe fn reveal_in(
let reference_after = reference.and_then(|at| element_screen_rect(elements[at]));
let shown =
after.is_some_and(|rect| real_size(rect) && intersect(rect, viewport).is_some());
- match content_moved(reference_before, reference_after) {
- // Nothing visible moved: the container is at its end that way.
- Some(false) => {
- return finish(
- elements,
- Some(RevealOutcome {
- container_role: role.to_owned(),
- moved: 0.0,
- at_end: true,
- }),
- )
- }
- Some(true) => {}
- None if shown => {}
- None => return finish(elements, None),
+ if content_moved(reference_before, reference_after) != Some(true) && !shown {
+ return finish(elements, None);
}
after.filter(|rect| real_size(*rect)).map(|after| {
let past = (pick.rows as f64 - 1.0) * pick.pitch;
@@ -402,15 +383,10 @@ unsafe fn reveal_in(
RevealOutcome {
container_role: role.to_owned(),
moved: moved.max(1.0),
- at_end: false,
}
})
} else {
- Some(RevealOutcome {
- container_role: role.to_owned(),
- moved: 0.0,
- at_end: true,
- })
+ None
};
finish(elements, outcome)
}
@@ -487,7 +463,7 @@ mod tests {
}
#[test]
- fn at_the_end_or_with_only_containers_beyond_nothing_is_picked() {
+ fn without_candidates_beyond_the_viewport_nothing_is_picked() {
let visible = vec![row(120.0), row(300.0), row(550.0)];
assert_eq!(pick_reveal_target(VIEWPORT, "down", 120.0, &visible), None);
// A page-sized wrapper reaching past the edge is not a target.
@@ -542,7 +518,7 @@ mod tests {
}
#[test]
- fn a_reveal_that_moves_nothing_visible_is_the_end() {
+ fn a_reveal_that_moves_nothing_visible_is_not_confirmed_movement() {
let row = [518.0, 301.0, 600.0, 41.0];
// Seen live at the top of the page: the reveal changed nothing.
assert_eq!(content_moved(Some(row), Some(row)), Some(false));
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
index 3a2554aaf7..87a6f05c67 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
@@ -60,8 +60,8 @@ fn clamp_amount(requested: u64) -> usize {
const ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE: &str = "Background scroll is unavailable for \
this Electron/Chromium window on macOS: it drops background wheel events and keystrokes, \
- and no scroll container under the target moved through accessibility, so nothing \
- scrolled. Retry with delivery_mode:\"foreground\" (and the same window_id, element_token \
+ and background scroll movement could not be established through accessibility. \
+ Retry with delivery_mode:\"foreground\" (and the same window_id, element_token \
or x,y); the driver briefly fronts that window, scrolls, and restores the prior frontmost \
app.";
@@ -749,17 +749,6 @@ async fn electron_reveal_scroll(
})
.await;
match outcome {
- Ok(Some(outcome)) if outcome.at_end => ToolResult::text(format!(
- "Nothing to scroll {direction}: the {} under the target is already at its end \
- (checked through accessibility; this Electron/Chromium window drops background \
- wheel events).",
- outcome.container_role
- ))
- .with_structured(serde_json::json!({
- "path": "ax",
- "verified": false,
- "effect": "unverifiable",
- })),
Ok(Some(outcome)) => ToolResult::text(format!(
"✅ Scrolled {direction} about {:.0} pt in the background through accessibility \
(revealed the content just past the {}'s edge; this Electron/Chromium window drops \