Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): scroll an inner Electron scroller in the background on macOS (#5079)

Francesco Bonaccif72e033269cd
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs
index 77925cc654..fcf5071599 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs
@@ -1654,9 +1654,9 @@ fn shared_case_with_native_hyprland(
("electron", "type_text" | "type_submit" | "editor_save", Targeting::Ax) => {
vec![RefusalCode::BackgroundUnavailable]
}
- ("electron", "scroll", _) | (_, "drag", Targeting::Px) => {
- vec![RefusalCode::BackgroundUnavailable]
- }
+ // Electron scroll is delivered in the background through
+ // accessibility (reveal scroll), including inner scrollers.
+ (_, "drag", Targeting::Px) => vec![RefusalCode::BackgroundUnavailable],
_ => Vec::new(),
}
} else if cfg!(target_os = "linux")
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
index a8e324b7d2..5d9ead77a0 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
@@ -160,6 +160,13 @@ pub fn pick_clamped(
/// scroll container under the point or the reveal moved nothing; the caller
/// then reports that background scroll is unavailable.
///
+/// Chromium exposes an `overflow: auto` element as a plain group, not an
+/// AXScrollArea, so the nearest scroll-role ancestor of a point inside an
+/// inner scroller is the page's AXWebArea. The groups between the hit
+/// element and that ancestor that clip content past their edge are tried
+/// first, innermost out; the scroll-role container is the last resort. The
+/// first one that moves wins, and the end is reported only when none moved.
+///
/// # Safety
///
/// Performs AX calls on `pid`'s elements; call from a blocking context.
@@ -171,33 +178,100 @@ pub unsafe fn reveal_scroll(
distance: f64,
) -> Option<RevealOutcome> {
let hit = element_at_screen_position(pid, point.0, point.1)?;
- let container = scroll_container(hit);
+ let candidates = scroll_candidates(hit, direction);
CFRelease(hit as CFTypeRef);
- let (container, role) = container?;
- let outcome = reveal_in(container, &role, window, direction, distance);
- CFRelease(container as CFTypeRef);
- outcome
+ let mut at_end = None;
+ let mut candidates = candidates.into_iter();
+ while let Some((container, role)) = candidates.next() {
+ let outcome = reveal_in(container, &role, window, direction, distance);
+ CFRelease(container as CFTypeRef);
+ match outcome {
+ Some(outcome) if !outcome.at_end => {
+ for (rest, _) in candidates {
+ CFRelease(rest as CFTypeRef);
+ }
+ return Some(outcome);
+ }
+ // Keep the outermost end: the scroll-role container comes last.
+ Some(outcome) => at_end = Some(outcome),
+ None => {}
+ }
+ }
+ at_end
}
-/// The nearest scroll container at or above `element`, retained.
-unsafe fn scroll_container(element: AXUIElementRef) -> Option<(AXUIElementRef, String)> {
+/// The elements at or above `element` to try, innermost first and retained:
+/// each one that clips content past its edge in `direction`, then the
+/// nearest scroll-role container. Empty without a scroll-role container.
+unsafe fn scroll_candidates(
+ element: AXUIElementRef,
+ direction: &str,
+) -> Vec<(AXUIElementRef, String)> {
+ let mut candidates = Vec::new();
CFRetain(element as CFTypeRef);
let mut current = element;
for _ in 0..MAX_ASCENT {
let role = copy_string_attr(current, "AXRole");
match role.as_deref() {
Some(role) if CONTAINER_ROLES.contains(&role) => {
- return Some((current, role.to_owned()));
+ candidates.push((current, role.to_owned()));
+ return candidates;
}
Some("AXWindow") | Some("AXApplication") | None => break,
- _ => {}
+ Some(role) => {
+ if clips_past_edge(current, direction) {
+ CFRetain(current as CFTypeRef);
+ candidates.push((current, role.to_owned()));
+ }
+ }
}
- let parent = copy_element_attr(current, "AXParent");
+ let Some(parent) = copy_element_attr(current, "AXParent") else {
+ CFRelease(current as CFTypeRef);
+ current = std::ptr::null_mut();
+ break;
+ };
CFRelease(current as CFTypeRef);
- current = parent?;
+ current = parent;
+ }
+ if !current.is_null() {
+ CFRelease(current as CFTypeRef);
+ }
+ for (candidate, _) in candidates {
+ CFRelease(candidate as CFTypeRef);
+ }
+ Vec::new()
+}
+
+/// Whether `element` holds content past its edge in `direction`: a direct
+/// child lies beyond it, or is a sliver clamped to it.
+unsafe fn clips_past_edge(element: AXUIElementRef, direction: &str) -> bool {
+ let Some(frame) = element_screen_rect(element).filter(|frame| real_size(*frame)) else {
+ return false;
+ };
+ let mut clips = false;
+ for child in copy_children(element) {
+ if !clips {
+ clips =
+ element_screen_rect(child).is_some_and(|rect| past_edge(frame, direction, rect));
+ }
+ CFRelease(child as CFTypeRef);
+ }
+ clips
+}
+
+/// Whether `rect`, a child of the element with `frame`, lies past that
+/// frame's edge in `direction` or is a sliver clamped to that edge (how
+/// Chromium reports content scrolled out of an overflow container).
+pub fn past_edge(frame: Rect, direction: &str, rect: Rect) -> bool {
+ let [fx, fy, fw, fh] = frame;
+ let [x, y, w, h] = rect;
+ match direction {
+ "down" => y + h > fy + fh + 1.0 || (h <= 1.5 && y >= fy + fh - 2.5),
+ "up" => y < fy - 1.0 || (h <= 1.5 && y <= fy + 1.5),
+ "right" => x + w > fx + fw + 1.0 || (w <= 1.5 && x >= fx + fw - 2.5),
+ "left" => x < fx - 1.0 || (w <= 1.5 && x <= fx + 1.5),
+ _ => false,
}
- CFRelease(current as CFTypeRef);
- None
}
unsafe fn reveal_in(
@@ -496,4 +570,19 @@ mod tests {
None
);
}
+ #[test]
+ fn a_child_past_or_clamped_to_the_edge_marks_an_inner_scroller() {
+ let frame = [0.0, 100.0, 300.0, 128.0];
+ // A row below the visible area, and Chromium's 1 pt sliver clamped
+ // to the bottom edge for a row scrolled out of view.
+ assert!(past_edge(frame, "down", [0.0, 240.0, 300.0, 16.0]));
+ assert!(past_edge(frame, "down", [0.0, 227.0, 300.0, 1.0]));
+ // A visible row inside the frame is not.
+ assert!(!past_edge(frame, "down", [0.0, 110.0, 300.0, 16.0]));
+ // At the top nothing is above; a row above marks it for "up".
+ assert!(!past_edge(frame, "up", [0.0, 110.0, 300.0, 16.0]));
+ assert!(past_edge(frame, "up", [0.0, 60.0, 300.0, 16.0]));
+ assert!(past_edge(frame, "right", [250.0, 110.0, 100.0, 16.0]));
+ assert!(!past_edge(frame, "left", [0.0, 110.0, 300.0, 16.0]));
+ }
}