Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): scroll Electron windows in the background through accessibility (#4861)

Francesco Bonacciea32762c9207
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx
index f57eaff4ac..92b4797795 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx
@@ -136,7 +136,7 @@ Scroll the target pid.
- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.
- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.
-Electron/Chromium windows refuse background delivery; pass `delivery_mode:"foreground"`.
+Electron/Chromium windows drop background wheel events: there a background scroll reveals the content just past the edge of the scroll container under the target through accessibility (about `amount` notches); `delivery_mode:"foreground"` is the fallback.
`amount` is wheel notches (targeted) or key repetitions (keystroke).
**macOS parameters**
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs
index c521a21870..6ad8c6e089 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs
@@ -20,6 +20,7 @@ pub mod bindings;
pub mod enablement;
pub mod exact_target;
pub mod launch;
+pub mod reveal_scroll;
pub mod snapshot;
pub mod tree;
pub mod window_scope;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
new file mode 100644
index 0000000000..a8e324b7d2
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/reveal_scroll.rs
@@ -0,0 +1,499 @@
+//! Background scroll through accessibility, for surfaces that drop
+//! background wheel events and keystrokes (Electron/Chromium on macOS).
+//!
+//! There is no AX "scroll by N pixels" for web content, but every element
+//! supports `AXScrollToVisible`, which scrolls its container just enough to
+//! show it. Scrolling down by a distance is therefore: find the scroll
+//! container under the point, find the element whose bottom edge lies about
+//! that distance below the visible area, and reveal it. Up, left and right
+//! mirror this. No event is posted and nothing is fronted.
+
+use core_foundation::base::{CFRelease, CFRetain, CFTypeRef};
+
+use super::bindings::{
+ copy_children, copy_element_attr, copy_string_attr, element_at_screen_position,
+ element_screen_rect, perform_action, AXUIElementRef,
+};
+
+/// Roles that scroll their content.
+const CONTAINER_ROLES: &[&str] = &["AXScrollArea", "AXWebArea"];
+/// Ancestors searched from the hit element for a scroll container.
+const MAX_ASCENT: usize = 40;
+/// Descendants visited when looking for the element to reveal.
+const MAX_NODES: usize = 4_000;
+const MAX_DEPTH: usize = 80;
+
+/// What a reveal scroll did.
+#[derive(Debug, Clone, PartialEq)]
+pub struct RevealOutcome {
+ /// Role of the scroll container used.
+ pub container_role: String,
+ /// How far the content moved, in points, in the requested direction.
+ pub moved: f64,
+ /// Nothing lay beyond the edge: the container is already at its end.
+ pub at_end: bool,
+}
+
+/// `[x, y, w, h]` in screen points.
+type Rect = [f64; 4];
+
+fn intersect(a: Rect, b: Rect) -> Option<Rect> {
+ let left = a[0].max(b[0]);
+ let top = a[1].max(b[1]);
+ let right = (a[0] + a[2]).min(b[0] + b[2]);
+ let bottom = (a[1] + a[3]).min(b[1] + b[3]);
+ (right > left && bottom > top).then_some([left, top, right - left, bottom - top])
+}
+
+/// Which candidate to reveal so the content moves about `distance` points in
+/// `direction`: for "down", the element whose bottom edge is closest to
+/// `distance` below the viewport's bottom (and lies below it); "up", "left"
+/// and "right" mirror this. Elements larger than the viewport along the
+/// scroll axis are containers and are skipped. `None` when nothing lies
+/// beyond that edge.
+pub fn pick_reveal_target(
+ viewport: Rect,
+ direction: &str,
+ distance: f64,
+ candidates: &[Rect],
+) -> Option<usize> {
+ let [vx, vy, vw, vh] = viewport;
+ let mut best: Option<(usize, f64)> = None;
+ for (index, rect) in candidates.iter().enumerate() {
+ let [x, y, w, h] = *rect;
+ let (edge, goal, beyond, fits) = match direction {
+ "down" => (y + h, vy + vh + distance, y + h > vy + vh + 1.0, h <= vh),
+ "up" => (y, vy - distance, y < vy - 1.0, h <= vh),
+ "right" => (x + w, vx + vw + distance, x + w > vx + vw + 1.0, w <= vw),
+ "left" => (x, vx - distance, x < vx - 1.0, w <= vw),
+ _ => return None,
+ };
+ if !beyond || !fits {
+ continue;
+ }
+ let gap = (edge - goal).abs();
+ if best.is_none_or(|(_, best_gap)| gap < best_gap) {
+ best = Some((index, gap));
+ }
+ }
+ best.map(|(index, _)| index)
+}
+
+/// A clamped row chosen for a reveal.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ClampedPick {
+ /// Index into the nodes.
+ pub index: usize,
+ /// How many rows past the edge it is (1 = the first).
+ pub rows: usize,
+ /// Row pitch measured on the visible leaves.
+ pub pitch: f64,
+}
+
+/// Chromium reports an element scrolled out of view as a 1 pt sliver
+/// clamped to the viewport edge, not at its real position, and its
+/// AXScrollToVisible centers the element. So when nothing lies geometrically
+/// beyond the edge, the off-screen content is the leaves clamped to that
+/// edge, in document order, and revealing the k-th of them moves the content
+/// by about half the viewport plus (k - 0.5) rows. Pick k for `distance`,
+/// counting rows at the pitch of the visible leaves. `nodes` are
+/// (frame, is_leaf) in document order. `None` when nothing is clamped there.
+pub fn pick_clamped(
+ viewport: Rect,
+ direction: &str,
+ distance: f64,
+ nodes: &[(Rect, bool)],
+) -> Option<ClampedPick> {
+ let [vx, vy, vw, vh] = viewport;
+ let vertical = matches!(direction, "up" | "down");
+ let sliver = |r: &Rect| if vertical { r[3] <= 1.5 } else { r[2] <= 1.5 };
+ let at_edge = |r: &Rect| match direction {
+ "down" => r[1] >= vy + vh - 2.5,
+ "up" => r[1] <= vy + 1.5,
+ "right" => r[0] >= vx + vw - 2.5,
+ "left" => r[0] <= vx + 1.5,
+ _ => false,
+ };
+ let mut clamped: Vec<usize> = nodes
+ .iter()
+ .enumerate()
+ .filter(|(_, (rect, leaf))| *leaf && sliver(rect) && at_edge(rect))
+ .map(|(index, _)| index)
+ .collect();
+ if clamped.is_empty() {
+ return None;
+ }
+ if matches!(direction, "up" | "left") {
+ clamped.reverse();
+ }
+ // Row pitch: the median gap between the visible leaves' starts.
+ let mut starts: Vec<f64> = nodes
+ .iter()
+ .filter(|(rect, leaf)| {
+ *leaf
+ && !sliver(rect)
+ && rect[1] >= vy
+ && rect[1] + rect[3] <= vy + vh
+ && rect[0] >= vx
+ && rect[0] + rect[2] <= vx + vw
+ })
+ .map(|(rect, _)| if vertical { rect[1] } else { rect[0] })
+ .collect();
+ starts.sort_by(f64::total_cmp);
+ starts.dedup_by(|a, b| (*a - *b).abs() < 2.0);
+ let mut gaps: Vec<f64> = starts.windows(2).map(|pair| pair[1] - pair[0]).collect();
+ gaps.sort_by(f64::total_cmp);
+ let pitch = gaps.get(gaps.len() / 2).copied().unwrap_or(40.0).max(8.0);
+ let half = if vertical { vh } else { vw } / 2.0;
+ let rows = ((distance - half) / pitch + 0.5).round().max(1.0) as usize;
+ let rows = rows.min(clamped.len());
+ Some(ClampedPick {
+ index: clamped[rows - 1],
+ rows,
+ pitch,
+ })
+}
+
+/// Scroll the container under screen point `point` by about `distance`
+/// points in `direction`, by revealing the element just past its edge.
+/// `window` clips the container's visible area. `None` when there is no
+/// scroll container under the point or the reveal moved nothing; the caller
+/// then reports that background scroll is unavailable.
+///
+/// # Safety
+///
+/// Performs AX calls on `pid`'s elements; call from a blocking context.
+pub unsafe fn reveal_scroll(
+ pid: i32,
+ point: (f64, f64),
+ window: Option<Rect>,
+ direction: &str,
+ distance: f64,
+) -> Option<RevealOutcome> {
+ let hit = element_at_screen_position(pid, point.0, point.1)?;
+ let container = scroll_container(hit);
+ CFRelease(hit as CFTypeRef);
+ let (container, role) = container?;
+ let outcome = reveal_in(container, &role, window, direction, distance);
+ CFRelease(container as CFTypeRef);
+ outcome
+}
+
+/// The nearest scroll container at or above `element`, retained.
+unsafe fn scroll_container(element: AXUIElementRef) -> Option<(AXUIElementRef, String)> {
+ CFRetain(element as CFTypeRef);
+ let mut current = element;
+ for _ in 0..MAX_ASCENT {
+ let role = copy_string_attr(current, "AXRole");
+ match role.as_deref() {
+ Some(role) if CONTAINER_ROLES.contains(&role) => {
+ return Some((current, role.to_owned()));
+ }
+ Some("AXWindow") | Some("AXApplication") | None => break,
+ _ => {}
+ }
+ let parent = copy_element_attr(current, "AXParent");
+ CFRelease(current as CFTypeRef);
+ current = parent?;
+ }
+ CFRelease(current as CFTypeRef);
+ None
+}
+
+unsafe fn reveal_in(
+ container: AXUIElementRef,
+ role: &str,
+ window: Option<Rect>,
+ direction: &str,
+ distance: f64,
+) -> Option<RevealOutcome> {
+ let frame = element_screen_rect(container)?;
+ let viewport = match window {
+ Some(window) => intersect(frame, window)?,
+ None => frame,
+ };
+ // Visit descendants in document order, keeping each element with a
+ // frame. Subtrees that lie wholly on the near side of the viewport
+ // cannot hold the target.
+ let mut elements: Vec<AXUIElementRef> = Vec::new();
+ let mut nodes: Vec<(Rect, bool)> = Vec::new();
+ let mut stack: Vec<(AXUIElementRef, usize)> = copy_children(container)
+ .into_iter()
+ .rev()
+ .map(|child| (child, 1))
+ .collect();
+ let mut visited = 0usize;
+ while let Some((element, depth)) = stack.pop() {
+ visited += 1;
+ let rect = element_screen_rect(element);
+ let behind = rect.is_some_and(|[x, y, w, h]| match direction {
+ "down" => y + h < viewport[1],
+ "up" => y > viewport[1] + viewport[3],
+ "right" => x + w < viewport[0],
+ "left" => x > viewport[0] + viewport[2],
+ _ => false,
+ });
+ let mut leaf = true;
+ if !behind && depth < MAX_DEPTH && visited < MAX_NODES {
+ let children = copy_children(element);
+ leaf = children.is_empty();
+ for child in children.into_iter().rev() {
+ stack.push((child, depth + 1));
+ }
+ }
+ match rect {
+ Some(rect) if !behind => {
+ elements.push(element);
+ nodes.push((rect, leaf));
+ }
+ _ => CFRelease(element as CFTypeRef),
+ }
+ if visited >= MAX_NODES {
+ break;
+ }
+ }
+ for (element, _) in stack {
+ CFRelease(element as CFTypeRef);
+ }
+
+ // Real frames past the edge first; Chromium's clamped slivers otherwise.
+ let real: Vec<Rect> = nodes
+ .iter()
+ .map(|(rect, _)| *rect)
+ .map(|rect| {
+ if rect[2] <= 1.5 || rect[3] <= 1.5 {
+ [0.0, 0.0, 0.0, 0.0]
+ } else {
+ rect
+ }
+ })
+ .collect();
+ let outcome = if let Some(index) = pick_reveal_target(viewport, direction, distance, &real) {
+ // A real frame: measure the move on the element itself.
+ let before = nodes[index].0;
+ perform_action(elements[index], "AXScrollToVisible");
+ std::thread::sleep(std::time::Duration::from_millis(150));
+ let after = element_screen_rect(elements[index]).unwrap_or(before);
+ let moved = match direction {
+ "down" => before[1] - after[1],
+ "up" => after[1] - before[1],
+ "right" => before[0] - after[0],
+ _ => after[0] - before[0],
+ };
+ (moved > 0.5).then(|| RevealOutcome {
+ container_role: role.to_owned(),
+ moved,
+ at_end: false,
+ })
+ } else if let Some(pick) = pick_clamped(viewport, direction, distance, &nodes) {
+ // A clamped sliver: it scrolled when it now shows at a real size
+ // inside the viewport and a visible element moved (or left). Its old
+ // position is estimated from the row count, so the distance is
+ // approximate.
+ let reference = nodes
+ .iter()
+ .position(|(rect, leaf)| *leaf && real_size(*rect) && contains(viewport, *rect));
+ let reference_before = reference.map(|at| nodes[at].0);
+ perform_action(elements[pick.index], "AXScrollToVisible");
+ std::thread::sleep(std::time::Duration::from_millis(150));
+ let after = element_screen_rect(elements[pick.index]);
+ let reference_after = reference.and_then(|at| element_screen_rect(elements[at]));
+ let shown =
+ after.is_some_and(|rect| real_size(rect) && intersect(rect, viewport).is_some());
+ match content_moved(reference_before, reference_after) {
+ // Nothing visible moved: the container is at its end that way.
+ Some(false) => {
+ return finish(
+ elements,
+ Some(RevealOutcome {
+ container_role: role.to_owned(),
+ moved: 0.0,
+ at_end: true,
+ }),
+ )
+ }
+ Some(true) => {}
+ None if shown => {}
+ None => return finish(elements, None),
+ }
+ after.filter(|rect| real_size(*rect)).map(|after| {
+ let past = (pick.rows as f64 - 1.0) * pick.pitch;
+ let [vx, vy, vw, vh] = viewport;
+ let moved = match direction {
+ "down" => vy + vh + past - after[1],
+ "up" => after[1] + after[3] - (vy - past),
+ "right" => vx + vw + past - after[0],
+ _ => after[0] + after[2] - (vx - past),
+ };
+ RevealOutcome {
+ container_role: role.to_owned(),
+ moved: moved.max(1.0),
+ at_end: false,
+ }
+ })
+ } else {
+ Some(RevealOutcome {
+ container_role: role.to_owned(),
+ moved: 0.0,
+ at_end: true,
+ })
+ };
+ finish(elements, outcome)
+}
+
+/// Release the visited elements and return `outcome`.
+unsafe fn finish(
+ elements: Vec<AXUIElementRef>,
+ outcome: Option<RevealOutcome>,
+) -> Option<RevealOutcome> {
+ for element in elements {
+ CFRelease(element as CFTypeRef);
+ }
+ outcome
+}
+
+fn real_size(rect: Rect) -> bool {
+ rect[2] > 1.5 && rect[3] > 1.5
+}
+
+fn contains(outer: Rect, inner: Rect) -> bool {
+ inner[0] >= outer[0]
+ && inner[1] >= outer[1]
+ && inner[0] + inner[2] <= outer[0] + outer[2]
+ && inner[1] + inner[3] <= outer[1] + outer[3]
+}
+
+/// Whether a reveal moved the content, judged on an element that was fully
+/// visible before: it moved, or it left (now a clamped sliver or gone).
+/// `None` without such an element.
+pub fn content_moved(before: Option<Rect>, after: Option<Rect>) -> Option<bool> {
+ let before = before?;
+ Some(match after {
+ None => true,
+ Some(after) if !real_size(after) => true,
+ Some(after) => (after[0] - before[0]).abs() > 0.5 || (after[1] - before[1]).abs() > 0.5,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const VIEWPORT: Rect = [0.0, 100.0, 800.0, 500.0];
+
+ fn row(y: f64) -> Rect {
+ [10.0, y, 600.0, 40.0]
+ }
+
+ #[test]
+ fn down_reveals_the_row_closest_to_the_distance_below_the_edge() {
+ // Rows every 50 pt from y=100; the viewport ends at y=600.
+ let rows: Vec<Rect> = (0..30).map(|i| row(100.0 + 50.0 * i as f64)).collect();
+ let picked = pick_reveal_target(VIEWPORT, "down", 360.0, &rows).unwrap();
+ // Goal: a bottom edge at 960. Row 17 spans 950-990, row 16 900-940.
+ assert_eq!(rows[picked], row(900.0));
+ }
+
+ #[test]
+ fn up_left_and_right_mirror_down() {
+ let rows: Vec<Rect> = (-10..10).map(|i| row(100.0 + 50.0 * i as f64)).collect();
+ let picked = pick_reveal_target(VIEWPORT, "up", 120.0, &rows).unwrap();
+ assert_eq!(
+ rows[picked],
+ row(0.0),
+ "top edge closest to 100 - 120 = -20"
+ );
+
+ let columns: Vec<Rect> = (0..20)
+ .map(|i| [100.0 * i as f64, 200.0, 90.0, 40.0])
+ .collect();
+ let right = pick_reveal_target(VIEWPORT, "right", 120.0, &columns).unwrap();
+ assert_eq!(columns[right][0], 800.0);
+ assert_eq!(pick_reveal_target(VIEWPORT, "left", 120.0, &columns), None);
+ }
+
+ #[test]
+ fn at_the_end_or_with_only_containers_beyond_nothing_is_picked() {
+ let visible = vec![row(120.0), row(300.0), row(550.0)];
+ assert_eq!(pick_reveal_target(VIEWPORT, "down", 120.0, &visible), None);
+ // A page-sized wrapper reaching past the edge is not a target.
+ let wrapper = vec![[0.0, 100.0, 800.0, 3000.0]];
+ assert_eq!(pick_reveal_target(VIEWPORT, "down", 120.0, &wrapper), None);
+ assert_eq!(
+ pick_reveal_target(VIEWPORT, "sideways", 120.0, &[row(900.0)]),
+ None
+ );
+ }
+
+ #[test]
+ fn clamped_slivers_count_as_the_content_past_the_edge() {
+ // What Chromium reported live: rows past the viewport bottom (y=890)
+ // as 1 pt slivers at y=889; visible rows every 45 pt.
+ let mut nodes: Vec<(Rect, bool)> = (0..13)
+ .map(|i| ([518.0, 301.0 + 45.0 * i as f64, 600.0, 41.0], true))
+ .collect();
+ nodes.push(([518.0, 300.0, 700.0, 590.0], false)); // the list itself
+ let first_clamped = nodes.len();
+ for _ in 0..100 {
+ nodes.push(([518.0, 889.0, 600.0, 1.0], true));
+ }
+ let viewport = [500.0, 290.0, 800.0, 600.0];
+ // Revealing centers the row: 360 pt is half the 600 pt viewport plus
+ // about 1.5 rows of 45 pt, so the 2nd row past the edge.
+ let pick = pick_clamped(viewport, "down", 360.0, &nodes).unwrap();
+ assert_eq!(
+ (pick.index, pick.rows, pick.pitch),
+ (first_clamped + 1, 2, 45.0)
+ );
+ // A page down (600 pt) is about 7 rows on.
+ assert_eq!(
+ pick_clamped(viewport, "down", 600.0, &nodes).unwrap().rows,
+ 7
+ );
+ // A distance past the content picks the last row.
+ let far = pick_clamped(viewport, "down", 100_000.0, &nodes).unwrap();
+ assert_eq!(far.index, first_clamped + 99);
+ // Nothing clamped at the top edge: at the start already.
+ assert_eq!(pick_clamped(viewport, "up", 360.0, &nodes), None);
+ // Up counts from the nearest sliver above, the last in document order;
+ // a short distance still reveals at least the nearest row.
+ let mut above: Vec<(Rect, bool)> =
+ (0..5).map(|_| ([518.0, 290.0, 600.0, 1.0], true)).collect();
+ above.extend((0..13).map(|i| ([518.0, 301.0 + 45.0 * i as f64, 600.0, 41.0], true)));
+ assert_eq!(pick_clamped(viewport, "up", 90.0, &above).unwrap().index, 4);
+ assert_eq!(
+ pick_clamped(viewport, "up", 480.0, &above).unwrap().index,
+ 0
+ );
+ }
+
+ #[test]
+ fn a_reveal_that_moves_nothing_visible_is_the_end() {
+ let row = [518.0, 301.0, 600.0, 41.0];
+ // Seen live at the top of the page: the reveal changed nothing.
+ assert_eq!(content_moved(Some(row), Some(row)), Some(false));
+ assert_eq!(
+ content_moved(Some(row), Some([518.0, 260.0, 600.0, 41.0])),
+ Some(true)
+ );
+ // Scrolled out of view: clamped to a sliver, or gone.
+ assert_eq!(
+ content_moved(Some(row), Some([518.0, 222.0, 600.0, 1.0])),
+ Some(true)
+ );
+ assert_eq!(content_moved(Some(row), None), Some(true));
+ assert_eq!(content_moved(None, Some(row)), None);
+ }
+
+ #[test]
+ fn intersect_clips_to_the_window() {
+ assert_eq!(
+ intersect([0.0, 0.0, 1000.0, 3000.0], [10.0, 50.0, 800.0, 600.0]),
+ Some([10.0, 50.0, 800.0, 600.0])
+ );
+ assert_eq!(
+ intersect([0.0, 0.0, 10.0, 10.0], [20.0, 20.0, 5.0, 5.0]),
+ None
+ );
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
index 7a5efaed92..df35919e5e 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
@@ -59,10 +59,11 @@ fn clamp_amount(requested: u64) -> usize {
}
const ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE: &str = "Background scroll is unavailable for \
- Electron/Chromium windows on macOS: they drop background wheel events and keystrokes, so \
- nothing was sent. Retry with delivery_mode:\"foreground\" (and the same window_id, \
- element_token or x,y); the driver briefly fronts that window, scrolls, and restores the \
- prior frontmost app.";
+ this Electron/Chromium window on macOS: it drops background wheel events and keystrokes, \
+ and no scroll container under the target moved through accessibility, so nothing \
+ scrolled. Retry with delivery_mode:\"foreground\" (and the same window_id, element_token \
+ or x,y); the driver briefly fronts that window, scrolls, and restores the prior frontmost \
+ app.";
fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
@@ -71,7 +72,7 @@ fn def() -> &'static ToolDef {
- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n\
- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\
\n\
- Electron/Chromium windows refuse background delivery; pass `delivery_mode:\"foreground\"`.\n\
+ Electron/Chromium windows drop background wheel events: there a background scroll reveals the content just past the edge of the scroll container under the target through accessibility (about `amount` notches); `delivery_mode:\"foreground\"` is the fallback.\n\
`amount` is wheel notches (targeted) or key repetitions (keystroke).".into(),
input_schema: serde_json::json!({
"type": "object",
@@ -171,14 +172,10 @@ impl Tool for ScrollTool {
// background CGEvents). Only the pixel-wheel path honors it; the
// keystroke path is background-by-design and untouched.
let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());
- if !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid) {
- return cua_driver_core::delivery::background_unavailable_result(
- ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE,
- "background_unavailable",
- "Electron/Chromium windows drop background wheel events and keystrokes on macOS",
- serde_json::json!({ "effect": "refused" }),
- );
- }
+ // Electron/Chromium drop background wheel events and keystrokes; a
+ // background scroll there goes through accessibility instead (below).
+ let electron_background =
+ !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid);
let direction = match args.require_str("direction") {
Ok(v) => v,
Err(e) => return e,
@@ -444,6 +441,21 @@ impl Tool for ScrollTool {
None
};
+ if electron_background {
+ return electron_reveal_scroll(
+ pid,
+ window_id,
+ wheel_target
+ .as_ref()
+ .map(|target| (target.screen_x, target.screen_y)),
+ &direction,
+ (step.unsigned_abs() as usize * amount) as f64,
+ pre_focus_ptr,
+ &mut _mutation_lease,
+ )
+ .await;
+ }
+
if let Some(target) = wheel_target {
if !delivery_mode.is_foreground() {
if let Some(wid) = target.wid {
@@ -672,6 +684,92 @@ impl Tool for ScrollTool {
}
}
+/// Background scroll for Electron/Chromium windows, which drop background
+/// wheel events: reveal the content just past the edge of the scroll
+/// container under `point` (the window's center when there is no target)
+/// through accessibility. Refuses as before when no container moves.
+async fn electron_reveal_scroll(
+ pid: i32,
+ window_id: Option<u32>,
+ point: Option<(f64, f64)>,
+ direction: &str,
+ distance: f64,
+ pre_focus_ptr: Option<usize>,
+ lease: &mut Option<super::BackgroundMutationLease>,
+) -> ToolResult {
+ let unavailable = || {
+ cua_driver_core::delivery::background_unavailable_result(
+ ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE,
+ "background_unavailable",
+ "Electron/Chromium windows drop background wheel events and keystrokes on macOS",
+ serde_json::json!({ "effect": "refused" }),
+ )
+ };
+ let window = window_id.and_then(crate::windows::window_bounds_by_id);
+ let Some(point) = point.or_else(|| {
+ window
+ .as_ref()
+ .map(|b| (b.x + b.width / 2.0, b.y + b.height / 2.0))
+ }) else {
+ return unavailable();
+ };
+ if let Some(wid) = window_id {
+ let gate = match lease.as_ref() {
+ Some(lease) => {
+ lease
+ .gate_again(
+ wid,
+ pre_focus_ptr,
+ cua_driver_core::background_input::BackgroundAction::AxSemantic,
+ )
+ .await
+ }
+ None => super::gate_background_window_action(
+ pid,
+ wid,
+ pre_focus_ptr,
+ cua_driver_core::background_input::BackgroundAction::AxSemantic,
+ )
+ .await
+ .map(|granted| *lease = Some(granted)),
+ };
+ if let Err(refusal) = gate {
+ return refusal;
+ }
+ }
+ let window_rect = window.map(|b| [b.x, b.y, b.width, b.height]);
+ let direction_owned = direction.to_owned();
+ let outcome = tokio::task::spawn_blocking(move || unsafe {
+ crate::ax::reveal_scroll::reveal_scroll(pid, point, window_rect, &direction_owned, distance)
+ })
+ .await;
+ match outcome {
+ Ok(Some(outcome)) if outcome.at_end => ToolResult::text(format!(
+ "Nothing to scroll {direction}: the {} under the target is already at its end \
+ (checked through accessibility; this Electron/Chromium window drops background \
+ wheel events).",
+ outcome.container_role
+ ))
+ .with_structured(serde_json::json!({
+ "path": "ax",
+ "verified": false,
+ "effect": "unverifiable",
+ })),
+ Ok(Some(outcome)) => ToolResult::text(format!(
+ "✅ Scrolled {direction} about {:.0} pt in the background through accessibility \
+ (revealed the content just past the {}'s edge; this Electron/Chromium window drops \
+ background wheel events).",
+ outcome.moved, outcome.container_role
+ ))
+ .with_structured(serde_json::json!({
+ "path": "ax",
+ "verified": false,
+ "effect": "unverifiable",
+ })),
+ _ => unavailable(),
+ }
+}
+
unsafe fn scroll_native_text_area(
element: AXUIElementRef,
direction: &str,
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index e44a85b45e..f9290e51aa 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -1295,7 +1295,7 @@
"read_only": false
},
{
- "description": "Scroll the target pid.\n- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\nElectron/Chromium windows refuse background delivery; pass `delivery_mode:\"foreground\"`.\n`amount` is wheel notches (targeted) or key repetitions (keystroke).",
+ "description": "Scroll the target pid.\n- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\nElectron/Chromium windows drop background wheel events: there a background scroll reveals the content just past the edge of the scroll container under the target through accessibility (about `amount` notches); `delivery_mode:\"foreground\"` is the fallback.\n`amount` is wheel notches (targeted) or key repetitions (keystroke).",
"destructive": false,
"idempotent": false,
"input_schema": {