Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): support fractionally scaled Hyprland desktops (#4244)
lorenzozane3034435724d4
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/capture_action_frame.rs b/libs/cua-driver/rust/crates/platform-linux/src/capture_action_frame.rsindex 9d3b824ecc..4ca394df82 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/capture_action_frame.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/capture_action_frame.rs@@ -170,9 +170,8 @@ fn live_action_dimensions(_target: &CaptureTarget) -> anyhow::Result<NativeActio#[cfg(target_os = "linux")]pub(crate) fn desktop_action_dimensions(native: (u32, u32)) -> anyhow::Result<(u32, u32)> {- let logical = if crate::wayland::is_wayland() && crate::wayland::hyprland::is_session() {- let (width, height, _) = crate::wayland::hyprland::screen_size()?;- Some((width, height))+ let logical = if crate::wayland::is_wayland() {+ crate::wayland::compositor_logical_frame().transpose()?} else {None};diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex dad3ba8c8d..6fd6622c7b 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -11927,7 +11927,8 @@ impl Tool for GetScreenSizeTool {// Shared manifest admission needs content-free display// metadata even when this native desktop has no X11 DISPLAY.// The adapter attests the IPC/Wayland compositor peer and- // refuses layouts outside its qualified 1:1 single-output frame.+ // refuses layouts outside its qualified single-output frame.+ // Scaled outputs report logical pixels plus the output scale.return crate::wayland::hyprland::screen_size();}// X11 reports pixel dimensions; scale factor on X11 is notdiff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rsindex 7c9dce8247..b97f3e3fde 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs@@ -317,9 +317,13 @@ fn valid_dimensions(width: u32, height: u32) -> bool {width > 0 && height > 0 && u64::from(width) * u64::from(height) <= MAX_LOGICAL_PIXELS}-/// Content-free geometry for the qualified single-output, 1:1 desktop.-/// The common policy adapter uses this for display-scoped observation. Never-/// substitute a screenshot, XWayland root, or guessed primary monitor here.+/// Content-free logical geometry for a qualified single-output desktop: the+/// output mode divided by its scale, rounded as Hyprland rounds its logical+/// monitor size, plus that scale. Desktop capture, desktop action admission,+/// and the virtual-pointer extent all use this frame, matching the logical+/// window geometry and window captures. The common policy adapter uses it for+/// display-scoped observation. Never substitute a screenshot, XWayland root,+/// or guessed primary monitor here.pub fn screen_size() -> Result<(u32, u32, f64)> {screen_size_from_monitors(query("j/monitors")?)}@@ -328,13 +332,32 @@ fn screen_size_from_monitors(monitors: Vec<DisplayMonitor>) -> Result<(u32, u32,let [monitor] = monitors.as_slice() else {bail!("Hyprland display identity requires exactly one active output");};- if monitor.scale != 1.0 || monitor.transform != 0 || monitor.x != 0 || monitor.y != 0 {- bail!("Hyprland display identity requires an unscaled, unrotated output at the origin");+ if !monitor.scale.is_finite() || monitor.scale <= 0.0 {+ bail!("invalid Hyprland display scale");+ }+ if monitor.transform != 0 || monitor.x != 0 || monitor.y != 0 {+ bail!("Hyprland display identity requires an unrotated output at the origin");}if !valid_dimensions(monitor.width, monitor.height) {bail!("invalid Hyprland display dimensions");}- Ok((monitor.width, monitor.height, monitor.scale))++ let logical_width = (f64::from(monitor.width) / monitor.scale).round();+ let logical_height = (f64::from(monitor.height) / monitor.scale).round();+ if !logical_width.is_finite()+ || !logical_height.is_finite()+ || logical_width < 1.0+ || logical_height < 1.0+ || logical_width > f64::from(u32::MAX)+ || logical_height > f64::from(u32::MAX)+ {+ bail!("invalid Hyprland logical display dimensions");+ }+ let (logical_width, logical_height) = (logical_width as u32, logical_height as u32);+ if !valid_dimensions(logical_width, logical_height) {+ bail!("invalid Hyprland logical display dimensions");+ }+ Ok((logical_width, logical_height, monitor.scale))}pub fn list_windows() -> Result<Vec<Window>> {@@ -843,19 +866,86 @@ mod tests {.unwrap()}+ fn scaled_monitor(width: u32, height: u32, scale: f64) -> DisplayMonitor {+ let mut monitor = display_monitor();+ (monitor.width, monitor.height, monitor.scale) = (width, height, scale);+ monitor+ }++ // (output mode, scale, logical frame). 1.6666666 is the #4219 laptop.+ const SCALED_OUTPUTS: [((u32, u32), f64, (u32, u32)); 5] = [+ ((1920, 1080), 1.0, (1920, 1080)),+ ((2560, 1600), 1.25, (2048, 1280)),+ ((2880, 1800), 1.5, (1920, 1200)),+ ((2160, 1350), 1.6666666, (1296, 810)),+ ((3840, 2160), 2.0, (1920, 1080)),+ ];+#[test]- fn display_identity_accepts_qualified_native_geometry() {- assert_eq!(- screen_size_from_monitors(vec![display_monitor()]).unwrap(),- (1920, 1080, 1.0)- );+ fn display_identity_publishes_the_logical_frame_and_scale() {+ for ((width, height), scale, logical) in SCALED_OUTPUTS {+ assert_eq!(+ screen_size_from_monitors(vec![scaled_monitor(width, height, scale)]).unwrap(),+ (logical.0, logical.1, scale),+ "{width}x{height} @ {scale}"+ );+ }+ }++ /// Hyprland maps `motion_absolute(x, y, x_extent, y_extent)` onto its+ /// logical layout as `x / x_extent`. A desktop screenshot pixel must put+ /// the pointer on the physical pixel the agent saw in the native capture.+ #[test]+ fn scaled_desktop_frame_and_virtual_pointer_extent_agree() {+ for ((mode_w, mode_h), scale, _) in SCALED_OUTPUTS {+ let (frame_w, frame_h, _) =+ screen_size_from_monitors(vec![scaled_monitor(mode_w, mode_h, scale)]).unwrap();+ let extent = super::super::select_virtual_pointer_extent(+ (mode_w, mode_h),+ Some((frame_w, frame_h)),+ );+ assert_eq!(extent, (frame_w, frame_h), "{mode_w}x{mode_h} @ {scale}");++ let landed = |(x, y): (u32, u32), (extent_w, extent_h): (u32, u32)| {+ (+ f64::from(x) / f64::from(extent_w) * f64::from(frame_w) * scale,+ f64::from(y) / f64::from(extent_h) * f64::from(frame_h) * scale,+ )+ };+ for point in [+ (0, 0),+ (frame_w / 3, frame_h / 5),+ (frame_w / 2, frame_h / 2),+ (frame_w - 1, frame_h - 1),+ ] {+ // Desktop capture resizes the native buffer to the frame.+ let seen = (+ f64::from(point.0) * f64::from(mode_w) / f64::from(frame_w),+ f64::from(point.1) * f64::from(mode_h) / f64::from(frame_h),+ );+ let (x, y) = landed(point, extent);+ assert!(+ (x - seen.0).abs() < 0.5 && (y - seen.1).abs() < 0.5,+ "{mode_w}x{mode_h} @ {scale}: {point:?} landed at ({x}, {y}), saw {seen:?}"+ );+ }++ // The physical wl_output mode is the wrong extent on scaled+ // outputs: the pointer lands at 1 / scale of the target.+ if scale != 1.0 {+ let center = (frame_w / 2, frame_h / 2);+ let seen_x = f64::from(center.0) * f64::from(mode_w) / f64::from(frame_w);+ let (x, _) = landed(center, (mode_w, mode_h));+ assert!((x - seen_x / scale).abs() < 1.0 && (x - seen_x).abs() > 1.0);+ }+ }}#[test]fn display_identity_rejects_ambiguous_outputs_and_unsupported_frames() {assert!(screen_size_from_monitors(vec![]).is_err());assert!(screen_size_from_monitors(vec![display_monitor(), display_monitor()]).is_err());- for scale in [0.0, 1.25, 2.0, f64::NAN, f64::INFINITY] {+ for scale in [0.0, f64::NAN, f64::INFINITY] {let mut monitor = display_monitor();monitor.scale = scale;assert!(screen_size_from_monitors(vec![monitor]).is_err());diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rsindex 939a8ea773..df79a985f5 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs@@ -1254,10 +1254,49 @@ pub struct VptrSession {state: State,pub seat: WlSeat,pub vptr: ZwlrVirtualPointerV1,+ /// `motion_absolute` extent, in the same frame as caller coordinates.+ /// See [`virtual_pointer_extent`].pub output_w: u32,pub output_h: u32,}+/// Logical desktop frame published by a compositor adapter whose geometry is+/// in logical pixels, or `None` when callers keep the protocol-native frame+/// (for example the full-output screencopy buffer on Sway). Desktop capture+/// and the virtual pointer both read this, so a desktop screenshot pixel and a+/// pointer coordinate always share one frame.+pub(crate) fn compositor_logical_frame() -> Option<anyhow::Result<(u32, u32)>> {+ hyprland::is_session()+ .then(|| hyprland::screen_size().map(|(width, height, _scale)| (width, height)))+}++/// Extent to pass with `zwlr_virtual_pointer_v1::motion_absolute`. The+/// compositor maps `x / x_extent` onto its logical layout, so the extent must+/// be in the caller's coordinate frame. Hyprland window geometry, window+/// captures, and desktop captures are logical; a scaled output's `wl_output`+/// mode is physical and would land every motion at `1 / scale` of its target.+/// Layouts the Hyprland adapter cannot qualify (rotated, off-origin, or+/// multiple outputs) keep the first output's mode until that adapter+/// publishes a frame for them.+fn virtual_pointer_extent(output_mode: (u32, u32)) -> (u32, u32) {+ let logical = compositor_logical_frame().and_then(|frame| {+ frame+ .map_err(|error| {+ tracing::debug!("virtual pointer keeps the wl_output mode extent: {error:#}")+ })+ .ok()+ });+ select_virtual_pointer_extent(output_mode, logical)+}++fn select_virtual_pointer_extent(+ output_mode: (u32, u32),+ compositor_logical: Option<(u32, u32)>,+) -> (u32, u32) {+ let (width, height) = compositor_logical.unwrap_or(output_mode);+ (width.max(1), height.max(1))+}+/// Bind manager + seat + virtual-pointer + first output, optionally activate a/// foreign-toplevel by `window_id` so the synthesised events land on it, and/// return the live session that scroll / drag / click reuse. Wayland forbids a@@ -1324,7 +1363,7 @@ pub fn open_vptr_session(activate_window_id: Option<u64>) -> anyhow::Result<Vptr}let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());- let (output_w, output_h) = (state.output_w.max(1), state.output_h.max(1));+ let (output_w, output_h) = virtual_pointer_extent((state.output_w, state.output_h));Ok(VptrSession {conn,queue,