Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): support rotated Hyprland outputs (#3969)
Osamah Beig69cf20a32d3a
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rsindex ab5d7f3df7..ec46d746d4 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs@@ -345,15 +345,20 @@ fn screen_size_from_monitors(monitors: Vec<DisplayMonitor>) -> Result<(u32, u32,if !monitor.scale.is_finite() || monitor.scale <= 0.0 {bail!("invalid Hyprland display scale");}- if monitor.transform != 0 || monitor.x != 0 || monitor.y != 0 {- bail!("Hyprland display identity requires an unrotated output at the origin");+ // Hyprland reports the mode in the panel's native orientation. Rotated+ // outputs (wl_output transforms 1-3) are supported by reporting the logical+ // frame that windows and input use; flipped transforms (4-7) still refuse.+ if monitor.transform > 3 || monitor.x != 0 || monitor.y != 0 {+ bail!("Hyprland display identity requires an unflipped output at the origin");}if !valid_dimensions(monitor.width, monitor.height) {bail!("invalid Hyprland display dimensions");}- let logical_width = (f64::from(monitor.width) / monitor.scale).round();- let logical_height = (f64::from(monitor.height) / monitor.scale).round();+ let (width, height) =+ super::logical_output_size((monitor.width, monitor.height), monitor.transform);+ let logical_width = (f64::from(width) / monitor.scale).round();+ let logical_height = (f64::from(height) / monitor.scale).round();if !logical_width.is_finite()|| !logical_height.is_finite()|| logical_width < 1.0@@ -370,6 +375,17 @@ fn screen_size_from_monitors(monitors: Vec<DisplayMonitor>) -> Result<(u32, u32,Ok((logical_width, logical_height, monitor.scale))}+/// wl_output transform of the single active output. Full-display screencopy+/// frames arrive in the panel's native orientation; callers use this to turn+/// them into the logical desktop frame.+pub fn single_output_transform() -> Result<u32> {+ let monitors: Vec<DisplayMonitor> = query("j/monitors")?;+ let [monitor] = monitors.as_slice() else {+ bail!("Hyprland display identity requires exactly one active output");+ };+ Ok(monitor.transform)+}+pub fn list_windows() -> Result<Vec<Window>> {let monitors: Vec<Monitor> = query("j/monitors")?;let active = monitors@@ -983,6 +999,16 @@ mod tests {}}+ #[test]+ fn display_identity_reports_the_logical_frame_of_rotated_outputs() {+ for (transform, expected) in [(1, (1080, 1920)), (2, (1920, 1080)), (3, (1080, 1920))] {+ let mut monitor = display_monitor();+ monitor.transform = transform;+ let (width, height, _) = screen_size_from_monitors(vec![monitor]).unwrap();+ assert_eq!((width, height), expected, "transform {transform}");+ }+ }+#[test]fn display_identity_rejects_ambiguous_outputs_and_unsupported_frames() {assert!(screen_size_from_monitors(vec![]).is_err());@@ -992,7 +1018,7 @@ mod tests {monitor.scale = scale;assert!(screen_size_from_monitors(vec![monitor]).is_err());}- for (x, y, transform) in [(100, 0, 0), (0, -100, 0), (0, 0, 1), (0, 0, 7)] {+ for (x, y, transform) in [(100, 0, 0), (0, -100, 0), (0, 0, 4), (0, 0, 7)] {let mut monitor = display_monitor();(monitor.x, monitor.y, monitor.transform) = (x, y, transform);assert!(screen_size_from_monitors(vec![monitor]).is_err());diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rsindex 0a7379fb23..c846df1d98 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs@@ -394,8 +394,12 @@ struct State {// button press at the output centre (over the just-activated window).vptr_manager: Option<ZwlrVirtualPointerManagerV1>,output: Option<WlOutput>,+ // Logical (post-transform) output size: virtual-pointer absolute motion+ // maps onto the output's logical box, so rotated outputs swap axes.output_w: u32,output_h: u32,+ output_mode: (u32, u32),+ output_transform: u32,// Native screencopy capture state.scrcopy_manager: Option<ZwlrScreencopyManagerV1>,shm: Option<WlShm>,@@ -477,10 +481,22 @@ impl Dispatch<WlOutput, ()> for State {_: &QueueHandle<Self>,) {// Remember the output resolution so `click` can aim at its centre.- if let wl_output::Event::Mode { width, height, .. } = event {- state.output_w = width.max(0) as u32;- state.output_h = height.max(0) as u32;+ // Modes are reported in the panel's native orientation; the geometry+ // transform turns them into the logical frame input coordinates use.+ match event {+ wl_output::Event::Mode { width, height, .. } => {+ state.output_mode = (width.max(0) as u32, height.max(0) as u32);+ }+ wl_output::Event::Geometry {+ transform: WEnum::Value(transform),+ ..+ } => {+ state.output_transform = transform as u32;+ }+ _ => return,}+ (state.output_w, state.output_h) =+ logical_output_size(state.output_mode, state.output_transform);}}@@ -1152,7 +1168,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {}// Tier 2: native wlroots screencopy (fast, zero consent).match screenshot_bytes() {- Ok(bytes) => return Ok(bytes),+ Ok(bytes) => return orient_hyprland_display_png(bytes),Err(e) => {tracing::debug!("wlroots screencopy unavailable ({e}); trying ext-image-copy-capture-v1"@@ -1162,7 +1178,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {// Tier 3: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri,// hyprland, KDE 6.2+, GNOME 47+).match ext_screencopy::screenshot_via_ext_copy() {- Ok(bytes) => return Ok(bytes),+ Ok(bytes) => return orient_hyprland_display_png(bytes),Err(e) => {tracing::debug!("ext-image-copy-capture-v1 unavailable ({e}); trying xdg-desktop-portal"@@ -1185,6 +1201,43 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {crate::capture::screenshot_display_bytes_x11()}+/// Logical size of an output whose native mode is `mode` under wl_output+/// `transform`: quarter turns (odd transforms) swap the axes.+pub(crate) fn logical_output_size(mode: (u32, u32), transform: u32) -> (u32, u32) {+ if transform % 2 == 1 {+ (mode.1, mode.0)+ } else {+ mode+ }+}++/// Rotate a native-orientation output capture into the logical frame for a+/// wl_output transform. `image::rotate90` is clockwise, which is what a+/// transform-1 (portrait) Hyprland output needs. Flipped transforms refuse.+fn rotate_png_for_output_transform(png: Vec<u8>, transform: u32) -> anyhow::Result<Vec<u8>> {+ let rotate: fn(&image::DynamicImage) -> image::DynamicImage = match transform {+ 0 => return Ok(png),+ 1 => image::DynamicImage::rotate90,+ 2 => image::DynamicImage::rotate180,+ 3 => image::DynamicImage::rotate270,+ other => anyhow::bail!("flipped output transform {other} is not supported"),+ };+ let img = image::load_from_memory_with_format(&png, image::ImageFormat::Png)?;+ let mut out = Vec::new();+ rotate(&img).write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?;+ Ok(out)+}++/// Full-display screencopy returns the output buffer in the panel's native+/// orientation. On Hyprland, turn it into the logical desktop frame so the+/// screenshot matches what the user sees and what desktop-scope input uses.+fn orient_hyprland_display_png(png: Vec<u8>) -> anyhow::Result<Vec<u8>> {+ if !hyprland::is_session() {+ return Ok(png);+ }+ rotate_png_for_output_transform(png, hyprland::single_output_transform()?)+}+fn checked_shell_helper_capture(available: bool,capture: impl FnOnce() -> Option<Vec<u8>>,@@ -3556,6 +3609,61 @@ impl Dispatch<wl_registry::WlRegistry, ()> for ExtProbeState {#[allow(dead_code)]const _BTN_LEFT_ALIAS: u32 = BTN_LEFT;+#[cfg(test)]+mod output_transform_tests {+ use super::{logical_output_size, rotate_png_for_output_transform};++ #[test]+ fn logical_output_size_swaps_axes_for_quarter_turns() {+ assert_eq!(logical_output_size((2560, 1080), 0), (2560, 1080));+ assert_eq!(logical_output_size((2560, 1080), 1), (1080, 2560));+ assert_eq!(logical_output_size((2560, 1080), 2), (2560, 1080));+ assert_eq!(logical_output_size((2560, 1080), 3), (1080, 2560));+ assert_eq!(logical_output_size((2560, 1080), 5), (1080, 2560));+ }++ fn two_pixel_png() -> Vec<u8> {+ // Native frame: red on the left, blue on the right.+ let mut img = image::RgbaImage::new(2, 1);+ img.put_pixel(0, 0, image::Rgba([255, 0, 0, 255]));+ img.put_pixel(1, 0, image::Rgba([0, 0, 255, 255]));+ let mut png = Vec::new();+ image::DynamicImage::ImageRgba8(img)+ .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)+ .unwrap();+ png+ }++ fn decode(png: &[u8]) -> image::RgbaImage {+ image::load_from_memory_with_format(png, image::ImageFormat::Png)+ .unwrap()+ .to_rgba8()+ }++ #[test]+ fn capture_rotation_follows_the_output_transform() {+ let red = image::Rgba([255, 0, 0, 255]);+ let blue = image::Rgba([0, 0, 255, 255]);+ let same = rotate_png_for_output_transform(two_pixel_png(), 0).unwrap();+ assert_eq!(same, two_pixel_png());+ let quarter = decode(&rotate_png_for_output_transform(two_pixel_png(), 1).unwrap());+ assert_eq!(quarter.dimensions(), (1, 2));+ assert_eq!(+ (*quarter.get_pixel(0, 0), *quarter.get_pixel(0, 1)),+ (red, blue)+ );+ let half = decode(&rotate_png_for_output_transform(two_pixel_png(), 2).unwrap());+ assert_eq!((*half.get_pixel(0, 0), *half.get_pixel(1, 0)), (blue, red));+ let three = decode(&rotate_png_for_output_transform(two_pixel_png(), 3).unwrap());+ assert_eq!(three.dimensions(), (1, 2));+ assert_eq!(+ (*three.get_pixel(0, 0), *three.get_pixel(0, 1)),+ (blue, red)+ );+ assert!(rotate_png_for_output_transform(two_pixel_png(), 5).is_err());+ }+}+#[cfg(test)]mod tests {/// The pointer always moves before a desktop click or scroll, even when