Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): support rotated Hyprland outputs (#3969)

Osamah Beig69cf20a32d3a
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs
index ab5d7f3df7..ec46d746d4 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/hyprland.rs
@@ -345,15 +345,20 @@ fn screen_size_from_monitors(monitors: Vec<DisplayMonitor>) -> Result<(u32, u32,
if !monitor.scale.is_finite() || monitor.scale <= 0.0 {
bail!("invalid Hyprland display scale");
}
- if monitor.transform != 0 || monitor.x != 0 || monitor.y != 0 {
- bail!("Hyprland display identity requires an unrotated output at the origin");
+ // Hyprland reports the mode in the panel's native orientation. Rotated
+ // outputs (wl_output transforms 1-3) are supported by reporting the logical
+ // frame that windows and input use; flipped transforms (4-7) still refuse.
+ if monitor.transform > 3 || monitor.x != 0 || monitor.y != 0 {
+ bail!("Hyprland display identity requires an unflipped output at the origin");
}
if !valid_dimensions(monitor.width, monitor.height) {
bail!("invalid Hyprland display dimensions");
}
- let logical_width = (f64::from(monitor.width) / monitor.scale).round();
- let logical_height = (f64::from(monitor.height) / monitor.scale).round();
+ let (width, height) =
+ super::logical_output_size((monitor.width, monitor.height), monitor.transform);
+ let logical_width = (f64::from(width) / monitor.scale).round();
+ let logical_height = (f64::from(height) / monitor.scale).round();
if !logical_width.is_finite()
|| !logical_height.is_finite()
|| logical_width < 1.0
@@ -370,6 +375,17 @@ fn screen_size_from_monitors(monitors: Vec<DisplayMonitor>) -> Result<(u32, u32,
Ok((logical_width, logical_height, monitor.scale))
}
+/// wl_output transform of the single active output. Full-display screencopy
+/// frames arrive in the panel's native orientation; callers use this to turn
+/// them into the logical desktop frame.
+pub fn single_output_transform() -> Result<u32> {
+ let monitors: Vec<DisplayMonitor> = query("j/monitors")?;
+ let [monitor] = monitors.as_slice() else {
+ bail!("Hyprland display identity requires exactly one active output");
+ };
+ Ok(monitor.transform)
+}
+
pub fn list_windows() -> Result<Vec<Window>> {
let monitors: Vec<Monitor> = query("j/monitors")?;
let active = monitors
@@ -983,6 +999,16 @@ mod tests {
}
}
+ #[test]
+ fn display_identity_reports_the_logical_frame_of_rotated_outputs() {
+ for (transform, expected) in [(1, (1080, 1920)), (2, (1920, 1080)), (3, (1080, 1920))] {
+ let mut monitor = display_monitor();
+ monitor.transform = transform;
+ let (width, height, _) = screen_size_from_monitors(vec![monitor]).unwrap();
+ assert_eq!((width, height), expected, "transform {transform}");
+ }
+ }
+
#[test]
fn display_identity_rejects_ambiguous_outputs_and_unsupported_frames() {
assert!(screen_size_from_monitors(vec![]).is_err());
@@ -992,7 +1018,7 @@ mod tests {
monitor.scale = scale;
assert!(screen_size_from_monitors(vec![monitor]).is_err());
}
- for (x, y, transform) in [(100, 0, 0), (0, -100, 0), (0, 0, 1), (0, 0, 7)] {
+ for (x, y, transform) in [(100, 0, 0), (0, -100, 0), (0, 0, 4), (0, 0, 7)] {
let mut monitor = display_monitor();
(monitor.x, monitor.y, monitor.transform) = (x, y, transform);
assert!(screen_size_from_monitors(vec![monitor]).is_err());
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
index 0a7379fb23..c846df1d98 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
@@ -394,8 +394,12 @@ struct State {
// button press at the output centre (over the just-activated window).
vptr_manager: Option<ZwlrVirtualPointerManagerV1>,
output: Option<WlOutput>,
+ // Logical (post-transform) output size: virtual-pointer absolute motion
+ // maps onto the output's logical box, so rotated outputs swap axes.
output_w: u32,
output_h: u32,
+ output_mode: (u32, u32),
+ output_transform: u32,
// Native screencopy capture state.
scrcopy_manager: Option<ZwlrScreencopyManagerV1>,
shm: Option<WlShm>,
@@ -477,10 +481,22 @@ impl Dispatch<WlOutput, ()> for State {
_: &QueueHandle<Self>,
) {
// Remember the output resolution so `click` can aim at its centre.
- if let wl_output::Event::Mode { width, height, .. } = event {
- state.output_w = width.max(0) as u32;
- state.output_h = height.max(0) as u32;
+ // Modes are reported in the panel's native orientation; the geometry
+ // transform turns them into the logical frame input coordinates use.
+ match event {
+ wl_output::Event::Mode { width, height, .. } => {
+ state.output_mode = (width.max(0) as u32, height.max(0) as u32);
+ }
+ wl_output::Event::Geometry {
+ transform: WEnum::Value(transform),
+ ..
+ } => {
+ state.output_transform = transform as u32;
+ }
+ _ => return,
}
+ (state.output_w, state.output_h) =
+ logical_output_size(state.output_mode, state.output_transform);
}
}
@@ -1152,7 +1168,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
}
// Tier 2: native wlroots screencopy (fast, zero consent).
match screenshot_bytes() {
- Ok(bytes) => return Ok(bytes),
+ Ok(bytes) => return orient_hyprland_display_png(bytes),
Err(e) => {
tracing::debug!(
"wlroots screencopy unavailable ({e}); trying ext-image-copy-capture-v1"
@@ -1162,7 +1178,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
// Tier 3: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri,
// hyprland, KDE 6.2+, GNOME 47+).
match ext_screencopy::screenshot_via_ext_copy() {
- Ok(bytes) => return Ok(bytes),
+ Ok(bytes) => return orient_hyprland_display_png(bytes),
Err(e) => {
tracing::debug!(
"ext-image-copy-capture-v1 unavailable ({e}); trying xdg-desktop-portal"
@@ -1185,6 +1201,43 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
crate::capture::screenshot_display_bytes_x11()
}
+/// Logical size of an output whose native mode is `mode` under wl_output
+/// `transform`: quarter turns (odd transforms) swap the axes.
+pub(crate) fn logical_output_size(mode: (u32, u32), transform: u32) -> (u32, u32) {
+ if transform % 2 == 1 {
+ (mode.1, mode.0)
+ } else {
+ mode
+ }
+}
+
+/// Rotate a native-orientation output capture into the logical frame for a
+/// wl_output transform. `image::rotate90` is clockwise, which is what a
+/// transform-1 (portrait) Hyprland output needs. Flipped transforms refuse.
+fn rotate_png_for_output_transform(png: Vec<u8>, transform: u32) -> anyhow::Result<Vec<u8>> {
+ let rotate: fn(&image::DynamicImage) -> image::DynamicImage = match transform {
+ 0 => return Ok(png),
+ 1 => image::DynamicImage::rotate90,
+ 2 => image::DynamicImage::rotate180,
+ 3 => image::DynamicImage::rotate270,
+ other => anyhow::bail!("flipped output transform {other} is not supported"),
+ };
+ let img = image::load_from_memory_with_format(&png, image::ImageFormat::Png)?;
+ let mut out = Vec::new();
+ rotate(&img).write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?;
+ Ok(out)
+}
+
+/// Full-display screencopy returns the output buffer in the panel's native
+/// orientation. On Hyprland, turn it into the logical desktop frame so the
+/// screenshot matches what the user sees and what desktop-scope input uses.
+fn orient_hyprland_display_png(png: Vec<u8>) -> anyhow::Result<Vec<u8>> {
+ if !hyprland::is_session() {
+ return Ok(png);
+ }
+ rotate_png_for_output_transform(png, hyprland::single_output_transform()?)
+}
+
fn checked_shell_helper_capture(
available: bool,
capture: impl FnOnce() -> Option<Vec<u8>>,
@@ -3556,6 +3609,61 @@ impl Dispatch<wl_registry::WlRegistry, ()> for ExtProbeState {
#[allow(dead_code)]
const _BTN_LEFT_ALIAS: u32 = BTN_LEFT;
+#[cfg(test)]
+mod output_transform_tests {
+ use super::{logical_output_size, rotate_png_for_output_transform};
+
+ #[test]
+ fn logical_output_size_swaps_axes_for_quarter_turns() {
+ assert_eq!(logical_output_size((2560, 1080), 0), (2560, 1080));
+ assert_eq!(logical_output_size((2560, 1080), 1), (1080, 2560));
+ assert_eq!(logical_output_size((2560, 1080), 2), (2560, 1080));
+ assert_eq!(logical_output_size((2560, 1080), 3), (1080, 2560));
+ assert_eq!(logical_output_size((2560, 1080), 5), (1080, 2560));
+ }
+
+ fn two_pixel_png() -> Vec<u8> {
+ // Native frame: red on the left, blue on the right.
+ let mut img = image::RgbaImage::new(2, 1);
+ img.put_pixel(0, 0, image::Rgba([255, 0, 0, 255]));
+ img.put_pixel(1, 0, image::Rgba([0, 0, 255, 255]));
+ let mut png = Vec::new();
+ image::DynamicImage::ImageRgba8(img)
+ .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
+ .unwrap();
+ png
+ }
+
+ fn decode(png: &[u8]) -> image::RgbaImage {
+ image::load_from_memory_with_format(png, image::ImageFormat::Png)
+ .unwrap()
+ .to_rgba8()
+ }
+
+ #[test]
+ fn capture_rotation_follows_the_output_transform() {
+ let red = image::Rgba([255, 0, 0, 255]);
+ let blue = image::Rgba([0, 0, 255, 255]);
+ let same = rotate_png_for_output_transform(two_pixel_png(), 0).unwrap();
+ assert_eq!(same, two_pixel_png());
+ let quarter = decode(&rotate_png_for_output_transform(two_pixel_png(), 1).unwrap());
+ assert_eq!(quarter.dimensions(), (1, 2));
+ assert_eq!(
+ (*quarter.get_pixel(0, 0), *quarter.get_pixel(0, 1)),
+ (red, blue)
+ );
+ let half = decode(&rotate_png_for_output_transform(two_pixel_png(), 2).unwrap());
+ assert_eq!((*half.get_pixel(0, 0), *half.get_pixel(1, 0)), (blue, red));
+ let three = decode(&rotate_png_for_output_transform(two_pixel_png(), 3).unwrap());
+ assert_eq!(three.dimensions(), (1, 2));
+ assert_eq!(
+ (*three.get_pixel(0, 0), *three.get_pixel(0, 1)),
+ (blue, red)
+ );
+ assert!(rotate_png_for_output_transform(two_pixel_png(), 5).is_err());
+ }
+}
+
#[cfg(test)]
mod tests {
/// The pointer always moves before a desktop click or scroll, even when