Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): harden GNOME 50 Shell capture (#2226)
cresslankd250e532dbf5
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/capture.rsindex e00a528494..e7d7b97121 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs@@ -106,21 +106,28 @@ pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> {/// Dispatch:/// - Native Wayland (`CUA_DRIVER_RS_ENABLE_WAYLAND=1` + Wayland session):/// routes through [`crate::wayland::screenshot_display_dispatch`] which-/// cascades wlroots screencopy → ext-image-copy-capture-v1 → portal-/// Screenshot. Each tier falls through to the next on missing globals.+/// owns the complete GNOME helper → wlroots screencopy →+/// ext-image-copy-capture-v1 → portal Screenshot → X11 cascade. An+/// available GNOME helper's capture failure is terminal./// - X11 / Wayland-disabled: ImageMagick `import` → x11rb `XGetImage`.pub fn screenshot_display_bytes() -> Result<Vec<u8>> {- if crate::wayland::is_wayland() {- match crate::wayland::screenshot_display_dispatch() {- Ok(bytes) => return Ok(bytes),- Err(e) => {- tracing::debug!(- "wayland screenshot cascade unavailable ({e}); falling through to X11"- );- }- }+ screenshot_display_bytes_with_dispatch(+ crate::wayland::is_wayland(),+ crate::wayland::screenshot_display_dispatch,+ screenshot_display_bytes_x11,+ )+}++fn screenshot_display_bytes_with_dispatch(+ wayland_enabled: bool,+ wayland_capture: impl FnOnce() -> Result<Vec<u8>>,+ x11_capture: impl FnOnce() -> Result<Vec<u8>>,+) -> Result<Vec<u8>> {+ if wayland_enabled {+ wayland_capture()+ } else {+ x11_capture()}- screenshot_display_bytes_x11()}/// X11-only display capture path — extracted so the wayland cascade in@@ -205,3 +212,46 @@ pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result<Vec<u8>> {pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result<Vec<u8>> {cua_driver_core::image_utils::crosshair_png_bytes(png_bytes, cx, cy)}++#[cfg(test)]+mod tests {+ use super::*;+ use std::cell::Cell;++ #[test]+ fn available_gnome_helper_failure_is_terminal_at_public_boundary() {+ let x11_called = Cell::new(false);++ let result = screenshot_display_bytes_with_dispatch(+ true,+ || Err(anyhow::anyhow!("GNOME compositor helper capture failed")),+ || {+ x11_called.set(true);+ Ok(vec![1, 2, 3])+ },+ );++ assert_eq!(+ result.unwrap_err().to_string(),+ "GNOME compositor helper capture failed"+ );+ assert!(!x11_called.get(), "public boundary retried X11 capture");+ }++ #[test]+ fn wayland_disabled_uses_x11_capture() {+ let wayland_called = Cell::new(false);++ let result = screenshot_display_bytes_with_dispatch(+ false,+ || {+ wayland_called.set(true);+ Err(anyhow::anyhow!("Wayland capture should not run"))+ },+ || Ok(vec![1, 2, 3]),+ );++ assert_eq!(result.unwrap(), vec![1, 2, 3]);+ assert!(!wayland_called.get());+ }+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rsindex 3747af3608..d379737538 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs@@ -925,18 +925,26 @@ fn crop_png_to_rect(}/// Display-level capture dispatcher. Cascade:-/// 1. Native Wayland on wlroots: zwlr_screencopy_manager_v1 (fast, zero+/// 1. Opt-in GNOME compositor helper. If available, capture failure is+/// terminal rather than cascading into GNOME's portal implementation.+/// 2. Native Wayland on wlroots: zwlr_screencopy_manager_v1 (fast, zero/// consent).-/// 2. Wayland but no wlroots screencopy globals (GNOME/KDE/COSMIC):-/// xdg-desktop-portal Screenshot via ashpd. Triggers consent prompt-/// on first use per session.-/// 3. X11: existing root-window path.+/// 3. ext-image-copy-capture-v1 on supported compositors.+/// 4. xdg-desktop-portal Screenshot via ashpd. Triggers a consent prompt on+/// first use per session.+/// 5. X11: existing root-window path.pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {if is_wayland() {// Tier 1: the opt-in GNOME compositor helper. It avoids probing// wlroots-only protocols and captures the Shell stage without consent.- if let Some(bytes) = shell_helper::screenshot_display() {- return Ok(bytes);+ // If the helper is present but capture fails, do not fall through to+ // GNOME's portal implementation: on GNOME 50 a malformed 0x0 cursor+ // sprite can crash Shell in GNOME's unsafe stage-content capture path.+ if let Some(result) = checked_shell_helper_capture(+ shell_helper::available(),+ shell_helper::screenshot_display,+ ) {+ return result;}// Tier 2: native wlroots screencopy (fast, zero consent).match screenshot_bytes() {@@ -973,6 +981,14 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {crate::capture::screenshot_display_bytes_x11()}+fn checked_shell_helper_capture(+ available: bool,+ capture: impl FnOnce() -> Option<Vec<u8>>,+) -> Option<anyhow::Result<Vec<u8>>> {+ available+ .then(|| capture().ok_or_else(|| anyhow::anyhow!("GNOME compositor helper capture failed")))+}+/// Per-window capture dispatcher. On X11 forwards to the existing window/// capture path; on pure Wayland returns a typed error pointing at the/// staging `ext-image-copy-capture-v1` protocol — wlr-screencopy is@@ -3322,6 +3338,27 @@ mod tests {assert_eq!((decoded.width(), decoded.height()), (3, 4));}+ #[test]+ fn shell_helper_capture_failure_is_terminal() {+ let result = checked_shell_helper_capture(true, || None)+ .expect("available helper must produce a terminal result");+ assert_eq!(+ result.unwrap_err().to_string(),+ "GNOME compositor helper capture failed"+ );+ }++ #[test]+ fn unavailable_shell_helper_does_not_attempt_capture() {+ let called = std::cell::Cell::new(false);+ let result = checked_shell_helper_capture(false, || {+ called.set(true);+ Some(vec![1, 2, 3])+ });+ assert!(result.is_none());+ assert!(!called.get());+ }+#[test]fn injectable_text_accepts_printable_ascii_newline_and_tab() {validate_injectable_text("Hello, World! 123 @#$%\t\n").expect("printable ASCII is typable");diff --git a/libs/cua-driver/wayland-helper/winrects@cua/extension.js b/libs/cua-driver/wayland-helper/winrects@cua/extension.jsindex 678382859f..c3ed8c1780 100644--- a/libs/cua-driver/wayland-helper/winrects@cua/extension.js+++ b/libs/cua-driver/wayland-helper/winrects@cua/extension.js@@ -7,8 +7,7 @@ import Cairo from 'cairo';import * as Main from 'resource:///org/gnome/shell/ui/main.js';import {Extension} from 'resource:///org/gnome/shell/extensions/extension.js';-Gio._promisify(Shell.Screenshot.prototype, 'screenshot_stage_to_content');-Gio._promisify(Shell.Screenshot, 'composite_to_stream');+Gio._promisify(Shell.Screenshot.prototype, 'screenshot');const IFACE = `<node><interface name="org.cua.WinRects"><method name="GetVersion"><arg type="u" direction="out" name="version"/></method>@@ -578,15 +577,16 @@ export default class WinRectsExtension extends Extension {async CaptureAsync(_params, invocation) {try {const shooter = new Shell.Screenshot();- const [content, scale] = await shooter.screenshot_stage_to_content();const stream = Gio.MemoryOutputStream.new_resizable();- await Shell.Screenshot.composite_to_stream(- content.get_texture(),- 0, 0, -1, -1,- scale,- null, 0, 0, 1,- stream- );+ const [width, height] = global.display.get_size();+ if (width <= 0 || height <= 0)+ throw new Error(`Invalid display size ${width}x${height}`);+ // GNOME 50's stage-content capture copies the real cursor sprite+ // after painting the stage. Remote/headless pointer seats can+ // expose a 0x0 sprite, and that copy can crash Shell. The stream+ // API has an explicit include_cursor flag, so keep cursor capture+ // disabled; cua-driver draws its own agent cursor.+ await shooter.screenshot(false, stream);stream.close(null);const encoded = GLib.base64_encode(stream.steal_as_bytes().get_data());invocation.return_value(new GLib.Variant('(s)', [encoded]));