Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-spacesd): stream 6K displays and carry tool results over 4 MiB (#4850)
dduponta88f100b428a
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/h264_limits.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/h264_limits.rsnew file mode 100644index 0000000000..e43207f5c7--- /dev/null+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/h264_limits.rs@@ -0,0 +1,108 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The largest frame an H.264 stream encodes: the VideoToolbox hardware+//! encoder (and H.264 level 5.2) tops out at 4096 px on the long edge and+//! 36,864 macroblocks (4096x2304). A larger source, such as a 6K display+//! (6720x3780), is captured downscaled to fit instead of failing to encode.++/// Longest edge, in pixels, of an H.264 frame.+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]+pub(crate) const H264_MAX_DIMENSION: u32 = 4096;+/// Most 16x16 macroblocks in one H.264 frame (level 5.2 MaxFS).+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]+pub(crate) const H264_MAX_MACROBLOCKS: u64 = 36_864;++/// The long-edge cap an H.264 stream of a `width`x`height` (native px)+/// source captures at: `requested` (0 = native size) lowered, when needed,+/// so the downscaled frame fits [`H264_MAX_DIMENSION`] and+/// [`H264_MAX_MACROBLOCKS`]. Never 0.+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]+pub(crate) fn h264_max_dimension(width: u32, height: u32, requested: u32) -> u32 {+ let (width, height) = (width.max(1), height.max(1));+ let longest = width.max(height);+ let mut cap = match requested {+ 0 => longest,+ r => r.min(longest),+ }+ .min(H264_MAX_DIMENSION);+ // The frame area shrinks with the square of the cap: start from the+ // cap whose area fits, then step down (bounded) past rounding.+ if macroblocks(width, height, cap) > H264_MAX_MACROBLOCKS {+ let fit =+ (H264_MAX_MACROBLOCKS as f64 * 256.0 / (f64::from(width) * f64::from(height))).sqrt();+ // (The epsilon keeps an exact fit, 3072 for 5000x5000, from+ // rounding down a pixel.)+ cap = cap+ .min((f64::from(longest) * fit + 1e-6).floor() as u32)+ .max(16);+ for _ in 0..64 {+ if macroblocks(width, height, cap) <= H264_MAX_MACROBLOCKS || cap <= 16 {+ break;+ }+ cap -= 1;+ }+ }+ cap.max(1)+}++/// Macroblocks of `width`x`height` scaled so its long edge is at most `cap`.+fn macroblocks(width: u32, height: u32, cap: u32) -> u64 {+ let longest = width.max(height);+ let (w, h) = if longest <= cap {+ (width, height)+ } else {+ let ratio = f64::from(cap) / f64::from(longest);+ (+ ((f64::from(width) * ratio).round() as u32).max(1),+ ((f64::from(height) * ratio).round() as u32).max(1),+ )+ };+ u64::from(w.div_ceil(16)) * u64::from(h.div_ceil(16))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn fits(width: u32, height: u32, requested: u32) {+ let cap = h264_max_dimension(width, height, requested);+ assert!(cap <= H264_MAX_DIMENSION, "{width}x{height}: {cap}");+ assert!(+ macroblocks(width, height, cap) <= H264_MAX_MACROBLOCKS,+ "{width}x{height}: {cap}"+ );+ }++ /// A 6K main display (6720x3780) streamed at native size (the viewer's+ /// `max_dimension` 0) is captured at 4096x2304, which H.264 encodes+ /// (#4623); before, the 6720x3780 frame went to the encoder as is.+ #[test]+ fn a_6k_display_is_downscaled_to_the_h264_limit() {+ assert_eq!(h264_max_dimension(6720, 3780, 0), 4096);+ assert_eq!(macroblocks(6720, 3780, 4096), 36_864);+ // Portrait 5K (2880x5120): 2304x4096.+ assert_eq!(h264_max_dimension(2880, 5120, 0), 4096);+ // A square source over the macroblock budget shrinks further.+ assert_eq!(h264_max_dimension(5000, 5000, 0), 3072);+ for (w, h) in [+ (6720, 3780),+ (7680, 4320),+ (5120, 2880),+ (5000, 5000),+ (3840, 3840),+ ] {+ for requested in [0, 1280, 4096, 8192] {+ fits(w, h, requested);+ }+ }+ }++ #[test]+ fn a_frame_within_the_limit_keeps_the_requested_cap() {+ assert_eq!(h264_max_dimension(2560, 1440, 0), 2560);+ assert_eq!(h264_max_dimension(2560, 1440, 1920), 1920);+ assert_eq!(h264_max_dimension(6720, 3780, 1600), 1600);+ assert_eq!(h264_max_dimension(3840, 2160, 0), 3840);+ }+}diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rsindex 8606f9585d..e2b8cd2fd0 100644--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs@@ -43,6 +43,7 @@ mod driver_input;#[cfg(target_os = "linux")]mod encoded_capture;pub mod grpc;+mod h264_limits;#[cfg(target_os = "linux")]mod linux_capture;#[cfg(target_os = "linux")]diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rsindex 446ec7989c..c29c3e98b3 100644--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rs@@ -203,11 +203,13 @@ fn start_inner(let filter = &located.filter;let native_scale = f64::from(filter.point_pixel_scale()).max(1.0);+ let h264 = encoder.is_some();let initial_geometry = capture_geometry(located.width_points,located.height_points,native_scale,config.max_dimension,+ h264,);native_geometry(initial_geometry.native_width_px,@@ -324,6 +326,7 @@ fn start_inner(let monitor = spawn_monitor(source,config.max_dimension,+ h264,frame_interval,stream.clone(),sink,@@ -346,6 +349,7 @@ fn start_inner(fn spawn_monitor(source: CaptureSource,max_dimension: u32,+ h264: bool,frame_interval: screencapturekit::cm::CMTime,stream: Arc<SCStream>,sink: Arc<dyn CaptureSink>,@@ -408,6 +412,7 @@ fn spawn_monitor(located.height_points,f64::from(located.filter.point_pixel_scale()),max_dimension,+ h264,);let current = *expected_geometry.lock()@@ -464,16 +469,24 @@ fn frame_status_allows_content(status: Option<SCFrameStatus>) -> bool {status.is_none_or(SCFrameStatus::has_content)}+/// The frame size a capture delivers. An `h264` stream is also kept within+/// what the encoder accepts (a 6K display is downscaled, not refused).fn capture_geometry(width_points: f64,height_points: f64,native_scale: f64,max_dimension: u32,+ h264: bool,) -> CaptureGeometry {let width_points = width_points.max(1.0);let height_points = height_points.max(1.0);let native_width = (width_points * native_scale.max(1.0)).round() as u32;let native_height = (height_points * native_scale.max(1.0)).round() as u32;+ let max_dimension = if h264 {+ crate::h264_limits::h264_max_dimension(native_width, native_height, max_dimension)+ } else {+ max_dimension+ };let (width_px, height_px) = fit_dimensions(native_width, native_height, max_dimension);CaptureGeometry {width_px,@@ -567,11 +580,26 @@ mod tests {#[test]fn capture_geometry_keeps_native_action_dimensions_when_stream_is_scaled() {- let geometry = capture_geometry(960.0, 480.0, 2.0, 1280);+ let geometry = capture_geometry(960.0, 480.0, 2.0, 1280, true);assert_eq!(geometry.width_px, 1280);assert_eq!(geometry.height_px, 640);assert_eq!(geometry.native_width_px, 1920);assert_eq!(geometry.native_height_px, 960);assert!((geometry.scale_factor - (4.0 / 3.0)).abs() < f64::EPSILON);}++ /// #4623: a 6K main display (3360x1890 pt @2x) streamed at native size+ /// is encoded at 4096x2304, while clicks still map to native pixels; a+ /// one-frame (non-H.264) grab keeps the native size.+ #[test]+ fn a_6k_display_streams_within_the_h264_limit() {+ let geometry = capture_geometry(3360.0, 1890.0, 2.0, 0, true);+ assert_eq!((geometry.width_px, geometry.height_px), (4096, 2304));+ assert_eq!(+ (geometry.native_width_px, geometry.native_height_px),+ (6720, 3780)+ );+ let still = capture_geometry(3360.0, 1890.0, 2.0, 0, false);+ assert_eq!((still.width_px, still.height_px), (6720, 3780));+ }}diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rsindex 11d9a397e3..06d65a0a36 100644--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rs@@ -374,7 +374,8 @@ fn encoder_loop(Ok(encoder) => native = Some(encoder),Err(error) => {sink.on_event(CaptureEvent::Suspended(format!(- "VideoToolbox encoder creation failed: {error}"+ "VideoToolbox encoder creation failed for a {}x{} frame: {error}",+ dimensions.0, dimensions.1)));continue;}diff --git a/libs/cua/crates/cua-daemon/src/client.rs b/libs/cua/crates/cua-daemon/src/client.rsindex 5aebc98968..1664464434 100644--- a/libs/cua/crates/cua-daemon/src/client.rs+++ b/libs/cua/crates/cua-daemon/src/client.rs@@ -178,15 +178,24 @@ impl DaemonClient {fn runtime(&self) -> RuntimeServiceClient<Chan> {RuntimeServiceClient::new(self.channel.clone())+ .max_decoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)+ .max_encoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)}fn daemon(&self) -> DaemonServiceClient<Chan> {DaemonServiceClient::new(self.channel.clone())+ .max_decoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)+ .max_encoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)}- /// The raw Spaces client.+ /// The raw Spaces client. Messages up to+ /// [`cua_spacesd_client::MAX_MESSAGE_BYTES`], as on the Space's own+ /// connection: a full-size screenshot (`get_desktop_state` of a 6K+ /// display) is larger than gRPC's 4 MiB default.pub fn spaces(&self) -> SpaceServiceClient<Chan> {SpaceServiceClient::new(self.channel.clone())+ .max_decoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)+ .max_encoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)}fn map(e: tonic::Status) -> Error {diff --git a/libs/cua/crates/cua-daemon/src/server.rs b/libs/cua/crates/cua-daemon/src/server.rsindex d91f66f3f8..508a59823b 100644--- a/libs/cua/crates/cua-daemon/src/server.rs+++ b/libs/cua/crates/cua-daemon/src/server.rs@@ -463,10 +463,29 @@ async fn accept_loop<L: Accept>(listener: L, app: BoxedApp, mut shutdown: watch:fn app(shared: Arc<Shared>, require_token: bool) -> BoxedApp {let svc = Svc(shared.clone());- let grpc = tonic::service::Routes::new(SandboxServiceServer::new(svc.clone()))- .add_service(SpaceServiceServer::new(svc.clone()))- .add_service(RuntimeServiceServer::new(svc.clone()))- .add_service(DaemonServiceServer::new(svc.clone()));+ // The clients' limit, not gRPC's 4 MiB default: a tool call can carry+ // a full-size screenshot or file either way.+ let limit = cua_spacesd_client::MAX_MESSAGE_BYTES;+ let grpc = tonic::service::Routes::new(+ SandboxServiceServer::new(svc.clone())+ .max_decoding_message_size(limit)+ .max_encoding_message_size(limit),+ )+ .add_service(+ SpaceServiceServer::new(svc.clone())+ .max_decoding_message_size(limit)+ .max_encoding_message_size(limit),+ )+ .add_service(+ RuntimeServiceServer::new(svc.clone())+ .max_decoding_message_size(limit)+ .max_encoding_message_size(limit),+ )+ .add_service(+ DaemonServiceServer::new(svc.clone())+ .max_decoding_message_size(limit)+ .max_encoding_message_size(limit),+ );#[cfg(feature = "spaces")]let grpc = grpc.add_service(cua_proto::env::v1::host_spaces_service_server::HostSpacesServiceServer::new(diff --git a/libs/cua/crates/cua-daemon/tests/large_messages.rs b/libs/cua/crates/cua-daemon/tests/large_messages.rsnew file mode 100644index 0000000000..aa01950665--- /dev/null+++ b/libs/cua/crates/cua-daemon/tests/large_messages.rs@@ -0,0 +1,87 @@+//! A Spaces tool call through the daemon carries more than gRPC's 4 MiB+//! default each way (#4623: `get_desktop_state` of a 6K display returned a+//! 9.5 MB screenshot and failed with "decoded message length too large").+//! Temp dirs and a loopback mock spacesd only.++use cua_daemon::client::{DaemonAddress, DaemonClient};+use cua_daemon::server::{ServerConfig, start};+use cua_daemon::{Runtime as DaemonRuntime, RuntimeConfig};+use cua_proto::daemon::v1 as pb;+use cua_spaces::thumbnails::Thumbnail;+use cua_spacesd_client::testing::{MockAuth, MockServer};+use std::time::{Duration, SystemTime};++#[tokio::test]+async fn a_tool_call_larger_than_4_mib_round_trips_through_the_daemon() {+ let dir = tempfile::tempdir().unwrap();+ let runtime = DaemonRuntime::new(RuntimeConfig {+ state_dir: Some(dir.path().join("sandboxes")),+ spaces_home: Some(dir.path().to_path_buf()),+ ..Default::default()+ })+ .unwrap();+ let handle = start(+ runtime.clone(),+ ServerConfig {+ socket_path: None,+ loopback: Some(([127, 0, 0, 1], 0).into()),+ token: "t0k3n".into(),+ discovery_path: None,+ bridge_ticket_ttl: Duration::from_secs(60),+ },+ )+ .await+ .unwrap();+ let client = DaemonClient::new(DaemonAddress::Url {+ url: handle.loopback_url.clone().unwrap(),+ token: handle.token.clone(),+ })+ .unwrap();+ let env = MockServer::start(MockAuth::default()).await;+ let space = runtime+ .spaces()+ .add(&env.url(), None, Some("big".into()))+ .await+ .unwrap();++ // More than 4 MiB to the daemon: the tool call's arguments.+ let word = "a".repeat(5 << 20);+ let r = client+ .spaces()+ .call_space_tool(pb::CallSpaceToolRequest {+ name: "space_bash".into(),+ arguments_json: serde_json::json!({+ "space": space.id,+ "command": format!("echo {word}"),+ })+ .to_string(),+ })+ .await+ .unwrap()+ .into_inner();+ assert!(!r.is_error, "{}", r.content_json);+ assert!(r.content_json.contains("aaaa"));++ // More than 4 MiB back: a full-size image.+ runtime.spaces().thumbnails().put(+ &space.id,+ Thumbnail {+ image: vec![0x5a; 5 << 20],+ format: "png".into(),+ width: 6720,+ height: 3780,+ captured_at: SystemTime::now(),+ },+ );+ let shot = client+ .spaces()+ .get_space_thumbnail(pb::GetSpaceThumbnailRequest {+ space: space.id.clone(),+ max_age: None,+ })+ .await+ .unwrap()+ .into_inner();+ assert_eq!(shot.image.len(), 5 << 20);+ handle.shutdown();+}