Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-spacesd): stream 6K displays and carry tool results over 4 MiB (#4850)

dduponta88f100b428a
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/h264_limits.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/h264_limits.rs
new file mode 100644
index 0000000000..e43207f5c7
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/h264_limits.rs
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The largest frame an H.264 stream encodes: the VideoToolbox hardware
+//! encoder (and H.264 level 5.2) tops out at 4096 px on the long edge and
+//! 36,864 macroblocks (4096x2304). A larger source, such as a 6K display
+//! (6720x3780), is captured downscaled to fit instead of failing to encode.
+
+/// Longest edge, in pixels, of an H.264 frame.
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+pub(crate) const H264_MAX_DIMENSION: u32 = 4096;
+/// Most 16x16 macroblocks in one H.264 frame (level 5.2 MaxFS).
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+pub(crate) const H264_MAX_MACROBLOCKS: u64 = 36_864;
+
+/// The long-edge cap an H.264 stream of a `width`x`height` (native px)
+/// source captures at: `requested` (0 = native size) lowered, when needed,
+/// so the downscaled frame fits [`H264_MAX_DIMENSION`] and
+/// [`H264_MAX_MACROBLOCKS`]. Never 0.
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+pub(crate) fn h264_max_dimension(width: u32, height: u32, requested: u32) -> u32 {
+ let (width, height) = (width.max(1), height.max(1));
+ let longest = width.max(height);
+ let mut cap = match requested {
+ 0 => longest,
+ r => r.min(longest),
+ }
+ .min(H264_MAX_DIMENSION);
+ // The frame area shrinks with the square of the cap: start from the
+ // cap whose area fits, then step down (bounded) past rounding.
+ if macroblocks(width, height, cap) > H264_MAX_MACROBLOCKS {
+ let fit =
+ (H264_MAX_MACROBLOCKS as f64 * 256.0 / (f64::from(width) * f64::from(height))).sqrt();
+ // (The epsilon keeps an exact fit, 3072 for 5000x5000, from
+ // rounding down a pixel.)
+ cap = cap
+ .min((f64::from(longest) * fit + 1e-6).floor() as u32)
+ .max(16);
+ for _ in 0..64 {
+ if macroblocks(width, height, cap) <= H264_MAX_MACROBLOCKS || cap <= 16 {
+ break;
+ }
+ cap -= 1;
+ }
+ }
+ cap.max(1)
+}
+
+/// Macroblocks of `width`x`height` scaled so its long edge is at most `cap`.
+fn macroblocks(width: u32, height: u32, cap: u32) -> u64 {
+ let longest = width.max(height);
+ let (w, h) = if longest <= cap {
+ (width, height)
+ } else {
+ let ratio = f64::from(cap) / f64::from(longest);
+ (
+ ((f64::from(width) * ratio).round() as u32).max(1),
+ ((f64::from(height) * ratio).round() as u32).max(1),
+ )
+ };
+ u64::from(w.div_ceil(16)) * u64::from(h.div_ceil(16))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn fits(width: u32, height: u32, requested: u32) {
+ let cap = h264_max_dimension(width, height, requested);
+ assert!(cap <= H264_MAX_DIMENSION, "{width}x{height}: {cap}");
+ assert!(
+ macroblocks(width, height, cap) <= H264_MAX_MACROBLOCKS,
+ "{width}x{height}: {cap}"
+ );
+ }
+
+ /// A 6K main display (6720x3780) streamed at native size (the viewer's
+ /// `max_dimension` 0) is captured at 4096x2304, which H.264 encodes
+ /// (#4623); before, the 6720x3780 frame went to the encoder as is.
+ #[test]
+ fn a_6k_display_is_downscaled_to_the_h264_limit() {
+ assert_eq!(h264_max_dimension(6720, 3780, 0), 4096);
+ assert_eq!(macroblocks(6720, 3780, 4096), 36_864);
+ // Portrait 5K (2880x5120): 2304x4096.
+ assert_eq!(h264_max_dimension(2880, 5120, 0), 4096);
+ // A square source over the macroblock budget shrinks further.
+ assert_eq!(h264_max_dimension(5000, 5000, 0), 3072);
+ for (w, h) in [
+ (6720, 3780),
+ (7680, 4320),
+ (5120, 2880),
+ (5000, 5000),
+ (3840, 3840),
+ ] {
+ for requested in [0, 1280, 4096, 8192] {
+ fits(w, h, requested);
+ }
+ }
+ }
+
+ #[test]
+ fn a_frame_within_the_limit_keeps_the_requested_cap() {
+ assert_eq!(h264_max_dimension(2560, 1440, 0), 2560);
+ assert_eq!(h264_max_dimension(2560, 1440, 1920), 1920);
+ assert_eq!(h264_max_dimension(6720, 3780, 1600), 1600);
+ assert_eq!(h264_max_dimension(3840, 2160, 0), 3840);
+ }
+}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs
index 8606f9585d..e2b8cd2fd0 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs
@@ -43,6 +43,7 @@ mod driver_input;
#[cfg(target_os = "linux")]
mod encoded_capture;
pub mod grpc;
+mod h264_limits;
#[cfg(target_os = "linux")]
mod linux_capture;
#[cfg(target_os = "linux")]
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rs
index 446ec7989c..c29c3e98b3 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_capture.rs
@@ -203,11 +203,13 @@ fn start_inner(
let filter = &located.filter;
let native_scale = f64::from(filter.point_pixel_scale()).max(1.0);
+ let h264 = encoder.is_some();
let initial_geometry = capture_geometry(
located.width_points,
located.height_points,
native_scale,
config.max_dimension,
+ h264,
);
native_geometry(
initial_geometry.native_width_px,
@@ -324,6 +326,7 @@ fn start_inner(
let monitor = spawn_monitor(
source,
config.max_dimension,
+ h264,
frame_interval,
stream.clone(),
sink,
@@ -346,6 +349,7 @@ fn start_inner(
fn spawn_monitor(
source: CaptureSource,
max_dimension: u32,
+ h264: bool,
frame_interval: screencapturekit::cm::CMTime,
stream: Arc<SCStream>,
sink: Arc<dyn CaptureSink>,
@@ -408,6 +412,7 @@ fn spawn_monitor(
located.height_points,
f64::from(located.filter.point_pixel_scale()),
max_dimension,
+ h264,
);
let current = *expected_geometry
.lock()
@@ -464,16 +469,24 @@ fn frame_status_allows_content(status: Option<SCFrameStatus>) -> bool {
status.is_none_or(SCFrameStatus::has_content)
}
+/// The frame size a capture delivers. An `h264` stream is also kept within
+/// what the encoder accepts (a 6K display is downscaled, not refused).
fn capture_geometry(
width_points: f64,
height_points: f64,
native_scale: f64,
max_dimension: u32,
+ h264: bool,
) -> CaptureGeometry {
let width_points = width_points.max(1.0);
let height_points = height_points.max(1.0);
let native_width = (width_points * native_scale.max(1.0)).round() as u32;
let native_height = (height_points * native_scale.max(1.0)).round() as u32;
+ let max_dimension = if h264 {
+ crate::h264_limits::h264_max_dimension(native_width, native_height, max_dimension)
+ } else {
+ max_dimension
+ };
let (width_px, height_px) = fit_dimensions(native_width, native_height, max_dimension);
CaptureGeometry {
width_px,
@@ -567,11 +580,26 @@ mod tests {
#[test]
fn capture_geometry_keeps_native_action_dimensions_when_stream_is_scaled() {
- let geometry = capture_geometry(960.0, 480.0, 2.0, 1280);
+ let geometry = capture_geometry(960.0, 480.0, 2.0, 1280, true);
assert_eq!(geometry.width_px, 1280);
assert_eq!(geometry.height_px, 640);
assert_eq!(geometry.native_width_px, 1920);
assert_eq!(geometry.native_height_px, 960);
assert!((geometry.scale_factor - (4.0 / 3.0)).abs() < f64::EPSILON);
}
+
+ /// #4623: a 6K main display (3360x1890 pt @2x) streamed at native size
+ /// is encoded at 4096x2304, while clicks still map to native pixels; a
+ /// one-frame (non-H.264) grab keeps the native size.
+ #[test]
+ fn a_6k_display_streams_within_the_h264_limit() {
+ let geometry = capture_geometry(3360.0, 1890.0, 2.0, 0, true);
+ assert_eq!((geometry.width_px, geometry.height_px), (4096, 2304));
+ assert_eq!(
+ (geometry.native_width_px, geometry.native_height_px),
+ (6720, 3780)
+ );
+ let still = capture_geometry(3360.0, 1890.0, 2.0, 0, false);
+ assert_eq!((still.width_px, still.height_px), (6720, 3780));
+ }
}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rs
index 11d9a397e3..06d65a0a36 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/macos_h264.rs
@@ -374,7 +374,8 @@ fn encoder_loop(
Ok(encoder) => native = Some(encoder),
Err(error) => {
sink.on_event(CaptureEvent::Suspended(format!(
- "VideoToolbox encoder creation failed: {error}"
+ "VideoToolbox encoder creation failed for a {}x{} frame: {error}",
+ dimensions.0, dimensions.1
)));
continue;
}
diff --git a/libs/cua/crates/cua-daemon/src/client.rs b/libs/cua/crates/cua-daemon/src/client.rs
index 5aebc98968..1664464434 100644
--- a/libs/cua/crates/cua-daemon/src/client.rs
+++ b/libs/cua/crates/cua-daemon/src/client.rs
@@ -178,15 +178,24 @@ impl DaemonClient {
fn runtime(&self) -> RuntimeServiceClient<Chan> {
RuntimeServiceClient::new(self.channel.clone())
+ .max_decoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)
+ .max_encoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)
}
fn daemon(&self) -> DaemonServiceClient<Chan> {
DaemonServiceClient::new(self.channel.clone())
+ .max_decoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)
+ .max_encoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)
}
- /// The raw Spaces client.
+ /// The raw Spaces client. Messages up to
+ /// [`cua_spacesd_client::MAX_MESSAGE_BYTES`], as on the Space's own
+ /// connection: a full-size screenshot (`get_desktop_state` of a 6K
+ /// display) is larger than gRPC's 4 MiB default.
pub fn spaces(&self) -> SpaceServiceClient<Chan> {
SpaceServiceClient::new(self.channel.clone())
+ .max_decoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)
+ .max_encoding_message_size(cua_spacesd_client::MAX_MESSAGE_BYTES)
}
fn map(e: tonic::Status) -> Error {
diff --git a/libs/cua/crates/cua-daemon/src/server.rs b/libs/cua/crates/cua-daemon/src/server.rs
index d91f66f3f8..508a59823b 100644
--- a/libs/cua/crates/cua-daemon/src/server.rs
+++ b/libs/cua/crates/cua-daemon/src/server.rs
@@ -463,10 +463,29 @@ async fn accept_loop<L: Accept>(listener: L, app: BoxedApp, mut shutdown: watch:
fn app(shared: Arc<Shared>, require_token: bool) -> BoxedApp {
let svc = Svc(shared.clone());
- let grpc = tonic::service::Routes::new(SandboxServiceServer::new(svc.clone()))
- .add_service(SpaceServiceServer::new(svc.clone()))
- .add_service(RuntimeServiceServer::new(svc.clone()))
- .add_service(DaemonServiceServer::new(svc.clone()));
+ // The clients' limit, not gRPC's 4 MiB default: a tool call can carry
+ // a full-size screenshot or file either way.
+ let limit = cua_spacesd_client::MAX_MESSAGE_BYTES;
+ let grpc = tonic::service::Routes::new(
+ SandboxServiceServer::new(svc.clone())
+ .max_decoding_message_size(limit)
+ .max_encoding_message_size(limit),
+ )
+ .add_service(
+ SpaceServiceServer::new(svc.clone())
+ .max_decoding_message_size(limit)
+ .max_encoding_message_size(limit),
+ )
+ .add_service(
+ RuntimeServiceServer::new(svc.clone())
+ .max_decoding_message_size(limit)
+ .max_encoding_message_size(limit),
+ )
+ .add_service(
+ DaemonServiceServer::new(svc.clone())
+ .max_decoding_message_size(limit)
+ .max_encoding_message_size(limit),
+ );
#[cfg(feature = "spaces")]
let grpc = grpc.add_service(
cua_proto::env::v1::host_spaces_service_server::HostSpacesServiceServer::new(
diff --git a/libs/cua/crates/cua-daemon/tests/large_messages.rs b/libs/cua/crates/cua-daemon/tests/large_messages.rs
new file mode 100644
index 0000000000..aa01950665
--- /dev/null
+++ b/libs/cua/crates/cua-daemon/tests/large_messages.rs
@@ -0,0 +1,87 @@
+//! A Spaces tool call through the daemon carries more than gRPC's 4 MiB
+//! default each way (#4623: `get_desktop_state` of a 6K display returned a
+//! 9.5 MB screenshot and failed with "decoded message length too large").
+//! Temp dirs and a loopback mock spacesd only.
+
+use cua_daemon::client::{DaemonAddress, DaemonClient};
+use cua_daemon::server::{ServerConfig, start};
+use cua_daemon::{Runtime as DaemonRuntime, RuntimeConfig};
+use cua_proto::daemon::v1 as pb;
+use cua_spaces::thumbnails::Thumbnail;
+use cua_spacesd_client::testing::{MockAuth, MockServer};
+use std::time::{Duration, SystemTime};
+
+#[tokio::test]
+async fn a_tool_call_larger_than_4_mib_round_trips_through_the_daemon() {
+ let dir = tempfile::tempdir().unwrap();
+ let runtime = DaemonRuntime::new(RuntimeConfig {
+ state_dir: Some(dir.path().join("sandboxes")),
+ spaces_home: Some(dir.path().to_path_buf()),
+ ..Default::default()
+ })
+ .unwrap();
+ let handle = start(
+ runtime.clone(),
+ ServerConfig {
+ socket_path: None,
+ loopback: Some(([127, 0, 0, 1], 0).into()),
+ token: "t0k3n".into(),
+ discovery_path: None,
+ bridge_ticket_ttl: Duration::from_secs(60),
+ },
+ )
+ .await
+ .unwrap();
+ let client = DaemonClient::new(DaemonAddress::Url {
+ url: handle.loopback_url.clone().unwrap(),
+ token: handle.token.clone(),
+ })
+ .unwrap();
+ let env = MockServer::start(MockAuth::default()).await;
+ let space = runtime
+ .spaces()
+ .add(&env.url(), None, Some("big".into()))
+ .await
+ .unwrap();
+
+ // More than 4 MiB to the daemon: the tool call's arguments.
+ let word = "a".repeat(5 << 20);
+ let r = client
+ .spaces()
+ .call_space_tool(pb::CallSpaceToolRequest {
+ name: "space_bash".into(),
+ arguments_json: serde_json::json!({
+ "space": space.id,
+ "command": format!("echo {word}"),
+ })
+ .to_string(),
+ })
+ .await
+ .unwrap()
+ .into_inner();
+ assert!(!r.is_error, "{}", r.content_json);
+ assert!(r.content_json.contains("aaaa"));
+
+ // More than 4 MiB back: a full-size image.
+ runtime.spaces().thumbnails().put(
+ &space.id,
+ Thumbnail {
+ image: vec![0x5a; 5 << 20],
+ format: "png".into(),
+ width: 6720,
+ height: 3780,
+ captured_at: SystemTime::now(),
+ },
+ );
+ let shot = client
+ .spaces()
+ .get_space_thumbnail(pb::GetSpaceThumbnailRequest {
+ space: space.id.clone(),
+ max_age: None,
+ })
+ .await
+ .unwrap()
+ .into_inner();
+ assert_eq!(shot.image.len(), 5 << 20);
+ handle.shutdown();
+}