Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

perf(cua-driver): cap macOS screenshots at 1366 px by default, desktop captures included (#4917)

Francesco Bonaccic4d0f62557ed
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx
index ec564c9691..976a9de2eb 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/index.mdx
@@ -80,7 +80,7 @@ The desktop, screen size, cursor position and zoomed crops. [Reference](/cua-dri
| Tool | Description | Platforms |
| --- | --- | --- |
-| <span id="get_desktop_state"></span>[`get_desktop_state`](/cua-driver/reference/mcp-tools/screen#get_desktop_state) | Capture the full display in true screen pixels, full size unless `max_image_dimension` caps it. | macOS, Linux, Windows |
+| <span id="get_desktop_state"></span>[`get_desktop_state`](/cua-driver/reference/mcp-tools/screen#get_desktop_state) | Capture the full display. | macOS, Linux, Windows |
| <span id="get_screen_size"></span>[`get_screen_size`](/cua-driver/reference/mcp-tools/screen#get_screen_size) | Return the logical size of the main display in points plus its backing scale factor. | macOS, Linux, Windows |
| <span id="get_cursor_position"></span>[`get_cursor_position`](/cua-driver/reference/mcp-tools/screen#get_cursor_position) | Return the current mouse cursor position in screen points (origin top-left). | macOS, Linux, Windows |
| <span id="zoom"></span>[`zoom`](/cua-driver/reference/mcp-tools/screen#zoom) | Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot pixel coordinates, with 20% padding added on each side. | macOS, Linux, Windows |
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/screen.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/screen.mdx
index 7fdec9a476..5ad6fa7b6d 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/screen.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/screen.mdx
@@ -14,7 +14,7 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs';
| Tool | Description | Platforms |
| --- | --- | --- |
-| [`get_desktop_state`](#get_desktop_state) | Capture the full display in true screen pixels, full size unless `max_image_dimension` caps it. | macOS, Linux, Windows |
+| [`get_desktop_state`](#get_desktop_state) | Capture the full display. | macOS, Linux, Windows |
| [`get_screen_size`](#get_screen_size) | Return the logical size of the main display in points plus its backing scale factor. | macOS, Linux, Windows |
| [`get_cursor_position`](#get_cursor_position) | Return the current mouse cursor position in screen points (origin top-left). | macOS, Linux, Windows |
| [`zoom`](#zoom) | Capture a cropped JPEG of a window region (x1,y1)–(x2,y2) in screenshot pixel coordinates, with 20% padding added on each side. | macOS, Linux, Windows |
@@ -28,7 +28,7 @@ Effect: read-only. Platforms: macOS, Linux, Windows.
<Tabs items={['macOS', 'Linux', 'Windows']} groupId="platform" persist>
<Tab value="macOS">
-Capture the full display in true screen pixels, full size unless `max_image_dimension` caps it. Use its PNG as the coordinate source for actions whose target is &#123;kind:"desktop",display_id:"primary"&#125;. Returns the true screen size and backing scale factor. Vision-only: no AX tree walk.
+Capture the full display. The long edge is capped at `max_image_dimension` (the configured default unless given); pixels of the returned image map back to the screen through its capture. Use its PNG as the coordinate source for actions whose target is &#123;kind:"desktop",display_id:"primary"&#125;. Returns the true screen size and backing scale factor. Vision-only: no AX tree walk.
</Tab>
<Tab value="Linux">
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
index 1bc64766b6..48d408a44a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
@@ -37,8 +37,9 @@ static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "get_desktop_state".into(),
- description: "Capture the full display in true screen pixels, full size unless \
- `max_image_dimension` caps it. Use its PNG as the coordinate source for actions \
+ description: "Capture the full display. The long edge is capped at \
+ `max_image_dimension` (the configured default unless given); pixels of the \
+ returned image map back to the screen through its capture. Use its PNG as the coordinate source for actions \
whose target is {kind:\"desktop\",display_id:\"primary\"}. Returns the true \
screen size and backing scale factor. Vision-only: no AX tree walk."
.into(),
@@ -70,7 +71,18 @@ impl Tool for GetDesktopStateTool {
Ok(input) => input,
Err(result) => return result,
};
- let max_image_dimension = input.max_image_dimension.filter(|cap| *cap > 0);
+ // Call arg > session override > global config, as get_window_state.
+ let max_image_dimension = input.max_image_dimension.or_else(|| {
+ use cua_driver_core::tool_args::ArgsExt;
+ let session_id = capture_args.opt_str("_session_id");
+ let cfg = self.state.config.read().unwrap();
+ Some(
+ self.state
+ .session_config
+ .effective_max_image_dimension(session_id.as_deref(), &cfg),
+ )
+ });
+ let max_image_dimension = max_image_dimension.filter(|cap| *cap > 0);
let screenshot_out_file = input.screenshot_out_file.map(|s| {
// Expand ~ prefix (mirrors get_window_state).
if let Some(relative) = s.strip_prefix("~/") {
@@ -104,8 +116,8 @@ impl Tool for GetDesktopStateTool {
let (png, overlay_capture) =
crate::capture::screenshot_display_bytes_excluding_overlay()?;
let full = crate::capture::png_dimensions(&png)?;
- // Opt-in cap; later desktop-scope pixels from the capped
- // image are mapped back at dispatch and by its capture_id.
+ // Capped image: later desktop-scope pixels from it are
+ // mapped back at dispatch and by its capture_id.
let png = match max_image_dimension {
Some(cap) if full.0.max(full.1) > cap => {
crate::capture::resize_png_if_needed(&png, cap)?
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index f7bac73cad..5c2b457340 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -598,18 +598,26 @@ fn point_within_rect([rx, ry, rw, rh]: [f64; 4], x: f64, y: f64) -> bool {
rw > 0.0 && rh > 0.0 && x >= rx && x < rx + rw && y >= ry && y < ry + rh
}
+/// Default long-edge cap for screenshots; see [`DriverConfig::max_image_dimension`].
+pub const DEFAULT_MAX_IMAGE_DIMENSION: u32 = 1366;
+
/// Runtime-mutable driver configuration persisted across calls within a session.
pub struct DriverConfig {
- /// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom.
- /// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` —
+ /// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom:
/// the long edge is downscaled to this before encoding.
+ ///
+ /// Default 1366. High-resolution Claude models read images up to 2576 px
+ /// at ~w*h/750 tokens each, so a 1568x882 capture costs ~1.8k tokens and a
+ /// 1366x768 one ~1.4k; Anthropic's computer-use guidance names 1366x768 as
+ /// a lower-cost size with strong performance. Callers that need more
+ /// detail use `zoom` or a per-call/session `max_image_dimension`.
pub max_image_dimension: u32,
}
impl Default for DriverConfig {
fn default() -> Self {
Self {
- max_image_dimension: 1568,
+ max_image_dimension: DEFAULT_MAX_IMAGE_DIMENSION,
}
}
}
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index ba180f0bf5..3f9418b153 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -1500,7 +1500,7 @@
"read_only": true
},
{
- "description": "Capture the full display in true screen pixels, full size unless `max_image_dimension` caps it. Use its PNG as the coordinate source for actions whose target is {kind:\"desktop\",display_id:\"primary\"}. Returns the true screen size and backing scale factor. Vision-only: no AX tree walk.",
+ "description": "Capture the full display. The long edge is capped at `max_image_dimension` (the configured default unless given); pixels of the returned image map back to the screen through its capture. Use its PNG as the coordinate source for actions whose target is {kind:\"desktop\",display_id:\"primary\"}. Returns the true screen size and backing scale factor. Vision-only: no AX tree walk.",
"destructive": false,
"idempotent": false,
"input_schema": {