Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver-rs): screenshot defaults — jpeg @ 85 + max_image_dimension 1568 (#1663)

Francesco Bonaccidd6b521b16da
diff --git a/libs/cua-driver-rs/PARITY.md b/libs/cua-driver-rs/PARITY.md
index e1c465765b..faaad766f3 100644
--- a/libs/cua-driver-rs/PARITY.md
+++ b/libs/cua-driver-rs/PARITY.md
@@ -855,11 +855,15 @@ Windows's `click` takes `{button: enum}` instead. Rationale:
Swift verbatim: `"✅ Window screenshot — WxH png [window_id: ID]"`
(em-dash, checkmark, window-id suffix). Display fallback uses
`"✅ Display screenshot — WxH png"` (Rust-only, see intentional below).
-2. **Default JPEG quality** — was 85, Swift defaults 95. Now 95.
-3. **Description** — multi-paragraph port from Swift adapted to Windows
+2. **Description** — multi-paragraph port from Swift adapted to Windows
(BitBlt + PrintWindow transport; no permission gate needed).
-4. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab
+3. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab
every call). Now matches Swift.
+4. **`max_image_dimension` default** — was 0 (no cap), Swift uses 1568.
+ Now 1568 on all 3 Rust platforms (matches
+ `CuaDriverConfig.defaultMaxImageDimension`). The 0 default was
+ producing 10MB screenshots on the Windows VM; 1568 caps the long
+ edge before encoding.
### Intentional Rust-only
@@ -868,6 +872,16 @@ Windows's `click` takes `{button: enum}` instead. Rationale:
Windows-only convenience that Swift can't easily provide because
macOS Screen Recording requires per-window grants. Schema accepts
both shapes; description explains.
+- **Default `format`** — Swift defaults `png`; all 3 Rust platforms now
+ default `jpeg`. Rationale: agents typically want compact images for
+ vision-model context windows; PNG is lossless but multi-MB on screen
+ content. Schema still accepts both; callers wanting PNG pass
+ `{"format":"png"}`. Swift may follow; tracked as a follow-up parity
+ question.
+- **Default JPEG `quality`** — Swift defaults 95; Rust defaults 85
+ (already Linux's default and the macOS Claude-Code-compat tool's
+ default). 85 is the typical sweet spot for screen content. Diverges
+ from Swift only when both sides actually emit JPEG.
### Verified on Windows
diff --git a/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs
index 76dd577d36..e95ee93ff0 100644
--- a/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs
+++ b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs
@@ -982,7 +982,7 @@ fn test_browser_eval_no_cdp_error() {
#[test]
#[cfg(target_os = "macos")]
fn test_screenshot_no_window_id() {
- //! Call screenshot without window_id — should capture the full display and return a PNG.
+ //! Call screenshot without window_id — should capture the full display and return image content.
let binary = binary_path();
if !binary.exists() { return; }
diff --git a/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs
index 135e81c019..9a1f7864e6 100644
--- a/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs
@@ -17,7 +17,7 @@ pub struct DriverConfig {
}
impl Default for DriverConfig {
- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }
+ fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }
}
pub struct ResizeRegistry {
@@ -953,12 +953,13 @@ impl Tool for ScreenshotTool {
SS_DEF.get_or_init(|| ToolDef {
name: "screenshot".into(),
description: "Capture a screenshot via XGetImage or `import` (ImageMagick). \
- Without window_id captures the full primary display. Supports png and jpeg formats.".into(),
+ Without window_id captures the full primary display. Supports png and jpeg formats \
+ (default jpeg, quality 85).".into(),
input_schema: json!({
"type":"object","properties":{
"window_id":{"type":"integer"},
- "format":{"type":"string","enum":["png","jpeg"]},
- "quality":{"type":"integer","minimum":1,"maximum":95}
+ "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."},
+ "quality":{"type":"integer","minimum":1,"maximum":95,"description":"JPEG quality 1-95; ignored for png. Default: 85."}
},"additionalProperties":false
}),
read_only: true, destructive: false, idempotent: true, open_world: false,
@@ -967,7 +968,7 @@ impl Tool for ScreenshotTool {
async fn invoke(&self, args: Value) -> ToolResult {
let xid_opt = args.get("window_id").and_then(|v| v.as_u64());
- let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();
+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();
let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;
let is_jpeg = format == "jpeg";
let max_dim = self.state.config.read().unwrap().max_image_dimension;
diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs
index 92dd00edea..1a3aa45971 100644
--- a/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs
@@ -109,6 +109,8 @@ pub struct DriverConfig {
/// Default capture_mode for get_window_state when not specified per-call.
pub capture_mode: String,
/// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom.
+ /// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` —
+ /// the long edge is downscaled to this before encoding.
pub max_image_dimension: u32,
}
@@ -116,7 +118,7 @@ impl Default for DriverConfig {
fn default() -> Self {
Self {
capture_mode: "som".to_owned(),
- max_image_dimension: 0,
+ max_image_dimension: 1568,
}
}
}
diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs
index eef822ae52..e8d1ba8106 100644
--- a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs
+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs
@@ -16,7 +16,7 @@ fn def() -> &'static ToolDef {
name: "screenshot".into(),
description:
"Capture a screenshot. Returns base64-encoded image data in the requested format \
- (default png).\n\n\
+ (default jpeg, quality 85).\n\n\
Without `window_id`, captures the full main display. With `window_id`, captures \
just that window (pair with `list_windows` or `get_accessibility_tree` which return \
window IDs).\n\n\
@@ -32,13 +32,13 @@ fn def() -> &'static ToolDef {
"format": {
"type": "string",
"enum": ["png", "jpeg"],
- "description": "Image format. Default: png."
+ "description": "Image format. Default: jpeg."
},
"quality": {
"type": "integer",
"minimum": 1,
"maximum": 95,
- "description": "JPEG quality 1-95; ignored for png. Default: 95."
+ "description": "JPEG quality 1-95; ignored for png. Default: 85."
}
},
"additionalProperties": false
@@ -56,8 +56,8 @@ impl Tool for ScreenshotTool {
async fn invoke(&self, args: Value) -> ToolResult {
let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);
- let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();
- let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;
+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();
+ let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;
let use_jpeg = format == "jpeg";
let max_dim = self.state.config.read().unwrap().max_image_dimension;
diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
index 7d4e1ff0da..dd52507fa0 100644
--- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
@@ -43,7 +43,7 @@ pub struct DriverConfig {
}
impl Default for DriverConfig {
- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }
+ fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }
}
pub struct ResizeRegistry {
@@ -2141,7 +2141,7 @@ impl Tool for ScreenshotTool {
// ScreenCaptureKit).
description: "Capture a screenshot of a single window via BitBlt + PrintWindow \
(no focus change). Returns base64-encoded image data in the requested format \
- (default png).\n\n\
+ (default jpeg, quality 85).\n\n\
`window_id` is recommended (use `list_windows` to find it). When omitted, \
captures the full primary display — a Windows-only convenience for whole-\
screen snapshots without a per-window target.\n\n\
@@ -2149,9 +2149,9 @@ impl Tool for ScreenshotTool {
input_schema: json!({
"type":"object","properties":{
"window_id":{"type":"integer","description":"HWND of the window to capture. When omitted, captures the full primary display (Windows-only)."},
- "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: png."},
+ "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."},
"quality":{"type":"integer","minimum":1,"maximum":95,
- "description":"JPEG quality 1-95; ignored for png."}
+ "description":"JPEG quality 1-95; ignored for png. Default: 85."}
},"additionalProperties":false
}),
read_only: true, destructive: false, idempotent: false, open_world: false,
@@ -2160,8 +2160,8 @@ impl Tool for ScreenshotTool {
async fn invoke(&self, args: Value) -> ToolResult {
let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());
- let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();
- let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;
+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();
+ let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;
let is_jpeg = format == "jpeg";
let max_dim = self.state.config.read().unwrap().max_image_dimension;
diff --git a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py
index 910b5e0b2c..1bfdccc316 100644
--- a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py
+++ b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py
@@ -255,8 +255,10 @@ class TestCursorVisibility(unittest.TestCase):
# 7. Wait for cursor animation + render loop to settle.
time.sleep(0.5)
- # 8. Take a full-screen screenshot.
- scr = client.call_tool("screenshot", {})
+ # 8. Take a full-screen screenshot. Force PNG so the per-pixel decode
+ # below works — the screenshot tool defaults to JPEG, which would be
+ # lossy and skew the cursor-colour search.
+ scr = client.call_tool("screenshot", {"format": "png"})
b64 = scr.get("content", [{}])[0].get("data", "")
self.assertTrue(b64, "screenshot returned no data")
png_bytes = base64.b64decode(b64)