Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver-rs): screenshot defaults — jpeg @ 85 + max_image_dimension 1568 (#1663)
Francesco Bonaccidd6b521b16da
diff --git a/libs/cua-driver-rs/PARITY.md b/libs/cua-driver-rs/PARITY.mdindex e1c465765b..faaad766f3 100644--- a/libs/cua-driver-rs/PARITY.md+++ b/libs/cua-driver-rs/PARITY.md@@ -855,11 +855,15 @@ Windows's `click` takes `{button: enum}` instead. Rationale:Swift verbatim: `"✅ Window screenshot — WxH png [window_id: ID]"`(em-dash, checkmark, window-id suffix). Display fallback uses`"✅ Display screenshot — WxH png"` (Rust-only, see intentional below).-2. **Default JPEG quality** — was 85, Swift defaults 95. Now 95.-3. **Description** — multi-paragraph port from Swift adapted to Windows+2. **Description** — multi-paragraph port from Swift adapted to Windows(BitBlt + PrintWindow transport; no permission gate needed).-4. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab+3. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grabevery call). Now matches Swift.+4. **`max_image_dimension` default** — was 0 (no cap), Swift uses 1568.+ Now 1568 on all 3 Rust platforms (matches+ `CuaDriverConfig.defaultMaxImageDimension`). The 0 default was+ producing 10MB screenshots on the Windows VM; 1568 caps the long+ edge before encoding.### Intentional Rust-only@@ -868,6 +872,16 @@ Windows's `click` takes `{button: enum}` instead. Rationale:Windows-only convenience that Swift can't easily provide becausemacOS Screen Recording requires per-window grants. Schema acceptsboth shapes; description explains.+- **Default `format`** — Swift defaults `png`; all 3 Rust platforms now+ default `jpeg`. Rationale: agents typically want compact images for+ vision-model context windows; PNG is lossless but multi-MB on screen+ content. Schema still accepts both; callers wanting PNG pass+ `{"format":"png"}`. Swift may follow; tracked as a follow-up parity+ question.+- **Default JPEG `quality`** — Swift defaults 95; Rust defaults 85+ (already Linux's default and the macOS Claude-Code-compat tool's+ default). 85 is the typical sweet spot for screen content. Diverges+ from Swift only when both sides actually emit JPEG.### Verified on Windowsdiff --git a/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rsindex 76dd577d36..e95ee93ff0 100644--- a/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs+++ b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs@@ -982,7 +982,7 @@ fn test_browser_eval_no_cdp_error() {#[test]#[cfg(target_os = "macos")]fn test_screenshot_no_window_id() {- //! Call screenshot without window_id — should capture the full display and return a PNG.+ //! Call screenshot without window_id — should capture the full display and return image content.let binary = binary_path();if !binary.exists() { return; }diff --git a/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rsindex 135e81c019..9a1f7864e6 100644--- a/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs@@ -17,7 +17,7 @@ pub struct DriverConfig {}impl Default for DriverConfig {- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }+ fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }}pub struct ResizeRegistry {@@ -953,12 +953,13 @@ impl Tool for ScreenshotTool {SS_DEF.get_or_init(|| ToolDef {name: "screenshot".into(),description: "Capture a screenshot via XGetImage or `import` (ImageMagick). \- Without window_id captures the full primary display. Supports png and jpeg formats.".into(),+ Without window_id captures the full primary display. Supports png and jpeg formats \+ (default jpeg, quality 85).".into(),input_schema: json!({"type":"object","properties":{"window_id":{"type":"integer"},- "format":{"type":"string","enum":["png","jpeg"]},- "quality":{"type":"integer","minimum":1,"maximum":95}+ "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."},+ "quality":{"type":"integer","minimum":1,"maximum":95,"description":"JPEG quality 1-95; ignored for png. Default: 85."}},"additionalProperties":false}),read_only: true, destructive: false, idempotent: true, open_world: false,@@ -967,7 +968,7 @@ impl Tool for ScreenshotTool {async fn invoke(&self, args: Value) -> ToolResult {let xid_opt = args.get("window_id").and_then(|v| v.as_u64());- let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;let is_jpeg = format == "jpeg";let max_dim = self.state.config.read().unwrap().max_image_dimension;diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rsindex 92dd00edea..1a3aa45971 100644--- a/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs@@ -109,6 +109,8 @@ pub struct DriverConfig {/// Default capture_mode for get_window_state when not specified per-call.pub capture_mode: String,/// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom.+ /// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` —+ /// the long edge is downscaled to this before encoding.pub max_image_dimension: u32,}@@ -116,7 +118,7 @@ impl Default for DriverConfig {fn default() -> Self {Self {capture_mode: "som".to_owned(),- max_image_dimension: 0,+ max_image_dimension: 1568,}}}diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rsindex eef822ae52..e8d1ba8106 100644--- a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs+++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs@@ -16,7 +16,7 @@ fn def() -> &'static ToolDef {name: "screenshot".into(),description:"Capture a screenshot. Returns base64-encoded image data in the requested format \- (default png).\n\n\+ (default jpeg, quality 85).\n\n\Without `window_id`, captures the full main display. With `window_id`, captures \just that window (pair with `list_windows` or `get_accessibility_tree` which return \window IDs).\n\n\@@ -32,13 +32,13 @@ fn def() -> &'static ToolDef {"format": {"type": "string","enum": ["png", "jpeg"],- "description": "Image format. Default: png."+ "description": "Image format. Default: jpeg."},"quality": {"type": "integer","minimum": 1,"maximum": 95,- "description": "JPEG quality 1-95; ignored for png. Default: 95."+ "description": "JPEG quality 1-95; ignored for png. Default: 85."}},"additionalProperties": false@@ -56,8 +56,8 @@ impl Tool for ScreenshotTool {async fn invoke(&self, args: Value) -> ToolResult {let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);- let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();- let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();+ let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;let use_jpeg = format == "jpeg";let max_dim = self.state.config.read().unwrap().max_image_dimension;diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rsindex 7d4e1ff0da..dd52507fa0 100644--- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs@@ -43,7 +43,7 @@ pub struct DriverConfig {}impl Default for DriverConfig {- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }+ fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }}pub struct ResizeRegistry {@@ -2141,7 +2141,7 @@ impl Tool for ScreenshotTool {// ScreenCaptureKit).description: "Capture a screenshot of a single window via BitBlt + PrintWindow \(no focus change). Returns base64-encoded image data in the requested format \- (default png).\n\n\+ (default jpeg, quality 85).\n\n\`window_id` is recommended (use `list_windows` to find it). When omitted, \captures the full primary display — a Windows-only convenience for whole-\screen snapshots without a per-window target.\n\n\@@ -2149,9 +2149,9 @@ impl Tool for ScreenshotTool {input_schema: json!({"type":"object","properties":{"window_id":{"type":"integer","description":"HWND of the window to capture. When omitted, captures the full primary display (Windows-only)."},- "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: png."},+ "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."},"quality":{"type":"integer","minimum":1,"maximum":95,- "description":"JPEG quality 1-95; ignored for png."}+ "description":"JPEG quality 1-95; ignored for png. Default: 85."}},"additionalProperties":false}),read_only: true, destructive: false, idempotent: false, open_world: false,@@ -2160,8 +2160,8 @@ impl Tool for ScreenshotTool {async fn invoke(&self, args: Value) -> ToolResult {let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());- let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();- let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;+ let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();+ let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;let is_jpeg = format == "jpeg";let max_dim = self.state.config.read().unwrap().max_image_dimension;diff --git a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.pyindex 910b5e0b2c..1bfdccc316 100644--- a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py+++ b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py@@ -255,8 +255,10 @@ class TestCursorVisibility(unittest.TestCase):# 7. Wait for cursor animation + render loop to settle.time.sleep(0.5)- # 8. Take a full-screen screenshot.- scr = client.call_tool("screenshot", {})+ # 8. Take a full-screen screenshot. Force PNG so the per-pixel decode+ # below works — the screenshot tool defaults to JPEG, which would be+ # lossy and skew the cursor-colour search.+ scr = client.call_tool("screenshot", {"format": "png"})b64 = scr.get("content", [{}])[0].get("data", "")self.assertTrue(b64, "screenshot returned no data")png_bytes = base64.b64decode(b64)