Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): desktop-scope Phase 1 — capture_scope config, get_desktop_state, Windows screen-absolute actions (#1968) (#2019)
Francesco Bonaccifc2718541497
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex ce5e36c50c..77c53d14d7 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -201,6 +201,7 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {"screen.capture.region",],"get_screen_size" => &["screen.dimensions"],+ "get_desktop_state" => &["screen.capture", "screen.dimensions"],"get_cursor_position" => &["screen.cursor.position"],// ── accessibility / window state ─────────────────────────────@@ -520,7 +521,8 @@ mod capability_tests {// keyboard"type_text", "type_text_chars", "press_key", "hotkey", "set_value",// screen- "zoom", "get_screen_size", "get_cursor_position",+ "zoom", "get_screen_size", "get_desktop_state",+ "get_cursor_position",// accessibility"get_accessibility_tree", "get_window_state",// app / windowdiff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex 66cfb6c0a3..da3e680867 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -19,11 +19,12 @@ use cursor_overlay::CursorRegistry;#[derive(Clone)]pub struct DriverConfig {pub capture_mode: String,+ pub capture_scope: String,pub max_image_dimension: u32,}impl Default for DriverConfig {- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }+ fn default() -> Self { Self { capture_mode: "som".into(), capture_scope: "window".into(), max_image_dimension: 1568 } }}/// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to@@ -36,6 +37,9 @@ pub fn load_driver_config() -> DriverConfig {if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) {cfg.capture_mode = v;}+ if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) {+ cfg.capture_scope = v;+ }if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) {if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; }}@@ -2872,33 +2876,11 @@ impl Tool for GetScreenSizeTool {}async fn invoke(&self, _args: Value) -> ToolResult {let result = tokio::task::spawn_blocking(|| {- use x11rb::connection::Connection;- use x11rb::rust_connection::RustConnection;- let (conn, screen_num) = RustConnection::connect(None)- .map_err(|e| anyhow::anyhow!("{e}{}", crate::no_display_hint()))?;- let setup = conn.setup();- let screen = &setup.roots[screen_num];- let w = screen.width_in_pixels as u32;- let h = screen.height_in_pixels as u32;- // WSLg / headless XWayland quirk: the X server connects but the- // root screen advertises a 0-px geometry until a real output is- // attached. Returning {width:0,height:0} here would propagate a- // success with zero dimensions to the client, which then either- // divides by zero when scaling or feeds the value into `int(...)`- // after the missing key collapses to None. Fail loudly with an- // actionable, typed error instead (never emit a 0/null where the- // client expects a usable int). See issue #2005.- if w == 0 || h == 0 {- anyhow::bail!(- "X11 connected but reports a 0x0 root screen — no usable \- display geometry.{}",- crate::no_display_hint()- );- }// X11 reports pixel dimensions; scale factor on X11 is not// well-defined per-monitor, so report 1.0 (matches DPI-unaware// assumption). Wayland/HiDPI X11 callers should query// `xrandr --query` for true scale.+ let (w, h) = x11_screen_size()?;Ok::<(u32, u32, f64), anyhow::Error>((w, h, 1.0))}).await;match result {@@ -2911,6 +2893,111 @@ impl Tool for GetScreenSizeTool {}}+/// Read the true X11 root-window size in pixels: (width, height).+/// Shared by `get_screen_size` and `get_desktop_state`.+fn x11_screen_size() -> anyhow::Result<(u32, u32)> {+ use x11rb::connection::Connection;+ use x11rb::rust_connection::RustConnection;+ let (conn, screen_num) = RustConnection::connect(None)+ .map_err(|e| anyhow::anyhow!("{e}{}", crate::no_display_hint()))?;+ let setup = conn.setup();+ let screen = &setup.roots[screen_num];+ let w = screen.width_in_pixels as u32;+ let h = screen.height_in_pixels as u32;+ // WSLg / headless XWayland quirk: the X server connects but the+ // root screen advertises a 0-px geometry until a real output is+ // attached. Returning {width:0,height:0} here would propagate a+ // success with zero dimensions to the client, which then either+ // divides by zero when scaling or feeds the value into `int(...)`+ // after the missing key collapses to None. Fail loudly with an+ // actionable, typed error instead (never emit a 0/null where the+ // client expects a usable int). See issue #2005.+ if w == 0 || h == 0 {+ anyhow::bail!(+ "X11 connected but reports a 0x0 root screen — no usable \+ display geometry.{}",+ crate::no_display_hint()+ );+ }+ Ok((w, h))+}++// ── get_desktop_state ─────────────────────────────────────────────────────────++pub struct GetDesktopStateTool;+static GDS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetDesktopStateTool {+ fn def(&self) -> &ToolDef {+ GDS_DEF.get_or_init(|| ToolDef {+ name: "get_desktop_state".into(),+ description: "Full-display vision screenshot in true screen pixels (no downscale), \+ for capture_scope=\"desktop\" GUI loops. Captures the entire display (root \+ window) as native-size PNG so screen-absolute pixel coordinates land exactly. \+ No AT-SPI walk.".into(),+ input_schema: json!({"type":"object","properties":{+ "session":{"type":"string","description":"Optional session id."},+ "screenshot_out_file":{"type":"string","description":"Write PNG here instead of base64."}+ },"additionalProperties":false}),+ read_only: true, destructive: false, idempotent: false, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ let out_file = args.opt_str("screenshot_out_file");++ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {+ // Vision-only: capture the FULL DISPLAY at native size. No downscale+ // so screen-absolute pixels land exactly.+ let png = crate::capture::screenshot_display_bytes()?;+ let (shot_w, shot_h) = crate::capture::png_dimensions_pub(&png)?;+ // True screen size from the X11 root window.+ let (screen_w, screen_h) = x11_screen_size()?;+ // Optional: write PNG to disk instead of returning base64.+ let written = if let Some(path) = out_file.as_deref() {+ std::fs::write(path, &png)?;+ Some(path.to_string())+ } else {+ None+ };+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};+ let b64 = if written.is_some() { None } else { Some(B64.encode(&png)) };+ Ok((b64, shot_w, shot_h, screen_w, screen_h, written))+ }).await;++ match result {+ Ok(Ok((b64_opt, shot_w, shot_h, screen_w, screen_h, written))) => {+ let mut content = Vec::new();+ let mut structured = json!({+ "platform": "linux",+ "screenshot_width": shot_w,+ "screenshot_height": shot_h,+ "screen_width": screen_w,+ "screen_height": screen_h,+ "screenshot_mime_type": "image/png",+ });+ if let Some(b64) = b64_opt {+ content.push(cua_driver_core::protocol::Content::image_png(b64));+ }+ if let Some(path) = written {+ structured["screenshot_file_path"] = json!(path);+ content.push(cua_driver_core::protocol::Content::text(format!(+ "✅ Desktop screenshot {shot_w}x{shot_h} written to {path} (screen {screen_w}x{screen_h})"+ )));+ } else {+ content.push(cua_driver_core::protocol::Content::text(format!(+ "✅ Desktop screenshot {shot_w}x{shot_h} (screen {screen_w}x{screen_h})"+ )));+ }+ ToolResult { content, is_error: None, structured_content: Some(structured) }+ }+ Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+ }+}+// ── get_cursor_position ───────────────────────────────────────────────────────pub struct GetCursorPositionTool;@@ -3373,6 +3460,7 @@ impl Tool for GetConfigTool {"version": env!("CARGO_PKG_VERSION"),"platform": "linux","capture_mode": cfg.capture_mode,+ "capture_scope": cfg.capture_scope,"max_image_dimension": cfg.max_image_dimension,"experimental_pip": pip_enabled,"experimental_pip_geometry": pip_geometry@@ -3405,6 +3493,7 @@ impl Tool for SetConfigTool {"key":{"type":"string","description":"Name of a single config field to write ({key, value} shape). Pair with `value`."},"value":{"description":"New value for `key`. JSON type depends on the key."},"capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Legacy per-field shape. Default capture mode for get_window_state."},+ "capture_scope":{"type":"string","enum":["window","desktop"],"description":"Capture scope: single window or whole desktop. Default window."},"max_image_dimension":{"type":"integer","description":"Legacy per-field shape. Max dimension for screenshot resizing (0 = no limit)."},"experimental_pip":{"type":"boolean","description":"Enable the experimental PiP preview window (applies next restart; Linux backend stubbed)."},"experimental_pip_geometry":{"type":"string","description":"PiP window size + optional position in `WxH` or `WxH+X+Y` form."}@@ -3435,6 +3524,17 @@ impl Tool for SetConfigTool {}None => return ToolResult::error(format!("`capture_mode` must be a string, got {val}.")),},+ "capture_scope" => match val.as_str() {+ Some(s @ ("window" | "desktop")) => {+ cfg.capture_scope = s.to_owned();+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(s.to_owned())) {+ tracing::warn!("set_config: failed to persist capture_scope: {e}");+ }+ parts.push(format!("capture_scope={s}"));+ }+ Some(other) => return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{other}\".")),+ None => return ToolResult::error(format!("`capture_scope` must be a string, got {val}.")),+ },"max_image_dimension" => match val.as_u64() {Some(n) => {cfg.max_image_dimension = n as u32;@@ -3469,7 +3569,7 @@ impl Tool for SetConfigTool {None => return ToolResult::error(format!("`experimental_pip_geometry` must be a string, got {val}.")),},other => return ToolResult::error(format!(- "Unknown config key `{other}`. Known: capture_mode, max_image_dimension, experimental_pip, experimental_pip_geometry."+ "Unknown config key `{other}`. Known: capture_mode, capture_scope, max_image_dimension, experimental_pip, experimental_pip_geometry.")),}}@@ -3481,6 +3581,16 @@ impl Tool for SetConfigTool {parts.push(format!("capture_mode={mode}"));cfg.capture_mode = mode;}+ if let Some(scope) = args.opt_str("capture_scope") {+ if scope != "window" && scope != "desktop" {+ return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."));+ }+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.clone())) {+ tracing::warn!("set_config: failed to persist capture_scope: {e}");+ }+ parts.push(format!("capture_scope={scope}"));+ cfg.capture_scope = scope;+ }if let Some(dim) = args.opt_u64("max_image_dimension") {cfg.max_image_dimension = dim as u32;if let Err(e) = pip_preview::write_config_key("max_image_dimension", Value::from(dim)) {@@ -3514,6 +3624,7 @@ impl Tool for SetConfigTool {ToolResult::text(msg).with_structured(json!({"capture_mode": cfg.capture_mode,+ "capture_scope": cfg.capture_scope,"max_image_dimension": cfg.max_image_dimension,"experimental_pip": pip_enabled,"experimental_pip_geometry": pip_geometry@@ -3867,6 +3978,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry {// screenshot path is `get_window_state` with `capture_mode:"vision"`.let _ = compat;r.register(Box::new(GetScreenSizeTool));+ r.register(Box::new(GetDesktopStateTool));r.register(Box::new(GetCursorPositionTool));r.register(Box::new(MoveCursorTool { state: state.clone() }));r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));@@ -3930,3 +4042,13 @@ mod click_button_schema_tests {assert!(lc.contains("wayland"), "description should call out wayland fallback");}}++#[cfg(test)]+mod driver_config_tests {+ use super::DriverConfig;++ #[test]+ fn capture_scope_defaults_to_window() {+ assert_eq!(DriverConfig::default().capture_scope, "window");+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rsindex f238a36d9f..0decd59c73 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs@@ -37,9 +37,11 @@ impl Tool for GetConfigTool {// sees its own override layered over the global; the anonymous session// (absent `_session_id`) sees the raw global — today's behavior.let session_id = args.opt_str("_session_id");- let (capture_mode, max_image_dimension) = {+ let (capture_mode, max_image_dimension, capture_scope) = {let cfg = self.state.config.read().unwrap();- self.state.session_config.effective(session_id.as_deref(), &cfg)+ let (mode, dim) = self.state.session_config.effective(session_id.as_deref(), &cfg);+ let scope = self.state.session_config.effective_scope(session_id.as_deref(), &cfg);+ (mode, dim, scope)};// Report the CALLING session's own cursor enabled-state, not a// nondeterministic HashMap.first(). Resolve the same key the click /@@ -61,6 +63,7 @@ impl Tool for GetConfigTool {"version": env!("CARGO_PKG_VERSION"),"platform": "macos","capture_mode": capture_mode,+ "capture_scope": capture_scope,"max_image_dimension": max_image_dimension,"agent_cursor": {"enabled": cursor_enabled,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rsnew file mode 100644index 0000000000..9413983471--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs@@ -0,0 +1,152 @@+//! `get_desktop_state` — full-display vision screenshot (macOS).+//!+//! Vision-only desktop capture: grabs the ENTIRE main display at native+//! pixel size (no downscale) so screen-absolute pixel picks land exactly,+//! then reports the true screen size + backing scale. No AX walk, no+//! pid/window_id — this is the capture surface for `capture_scope="desktop"`+//! GUI loops where the agent drives `click(x,y)` / `scroll(x,y)` against+//! screen-absolute coordinates.+//!+//! Mirrors `get_window_state.rs`'s vision ToolResult shape: an `image_png`+//! content part (or a written-out file path), a text summary line, and a+//! `structuredContent` object.++use async_trait::async_trait;+use cua_driver_core::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}};+use serde_json::Value;++use super::get_screen_size::main_screen_size;++pub struct GetDesktopStateTool;++static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++fn def() -> &'static ToolDef {+ DEF.get_or_init(|| ToolDef {+ name: "get_desktop_state".into(),+ description: "Capture a full-display vision screenshot in true screen pixels \+ (no downscale), for capture_scope=\"desktop\" GUI loops where the agent then \+ drives click(x,y)/scroll(x,y) with no pid/window_id. Returns the PNG at native \+ display resolution plus the true screen size and backing scale factor so \+ screen-absolute pixel picks land exactly. Vision-only: no AX tree walk."+ .into(),+ input_schema: serde_json::json!({+ "type": "object",+ "properties": {+ "session": { "type": "string", "description": "Optional session id." },+ "screenshot_out_file": { "type": "string", "description": "Write PNG here instead of base64." }+ },+ "additionalProperties": false+ }),+ read_only: true,+ destructive: false,+ idempotent: false,+ open_world: false,+ })+}++#[async_trait]+impl Tool for GetDesktopStateTool {+ fn def(&self) -> &ToolDef { def() }++ async fn invoke(&self, args: Value) -> ToolResult {+ use cua_driver_core::tool_args::ArgsExt;++ let screenshot_out_file = args.opt_str("screenshot_out_file").map(|s| {+ // Expand ~ prefix (mirrors get_window_state).+ if s.starts_with("~/") {+ let home = std::env::var("HOME").unwrap_or_default();+ format!("{home}/{}", &s[2..])+ } else {+ s+ }+ });++ // True screen geometry (points + backing scale). Safe off the main thread.+ let (screen_width, screen_height, scale_factor) = match main_screen_size() {+ Some(t) => t,+ None => return ToolResult::error("No main display detected."),+ };++ // Capture the FULL display at native size — no resize. Run the+ // blocking screencapture subprocess off the async runtime.+ let out_file = screenshot_out_file.clone();+ let res = tokio::task::spawn_blocking(+ move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32)> {+ use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+ let png = crate::capture::screenshot_display_bytes()?;+ let (w, h) = crate::capture::png_dimensions(&png)?;+ if let Some(ref path) = out_file {+ std::fs::write(path, &png)?;+ Ok((None, Some(path.clone()), w, h))+ } else {+ Ok((Some(BASE64.encode(&png)), None, w, h))+ }+ },+ )+ .await;++ let (b64_opt, file_path, screenshot_width, screenshot_height) = match res {+ Ok(Ok(v)) => v,+ Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),+ Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),+ };++ let mut content: Vec<Content> = Vec::new();+ if let Some(b64) = b64_opt {+ content.push(Content::image_png(b64));+ }+ let summary = format!(+ "desktop screenshot {screenshot_width}x{screenshot_height} px \+ (screen {screen_width}x{screen_height} pts @ {scale_factor}x)"+ );+ content.push(Content::text(summary));++ let mut structured = serde_json::json!({+ "platform": "macos",+ "screenshot_width": screenshot_width,+ "screenshot_height": screenshot_height,+ "screen_width": screen_width,+ "screen_height": screen_height,+ "scale_factor": scale_factor,+ "screenshot_mime_type": "image/png",+ });+ if let Some(ref fp) = file_path {+ structured["screenshot_file_path"] = serde_json::json!(fp);+ }++ ToolResult { content, is_error: None, structured_content: Some(structured) }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn schema_has_no_pid_or_window_id_and_is_read_only() {+ let d = def();+ assert!(d.read_only, "get_desktop_state must be read_only");+ assert!(!d.destructive);+ assert!(!d.idempotent);+ assert!(!d.open_world);++ let props = d.input_schema["properties"].as_object().unwrap();+ assert!(!props.contains_key("pid"), "must not accept pid");+ assert!(!props.contains_key("window_id"), "must not accept window_id");+ assert!(!props.contains_key("capture_mode"), "must not accept capture_mode");+ assert!(props.contains_key("session"));+ assert!(props.contains_key("screenshot_out_file"));+ assert_eq!(d.input_schema["additionalProperties"], serde_json::json!(false));+ }++ #[test]+ fn description_mentions_full_and_screen_or_display() {+ let desc = def().description.to_lowercase();+ assert!(desc.contains("full"), "description must mention 'full'");+ assert!(+ desc.contains("screen") || desc.contains("display"),+ "description must mention 'screen' or 'display'"+ );+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rsindex 5fa196c120..dc39dee39a 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs@@ -45,7 +45,7 @@ impl Tool for GetScreenSizeTool {/// The previous NSScreen-based implementation required `MainThreadMarker::new()`/// which always returns `None` on async tokio threads, causing the tool to/// return an error even when a display is attached.-fn main_screen_size() -> Option<(i64, i64, f64)> {+pub(crate) fn main_screen_size() -> Option<(i64, i64, f64)> {use core_graphics::display::{CGMainDisplayID, CGDisplayBounds};// SAFETY: CGMainDisplayID / CGDisplayBounds are thread-safe CG APIs.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rsindex 7d4457669e..6abd2db36b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs@@ -21,6 +21,7 @@ mod scroll;// etc.) live elsewhere under CuaDriverCore::Capture and are reached// through GetWindowStateTool.pub(crate) mod get_screen_size;+mod get_desktop_state;mod get_cursor_position;mod move_cursor;mod cursor_tools;@@ -114,6 +115,9 @@ impl ResizeRegistry {pub struct DriverConfig {/// Default capture_mode for get_window_state when not specified per-call.pub capture_mode: String,+ /// Capture scope for get_window_state: "window" (default) crops to the+ /// target window; "desktop" captures the full display.+ pub capture_scope: String,/// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom./// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` —/// the long edge is downscaled to this before encoding.@@ -124,6 +128,7 @@ impl Default for DriverConfig {fn default() -> Self {Self {capture_mode: "som".to_owned(),+ capture_scope: "window".to_string(),max_image_dimension: 1568,}}@@ -153,6 +158,9 @@ pub fn load_driver_config() -> DriverConfig {if let Some(v) = json.get("capture_mode").and_then(|v| v.as_str()) {cfg.capture_mode = v.to_owned();}+ if let Some(v) = json.get("capture_scope").and_then(|v| v.as_str()) {+ cfg.capture_scope = v.to_owned();+ }if let Some(v) = json.get("max_image_dimension").and_then(|v| v.as_u64()) {if let Ok(v32) = u32::try_from(v) {cfg.max_image_dimension = v32;@@ -194,6 +202,7 @@ pub fn write_driver_config_key(key: &str, value: &serde_json::Value) -> Result<(#[derive(Clone, Default)]pub struct ConfigOverrides {pub capture_mode: Option<String>,+ pub capture_scope: Option<String>,pub max_image_dimension: Option<u32>,}@@ -223,6 +232,9 @@ impl SessionConfigRegistry {if delta.capture_mode.is_some() {entry.capture_mode = delta.capture_mode;}+ if delta.capture_scope.is_some() {+ entry.capture_scope = delta.capture_scope;+ }if delta.max_image_dimension.is_some() {entry.max_image_dimension = delta.max_image_dimension;}@@ -242,6 +254,18 @@ impl SessionConfigRegistry {}}+ /// Resolve the effective `capture_scope` for `session`, layering its+ /// override over the global `DriverConfig`. Kept separate from `effective()`+ /// so existing `(String, u32)` call sites stay unchanged. `session = None`+ /// (anonymous) returns the global scope verbatim.+ pub fn effective_scope(&self, session_id: Option<&str>, global: &DriverConfig) -> String {+ let ov = session_id.and_then(|s| self.inner.lock().unwrap().get(s).cloned());+ match ov {+ Some(ov) => ov.capture_scope.unwrap_or_else(|| global.capture_scope.clone()),+ None => global.capture_scope.clone(),+ }+ }+/// Drop `session`'s overrides. No-op for an unknown id (so `session_end`/// for an anonymous / never-set session is harmless).pub fn clear(&self, session: &str) {@@ -330,6 +354,7 @@ pub fn register_all(registry: &mut ToolRegistry, compat: bool) {// screenshot path is `get_window_state` with `capture_mode:"vision"`.let _ = compat;registry.register(Box::new(get_screen_size::GetScreenSizeTool));+ registry.register(Box::new(get_desktop_state::GetDesktopStateTool));registry.register(Box::new(get_cursor_position::GetCursorPositionTool));registry.register(Box::new(move_cursor::MoveCursorTool::new(state.clone())));registry.register(Box::new(cursor_tools::SetAgentCursorEnabledTool::new(state.clone())));@@ -374,7 +399,35 @@ mod session_config_guard_tests {use cua_driver_core::session::fire_session_end;fn overrides(mode: &str) -> ConfigOverrides {- ConfigOverrides { capture_mode: Some(mode.to_owned()), max_image_dimension: None }+ ConfigOverrides { capture_mode: Some(mode.to_owned()), capture_scope: None, max_image_dimension: None }+ }++ #[test]+ fn default_capture_scope_is_window() {+ assert_eq!(DriverConfig::default().capture_scope, "window");+ }++ #[test]+ fn effective_scope_uses_session_override_then_global() {+ let reg = SessionConfigRegistry::new();+ let global = DriverConfig::default();+ let sid = "wb-scope-live-C1D2E3";+ assert!(!cua_driver_core::session::is_session_ended(sid));++ // No override yet → falls back to global ("window").+ assert_eq!(reg.effective_scope(Some(sid), &global), "window");+ // Anonymous → global.+ assert_eq!(reg.effective_scope(None, &global), "window");++ // Live session override applies.+ reg.set(sid, ConfigOverrides {+ capture_mode: None,+ capture_scope: Some("desktop".to_owned()),+ max_image_dimension: None,+ });+ assert_eq!(reg.effective_scope(Some(sid), &global), "desktop");+ // Other sessions / anonymous still see global.+ assert_eq!(reg.effective_scope(None, &global), "window");}#[test]diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rsindex 345c881dd0..d5c82420c5 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs@@ -31,6 +31,12 @@ fn def() -> &'static ToolDef {"enum": ["som", "vision", "ax"],"description": "Default capture mode for get_window_state."},+ "capture_scope": {+ "type": "string",+ "enum": ["window", "desktop"],+ "description": "Capture scope for get_window_state: 'window' crops to the \+ target window; 'desktop' captures the full display."+ },"max_image_dimension": {"type": "integer","description": "Max dimension for screenshot resizing (0 = no limit)."@@ -80,10 +86,22 @@ impl Tool for SetConfigTool {};let capture_mode = args.opt_str("capture_mode");+ // Validate capture_scope up front so both branches share the check and+ // we never half-apply an invalid value.+ let capture_scope = args.opt_str("capture_scope");+ if let Some(scope) = capture_scope.as_deref() {+ if scope != "window" && scope != "desktop" {+ return ToolResult::error(format!(+ "capture_scope `{scope}` is invalid; expected `window` or `desktop`"+ ));+ }+ }+let (effective_mode, effective_dim) = if let Some(sid) = session_id.as_deref() {// Session-scoped override: in-memory only, no global write, no disk.self.state.session_config.set(sid, ConfigOverrides {capture_mode: capture_mode.clone(),+ capture_scope: capture_scope.clone(),max_image_dimension: max_dim,});self.state.session_config.effective(Some(sid), &self.state.config.read().unwrap())@@ -97,6 +115,12 @@ impl Tool for SetConfigTool {tracing::warn!("set_config: failed to persist capture_mode: {e}");}}+ if let Some(scope) = capture_scope.clone() {+ cfg.capture_scope = scope.clone();+ if let Err(e) = write_driver_config_key("capture_scope", &Value::String(scope)) {+ tracing::warn!("set_config: failed to persist capture_scope: {e}");+ }+ }if let Some(dim32) = max_dim {cfg.max_image_dimension = dim32;if let Err(e) = write_driver_config_key("max_image_dimension", &Value::Number(u64::from(dim32).into())) {@@ -105,6 +129,13 @@ impl Tool for SetConfigTool {}(cfg.capture_mode.clone(), cfg.max_image_dimension)};+ // Resolve the effective capture_scope for the echo via the same+ // session/global precedence (kept separate from `effective()` so its+ // `(String, u32)` signature stays unchanged for other call sites).+ let effective_scope = self+ .state+ .session_config+ .effective_scope(session_id.as_deref(), &self.state.config.read().unwrap());// PiP keys persist to the same config.json but take effect only on// next daemon restart — the backend is initialised once at startup.let mut pip_note = String::new();@@ -134,8 +165,8 @@ impl Tool for SetConfigTool {""};ToolResult::text(format!(- "Config updated: capture_mode={}, max_image_dimension={}{}{}",- effective_mode, effective_dim, scope_note, pip_note+ "Config updated: capture_mode={}, capture_scope={}, max_image_dimension={}{}{}",+ effective_mode, effective_scope, effective_dim, scope_note, pip_note))}}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rsindex d1bb662f88..3209681ca4 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs@@ -15,7 +15,7 @@ pub mod dispatch;pub mod inject;pub use inject::{inject_click_screen, inject_key_cloaked, inject_text_cloaked, NoActivateGuard};-pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized};+pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized, send_wheel_synthesized};pub use keyboard::{is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,send_key_synthesized,diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rsindex 7f8100c7ea..2d1e10c5fb 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs@@ -10,9 +10,10 @@ use std::time::Duration;use windows::Win32::Foundation::{HWND, LPARAM, POINT, WPARAM};use windows::Win32::Graphics::Gdi::{ClientToScreen, ScreenToClient};use windows::Win32::UI::Input::KeyboardAndMouse::{- INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,- MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN,- MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEINPUT, SendInput,+ INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL, MOUSEEVENTF_LEFTDOWN,+ MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE,+ MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEEVENTF_WHEEL,+ MOUSEINPUT, SendInput,};use windows::Win32::UI::WindowsAndMessaging::{ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT,@@ -594,3 +595,103 @@ pub fn send_drag_synthesized(Ok(())}++/// Standard wheel notch delta. A `mouseData` value of `±WHEEL_DELTA` is one+/// detent of the physical mouse wheel.+const WHEEL_DELTA: i32 = 120;++/// Compute the `MOUSEINPUT::mouseData` value for a wheel event of `ticks`+/// detents. Positive ticks = wheel forward/up (vertical) or right (horizontal);+/// negative = down / left. `mouseData` is a `u32` field carrying a signed+/// 32-bit delta, so we compute as `i32` then bit-cast to `u32` (this is what+/// the Win32 docs mean by "the value is a multiple of WHEEL_DELTA").+///+/// Factored out of [`send_wheel_synthesized`] so the sign/magnitude encoding is+/// unit-testable without a live display / `SendInput`.+fn wheel_mouse_data(ticks: i32) -> u32 {+ (WHEEL_DELTA * ticks) as u32+}++/// Synthesize a single mouse-wheel event at screen coordinates `(sx, sy)` via+/// `SendInput`.+///+/// The OS routes wheel input to the window **under the cursor**, not the+/// foreground window, so we `SetCursorPos(sx, sy)` first to place the wheel+/// over the intended target. `ticks` encodes both magnitude and direction:+/// positive scrolls up (vertical) / right (horizontal), negative scrolls down /+/// left — matching the `MOUSEEVENTF_WHEEL` / `MOUSEEVENTF_HWHEEL` convention+/// where `mouseData = WHEEL_DELTA * ticks`.+///+/// Unlike [`send_click_synthesized`] this does NOT do a foreground swap: wheel+/// delivery follows the cursor, so positioning the cursor is sufficient. The+/// cursor is restored to its previous position afterward.+pub fn send_wheel_synthesized(sx: i32, sy: i32, ticks: i32, horizontal: bool) -> Result<()> {+ let flag = if horizontal { MOUSEEVENTF_HWHEEL } else { MOUSEEVENTF_WHEEL };+ let mouse_data = wheel_mouse_data(ticks);++ let wheel_input = INPUT {+ r#type: INPUT_MOUSE,+ Anonymous: INPUT_0 {+ mi: MOUSEINPUT {+ dx: 0,+ dy: 0,+ mouseData: mouse_data,+ dwFlags: flag,+ time: 0,+ dwExtraInfo: 0,+ },+ },+ };++ unsafe {+ let mut prev_cursor = POINT::default();+ let _ = GetCursorPos(&mut prev_cursor);++ // Wheel routes to the window under the cursor — place it on the target.+ let _ = SetCursorPos(sx, sy);++ let events = [wheel_input];+ let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);+ if sent as usize != events.len() {+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ bail!("SendInput inserted {sent}/{} wheel events", events.len());+ }++ // Brief settle, then restore the cursor.+ sleep(Duration::from_millis(20));+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ }++ Ok(())+}++#[cfg(test)]+mod wheel_tests {+ use super::{wheel_mouse_data, WHEEL_DELTA};++ #[test]+ fn wheel_data_up_is_positive_one_notch() {+ // +1 tick (up / right) → +WHEEL_DELTA, bit-cast to u32.+ assert_eq!(wheel_mouse_data(1), WHEEL_DELTA as u32);+ assert_eq!(wheel_mouse_data(1), 120u32);+ }++ #[test]+ fn wheel_data_down_is_negative_one_notch() {+ // -1 tick (down / left) → -WHEEL_DELTA, bit-cast: 0xFFFFFF88.+ assert_eq!(wheel_mouse_data(-1), (-WHEEL_DELTA) as u32);+ assert_eq!(wheel_mouse_data(-1), 0xFFFF_FF88);+ }++ #[test]+ fn wheel_data_scales_with_ticks() {+ assert_eq!(wheel_mouse_data(3), (3 * WHEEL_DELTA) as u32);+ assert_eq!(wheel_mouse_data(3), 360u32);+ assert_eq!(wheel_mouse_data(-3) as i32, -360);+ }++ #[test]+ fn wheel_data_zero_is_zero() {+ assert_eq!(wheel_mouse_data(0), 0);+ }+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 1e4249c6db..c930ad460e 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -154,16 +154,41 @@ pub(crate) fn resolve_cursor_key(args: &Value) -> String {NO_CURSOR.to_owned()}+/// Returns `true` when a click/scroll invocation should take the **window-less+/// screen-absolute** branch: the caller gave numeric `x` AND `y`, gave NO `pid`+/// and NO `window_id`, and the effective `capture_scope` is `"desktop"`.+///+/// Pure + arg-shape agnostic (works for both click and scroll args) so it's+/// unit-testable without Win32. When this returns `false` but `x,y` are present+/// with no pid/window_id, the caller returns a `desktop_scope_disabled`+/// structured error telling the agent to `set_config capture_scope=desktop`.+fn is_windowless_desktop_action(args: &serde_json::Value, scope: &str) -> bool {+ if scope != "desktop" {+ return false;+ }+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);+ if has_pid || has_window_id {+ return false;+ }+ let has_num = |k: &str| args.get(k).map(|v| v.is_number()).unwrap_or(false);+ has_num("x") && has_num("y")+}+// ── DriverConfig + ResizeRegistry + ZoomRegistry ─────────────────────────────#[derive(Clone)]pub struct DriverConfig {pub capture_mode: String,+ /// Capture scope for vision loops: `"window"` (per-window, the default) or+ /// `"desktop"` (full-display). Gates the window-less screen-absolute+ /// click/scroll branches — those activate only under `"desktop"`.+ pub capture_scope: String,pub max_image_dimension: u32,}impl Default for DriverConfig {- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }+ fn default() -> Self { Self { capture_mode: "som".into(), capture_scope: "window".into(), max_image_dimension: 1568 } }}/// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to@@ -176,6 +201,9 @@ pub fn load_driver_config() -> DriverConfig {if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) {cfg.capture_mode = v;}+ if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) {+ cfg.capture_scope = v;+ }if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) {if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; }}@@ -1947,8 +1975,8 @@ impl Tool for ClickTool {tool). The Swift-only `action` / `modifier` / `debug_image_out` schema \fields aren't supported yet.".into(),input_schema: json!({- "type":"object","required":["pid"],"properties":{- "pid":{"type":"integer","description":"Target process ID."},+ "type":"object","properties":{+ "pid":{"type":"integer","description":"Target process ID. Required UNLESS using window-less desktop-scope clicks: with capture_scope=\"desktop\" and no pid/window_id, x/y are treated as TRUE SCREEN pixels (call get_desktop_state first)."},"window_id":{"type":"integer","description":"HWND for the window whose get_window_state produced the element_index. Required when element_index is used. Optional when element_token is supplied (the token carries it)."},"element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},"element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},@@ -1969,6 +1997,78 @@ impl Tool for ClickTool {use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error};use crate::uia::cache::SnapshotKind;let cursor_key = resolve_cursor_key(&args);++ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────+ // When the caller gives x,y with NO pid/window_id, treat x,y as TRUE+ // SCREEN pixels. Gate on effective capture_scope: only "desktop" enables+ // this; under "window" we return a structured `desktop_scope_disabled`+ // error pointing the caller at set_config.+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);+ if !has_pid && !has_window_id && has_xy {+ let scope = self.state.config.read().unwrap().capture_scope.clone();+ if !is_windowless_desktop_action(&args, &scope) {+ return ToolResult::error(+ "click: x,y given with no pid/window_id, but capture_scope is \+ \"window\". Screen-absolute clicks require desktop scope. Call \+ set_config with capture_scope=desktop (and use get_desktop_state \+ to pick coordinates), or pass a pid/window_id."+ )+ .with_structured(json!({+ "code": "desktop_scope_disabled",+ "capture_scope": scope,+ "suggestion": "set_config capture_scope=desktop",+ }));+ }+ // Resolve button (reuse the same validation as the pid path).+ let button_raw = args.str_or("button", "left").to_lowercase();+ if !matches!(button_raw.as_str(), "" | "left" | "right" | "middle") {+ return ToolResult::error(format!(+ "click: unknown button \"{button_raw}\" — expected one of left, right, middle."+ ));+ }+ let button = if button_raw.is_empty() { "left".to_string() } else { button_raw };+ let count = args.u64_or("count", 1) as usize;+ let sx = args.f64_or("x", 0.0) as i32;+ let sy = args.f64_or("y", 0.0) as i32;++ // Animate the agent cursor to the screen point, then click.+ overlay_glide_to(&cursor_key, sx as f64, sy as f64).await;+ crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {+ x: sx as f64, y: sy as f64,+ });++ // Resolve the HWND that owns this screen pixel and click it via+ // send_click_synthesized — it does the foreground-swap + UIPI checks+ // on whatever owns the pixel, which is what lands Chromium-content+ // clicks. WindowFromPoint walks to the leaf window at the point.+ // (send_click_synthesized restores the previous foreground + cursor+ // itself ~40ms after the click, so no extra restore guard here.)+ let send_result = tokio::task::spawn_blocking(move || -> anyhow::Result<u64> {+ use windows::Win32::Foundation::POINT;+ use windows::Win32::UI::WindowsAndMessaging::WindowFromPoint;+ let target = unsafe { WindowFromPoint(POINT { x: sx, y: sy }) };+ if target.0.is_null() {+ anyhow::bail!("No window under screen point ({sx},{sy}).");+ }+ let hwnd_u = target.0 as u64;+ crate::input::send_click_synthesized(hwnd_u, sx, sy, count, &button)?;+ Ok(hwnd_u)+ }).await;+ return match send_result {+ Ok(Ok(hwnd_u)) => {+ let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" };+ ToolResult::text(format!(+ "✅ Sent {click_word} via SendInput at screen ({sx},{sy}) on HWND 0x{hwnd_u:x} (desktop scope)."+ ))+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };// Surface 6: element_token / element_index precedence resolution.// Windows uses u64 HWND but the token registry stores u32; truncate@@ -3318,7 +3418,9 @@ impl Tool for SetValueTool {// ── scroll ────────────────────────────────────────────────────────────────────-pub struct ScrollTool;+pub struct ScrollTool {+ state: Arc<ToolState>,+}static SCROLL_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();#[async_trait]@@ -3339,12 +3441,14 @@ impl Tool for ScrollTool {Note: `element_index` is accepted for cross-platform parity but currently \no-op on Windows (UIA SetFocus not wired up yet — same caveat as `press_key`).".into(),input_schema: json!({- "type":"object","required":["pid","direction"],"properties":{- "pid":{"type":"integer","description":"Target process ID."},+ "type":"object","required":["direction"],"properties":{+ "pid":{"type":"integer","description":"Target process ID. Required UNLESS using window-less desktop-scope scroll: with capture_scope=\"desktop\" and no pid/window_id, x/y are TRUE SCREEN pixels and the wheel routes to the window under that point."},"direction":{"type":"string","enum":["up","down","left","right"]},"by":{"type":"string","enum":["line","page"],"description":"Scroll granularity. Default: line."},"amount":{"type":"integer","minimum":1,"maximum":50,"description":"Number of scroll ticks. Default 3."},+ "x":{"type":"number","description":"Screen-absolute X (desktop scope only) — wheel routes to the window under (x,y). Must be paired with y and no pid/window_id."},+ "y":{"type":"number","description":"Screen-absolute Y (desktop scope only). Must be paired with x and no pid/window_id."},"window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."},"element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},"element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},@@ -3361,6 +3465,65 @@ impl Tool for ScrollTool {// scroll-specific structured error (see commit 9e30d2cb). The// dispatch enums are still in use just below.use crate::input::dispatch::{DispatchMode, EventKind};+ use cua_driver_core::tool_args::ArgsExt;+ // `direction` is required in both the pid path and the window-less+ // desktop path, so resolve it before the pid check.+ let direction = match args.get("direction").and_then(|v| v.as_str()) {+ Some(d) => d.to_owned(),+ None => return ToolResult::error("Missing required string field direction."),+ };+ let amount = args.u64_or("amount", 3).clamp(1, 50) as u32;++ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────+ // No pid/window_id + numeric x,y + desktop scope → synthesize a wheel+ // event at the screen point via SendInput. The wheel routes to whatever+ // window is under (x,y). up/down map to a vertical wheel (sign), and+ // left/right to a horizontal wheel; `amount` is the tick count.+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);+ if !has_pid && !has_window_id && has_xy {+ let scope = self.state.config.read().unwrap().capture_scope.clone();+ if !is_windowless_desktop_action(&args, &scope) {+ return ToolResult::error(+ "scroll: x,y given with no pid/window_id, but capture_scope is \+ \"window\". Screen-absolute scroll requires desktop scope. Call \+ set_config with capture_scope=desktop (and use get_desktop_state \+ to pick coordinates), or pass a pid/window_id."+ )+ .with_structured(serde_json::json!({+ "code": "desktop_scope_disabled",+ "capture_scope": scope,+ "suggestion": "set_config capture_scope=desktop",+ }));+ }+ let sx = args.f64_or("x", 0.0) as i32;+ let sy = args.f64_or("y", 0.0) as i32;+ // Direction → (horizontal?, sign). Positive ticks = up / right.+ let (horizontal, sign) = match direction.as_str() {+ "up" => (false, 1),+ "down" => (false, -1),+ "right" => (true, 1),+ "left" => (true, -1),+ other => return ToolResult::error(format!(+ "scroll: unknown direction \"{other}\" — expected up, down, left, right."+ )),+ };+ let ticks = sign * amount as i32;+ let dir_disp = direction.clone();+ let result = tokio::task::spawn_blocking(move || {+ crate::input::send_wheel_synthesized(sx, sy, ticks, horizontal)+ }).await;+ return match result {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Scrolled {dir_disp} via SendInput wheel ({amount} tick(s)) at screen ({sx},{sy}) (desktop scope)."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+// Swift error wording 1:1.let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {Some(p) => p,@@ -3368,15 +3531,9 @@ impl Tool for ScrollTool {};let pid = raw_pid as u32;let dispatch = DispatchMode::from_args(&args);- let direction = match args.get("direction").and_then(|v| v.as_str()) {- Some(d) => d.to_owned(),- None => return ToolResult::error("Missing required string field direction."),- };- use cua_driver_core::tool_args::ArgsExt;let by = args.str_or("by", "line");let direction_display = direction.clone();let by_display = by.clone();- let amount = args.u64_or("amount", 3).clamp(1, 50) as u32;// Surface 6: element_token / element_index precedence resolution.let resolved = match cua_driver_core::element_token::resolve_element_args(pid as i32,@@ -4286,6 +4443,17 @@ impl Tool for DragTool {// ── get_screen_size ───────────────────────────────────────────────────────────+/// Read the primary display size in PHYSICAL pixels.+///+/// With permonitorv2 DPI awareness (set in cua-driver.manifest),+/// `SM_CXSCREEN` / `SM_CYSCREEN` already return physical pixels — the same+/// coordinate space screenshots and pixel clicks use on Windows. Shared by+/// `get_screen_size` and `get_desktop_state`.+fn physical_screen_size() -> (i32, i32) {+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};+ unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) }+}+pub struct GetScreenSizeTool;static GSS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();@@ -4302,13 +4470,12 @@ impl Tool for GetScreenSizeTool {})}async fn invoke(&self, _args: Value) -> ToolResult {- use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};use windows::Win32::UI::HiDpi::GetDpiForSystem;// With permonitorv2 DPI awareness (set in cua-driver.manifest),// SM_CXSCREEN/SM_CYSCREEN return PHYSICAL pixels — the same// coordinate space screenshots and pixel clicks use on Windows.// Report these as-is, along with the scale factor for reference.- let (w, h) = unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) };+ let (w, h) = physical_screen_size();let dpi = unsafe { GetDpiForSystem() };let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };ToolResult::text(format!("✅ Main display: {w}x{h} pixels @ {scale}x"))@@ -4316,6 +4483,104 @@ impl Tool for GetScreenSizeTool {}}+// ── get_desktop_state ─────────────────────────────────────────────────────────++/// `get_desktop_state` — full-display vision screenshot (Windows).+///+/// Vision-only desktop capture: grabs the ENTIRE primary display at native+/// physical-pixel size (no downscale) so screen-absolute pixel picks land+/// exactly, then reports the true screen size. No UIA walk, no pid/window_id —+/// this is the capture surface for `capture_scope="desktop"` GUI loops where+/// the agent drives `click(x,y)` / `scroll(x,y)` against screen-absolute+/// coordinates.+///+/// Mirrors the `get_window_state` vision branch's ToolResult shape: an+/// `image_png` content part (or a written-out file path), a text summary line,+/// and a `structuredContent` object.+pub struct GetDesktopStateTool;+static GDS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();++#[async_trait]+impl Tool for GetDesktopStateTool {+ fn def(&self) -> &ToolDef {+ GDS_DEF.get_or_init(|| ToolDef {+ name: "get_desktop_state".into(),+ description: "Capture a full-display vision screenshot in true screen pixels \+ (no downscale), for capture_scope=\"desktop\" GUI loops where the agent then \+ drives click(x,y)/scroll(x,y) with no pid/window_id. Returns the PNG at native \+ display resolution plus the true screen size so screen-absolute pixel picks \+ land exactly. Vision-only: no UIA tree walk.".into(),+ input_schema: json!({+ "type": "object",+ "properties": {+ "session": { "type": "string", "description": "Optional session id." },+ "screenshot_out_file": { "type": "string", "description": "Write PNG here instead of base64." }+ },+ "additionalProperties": false+ }),+ read_only: true, destructive: false, idempotent: false, open_world: false,+ })+ }++ async fn invoke(&self, args: Value) -> ToolResult {+ use cua_driver_core::tool_args::ArgsExt;+ use cua_driver_core::protocol::Content;++ let screenshot_out_file = args.opt_str("screenshot_out_file");++ // True screen geometry in physical pixels (same space as the capture).+ let (screen_width, screen_height) = physical_screen_size();++ // Capture the FULL display at native size — no resize. Run the+ // blocking GDI capture off the async runtime.+ let out_file = screenshot_out_file.clone();+ let res = tokio::task::spawn_blocking(+ move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32)> {+ use base64::{engine::general_purpose::STANDARD as BASE64, Engine};+ let png = crate::capture::screenshot_display_bytes()?;+ let (w, h) = crate::capture::png_dimensions_pub(&png)?;+ if let Some(ref path) = out_file {+ std::fs::write(path, &png)?;+ Ok((None, Some(path.clone()), w, h))+ } else {+ Ok((Some(BASE64.encode(&png)), None, w, h))+ }+ },+ )+ .await;++ let (b64_opt, file_path, screenshot_width, screenshot_height) = match res {+ Ok(Ok(v)) => v,+ Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),+ Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),+ };++ let mut content: Vec<Content> = Vec::new();+ if let Some(b64) = b64_opt {+ content.push(Content::image_png(b64));+ }+ let summary = format!(+ "desktop screenshot {screenshot_width}x{screenshot_height} px \+ (screen {screen_width}x{screen_height} px)"+ );+ content.push(Content::text(summary));++ let mut structured = json!({+ "platform": "windows",+ "screenshot_width": screenshot_width,+ "screenshot_height": screenshot_height,+ "screen_width": screen_width,+ "screen_height": screen_height,+ "screenshot_mime_type": "image/png",+ });+ if let Some(ref fp) = file_path {+ structured["screenshot_file_path"] = json!(fp);+ }++ ToolResult { content, is_error: None, structured_content: Some(structured) }+ }+}+// ── get_cursor_position ───────────────────────────────────────────────────────pub struct GetCursorPositionTool;@@ -4970,6 +5235,7 @@ impl Tool for GetConfigTool {"version": env!("CARGO_PKG_VERSION"),"platform": "windows","capture_mode": cfg.capture_mode,+ "capture_scope": cfg.capture_scope,"max_image_dimension": cfg.max_image_dimension,"agent_cursor": { "enabled": cursor_enabled },"experimental_pip": pip_enabled,@@ -5013,6 +5279,7 @@ impl Tool for SetConfigTool {"key":{"type":"string","description":"Dotted snake_case path to a leaf config field (Swift-compatible shape). Pair with `value`."},"value":{"description":"New value for `key`. JSON type depends on the key."},"capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Legacy per-field shape."},+ "capture_scope":{"type":"string","enum":["window","desktop"],"description":"Capture scope: single window (default) or whole desktop. Desktop scope enables window-less screen-absolute click/scroll (no pid/window_id). Accepted in both the {key,value} and legacy per-field shapes."},"max_image_dimension":{"type":"integer","description":"Legacy per-field shape."},"experimental_pip":{"type":"boolean","description":"Legacy per-field shape. Enables PiP preview (applies next restart)."},"experimental_pip_geometry":{"type":"string","description":"Legacy per-field shape. PiP window size + optional position."}@@ -5039,6 +5306,17 @@ impl Tool for SetConfigTool {}None => return ToolResult::error(format!("`capture_mode` must be a string, got {val}.")),},+ "capture_scope" => match val.as_str() {+ Some(s @ ("window" | "desktop")) => {+ cfg.capture_scope = s.to_owned();+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(s.to_owned())) {+ tracing::warn!("set_config: failed to persist capture_scope: {e}");+ }+ applied = true;+ }+ Some(other) => return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{other}\".")),+ None => return ToolResult::error(format!("`capture_scope` must be a string, got {val}.")),+ },"max_image_dimension" => match val.as_u64() {Some(n) => {cfg.max_image_dimension = n as u32;@@ -5073,7 +5351,7 @@ impl Tool for SetConfigTool {None => return ToolResult::error(format!("`experimental_pip_geometry` must be a string, got {val}.")),},other => return ToolResult::error(format!(- "Unknown config key `{other}`. Known: capture_mode, max_image_dimension, experimental_pip, experimental_pip_geometry."+ "Unknown config key `{other}`. Known: capture_mode, capture_scope, max_image_dimension, experimental_pip, experimental_pip_geometry.")),}}@@ -5085,6 +5363,16 @@ impl Tool for SetConfigTool {}applied = true;}+ if let Some(scope) = args.get("capture_scope").and_then(|v| v.as_str()) {+ if !matches!(scope, "window" | "desktop") {+ return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."));+ }+ cfg.capture_scope = scope.to_owned();+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.to_owned())) {+ tracing::warn!("set_config: failed to persist capture_scope: {e}");+ }+ applied = true;+ }if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) {cfg.max_image_dimension = dim as u32;if let Err(e) = pip_preview::write_config_key("max_image_dimension", Value::from(dim)) {@@ -5124,6 +5412,7 @@ impl Tool for SetConfigTool {"version": env!("CARGO_PKG_VERSION"),"platform": "windows","capture_mode": cfg.capture_mode,+ "capture_scope": cfg.capture_scope,"max_image_dimension": cfg.max_image_dimension,"agent_cursor": { "enabled": cursor_enabled },"experimental_pip": pip_enabled,@@ -5897,7 +6186,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry {r.register(Box::new(PressKeyTool));r.register(Box::new(HotkeyTool));r.register(Box::new(SetValueTool { state: state.clone() }));- r.register(Box::new(ScrollTool));+ r.register(Box::new(ScrollTool { state: state.clone() }));// `screenshot` / `ScreenshotCompatTool` removed from the tool surface// — `get_window_state` with `capture_mode:"vision"` is the single// canonical path for getting a window screenshot. Reasons:@@ -5917,6 +6206,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry {// depend on them via `Tool` trait reflection.let _ = compat; // formerly drove the ScreenshotCompatTool branchr.register(Box::new(GetScreenSizeTool));+ r.register(Box::new(GetDesktopStateTool));r.register(Box::new(GetCursorPositionTool));r.register(Box::new(MoveCursorTool { state: state.clone() }));r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));@@ -6210,3 +6500,75 @@ mod click_button_schema_tests {}}}++#[cfg(test)]+mod desktop_scope_tests {+ use super::{is_windowless_desktop_action, DriverConfig, GetDesktopStateTool};+ use cua_driver_core::tool::Tool;+ use serde_json::json;++ // ── is_windowless_desktop_action ──────────────────────────────────────────++ #[test]+ fn windowless_true_for_xy_under_desktop_scope_click_shape() {+ // Click arg shape: {x, y}.+ assert!(is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "desktop"));+ }++ #[test]+ fn windowless_true_for_xy_under_desktop_scope_scroll_shape() {+ // Scroll arg shape: {direction, x, y}.+ assert!(is_windowless_desktop_action(+ &json!({"direction": "down", "x": 10, "y": 20}),+ "desktop"+ ));+ }++ #[test]+ fn windowless_false_when_pid_present() {+ assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20, "pid": 5}), "desktop"));+ }++ #[test]+ fn windowless_false_when_window_id_present() {+ assert!(!is_windowless_desktop_action(+ &json!({"x": 10, "y": 20, "window_id": 99}),+ "desktop"+ ));+ }++ #[test]+ fn windowless_false_under_window_scope() {+ assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "window"));+ }++ #[test]+ fn windowless_false_when_xy_missing() {+ assert!(!is_windowless_desktop_action(&json!({"x": 10}), "desktop"));+ assert!(!is_windowless_desktop_action(&json!({"y": 20}), "desktop"));+ assert!(!is_windowless_desktop_action(&json!({}), "desktop"));+ // Non-numeric x/y must not qualify.+ assert!(!is_windowless_desktop_action(&json!({"x": "10", "y": "20"}), "desktop"));+ }++ // ── DriverConfig default ──────────────────────────────────────────────────++ #[test]+ fn default_capture_scope_is_window() {+ assert_eq!(DriverConfig::default().capture_scope, "window");+ }++ // ── get_desktop_state schema ──────────────────────────────────────────────++ #[test]+ fn get_desktop_state_schema_shape() {+ let d = GetDesktopStateTool.def();+ assert!(d.read_only, "get_desktop_state must be read_only");+ let props = d.input_schema["properties"].as_object().unwrap();+ assert!(!props.contains_key("pid"), "must not accept pid");+ assert!(!props.contains_key("window_id"), "must not accept window_id");+ assert!(props.contains_key("session"));+ assert!(props.contains_key("screenshot_out_file"));+ assert_eq!(d.input_schema["additionalProperties"], json!(false));+ }+}