Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): desktop-scope Phase 1 — capture_scope config, get_desktop_state, Windows screen-absolute actions (#1968) (#2019)

Francesco Bonaccifc2718541497
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index ce5e36c50c..77c53d14d7 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -201,6 +201,7 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {
"screen.capture.region",
],
"get_screen_size" => &["screen.dimensions"],
+ "get_desktop_state" => &["screen.capture", "screen.dimensions"],
"get_cursor_position" => &["screen.cursor.position"],
// ── accessibility / window state ─────────────────────────────
@@ -520,7 +521,8 @@ mod capability_tests {
// keyboard
"type_text", "type_text_chars", "press_key", "hotkey", "set_value",
// screen
- "zoom", "get_screen_size", "get_cursor_position",
+ "zoom", "get_screen_size", "get_desktop_state",
+ "get_cursor_position",
// accessibility
"get_accessibility_tree", "get_window_state",
// app / window
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 66cfb6c0a3..da3e680867 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -19,11 +19,12 @@ use cursor_overlay::CursorRegistry;
#[derive(Clone)]
pub struct DriverConfig {
pub capture_mode: String,
+ pub capture_scope: String,
pub max_image_dimension: u32,
}
impl Default for DriverConfig {
- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }
+ fn default() -> Self { Self { capture_mode: "som".into(), capture_scope: "window".into(), max_image_dimension: 1568 } }
}
/// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to
@@ -36,6 +37,9 @@ pub fn load_driver_config() -> DriverConfig {
if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) {
cfg.capture_mode = v;
}
+ if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) {
+ cfg.capture_scope = v;
+ }
if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) {
if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; }
}
@@ -2872,33 +2876,11 @@ impl Tool for GetScreenSizeTool {
}
async fn invoke(&self, _args: Value) -> ToolResult {
let result = tokio::task::spawn_blocking(|| {
- use x11rb::connection::Connection;
- use x11rb::rust_connection::RustConnection;
- let (conn, screen_num) = RustConnection::connect(None)
- .map_err(|e| anyhow::anyhow!("{e}{}", crate::no_display_hint()))?;
- let setup = conn.setup();
- let screen = &setup.roots[screen_num];
- let w = screen.width_in_pixels as u32;
- let h = screen.height_in_pixels as u32;
- // WSLg / headless XWayland quirk: the X server connects but the
- // root screen advertises a 0-px geometry until a real output is
- // attached. Returning {width:0,height:0} here would propagate a
- // success with zero dimensions to the client, which then either
- // divides by zero when scaling or feeds the value into `int(...)`
- // after the missing key collapses to None. Fail loudly with an
- // actionable, typed error instead (never emit a 0/null where the
- // client expects a usable int). See issue #2005.
- if w == 0 || h == 0 {
- anyhow::bail!(
- "X11 connected but reports a 0x0 root screen — no usable \
- display geometry.{}",
- crate::no_display_hint()
- );
- }
// X11 reports pixel dimensions; scale factor on X11 is not
// well-defined per-monitor, so report 1.0 (matches DPI-unaware
// assumption). Wayland/HiDPI X11 callers should query
// `xrandr --query` for true scale.
+ let (w, h) = x11_screen_size()?;
Ok::<(u32, u32, f64), anyhow::Error>((w, h, 1.0))
}).await;
match result {
@@ -2911,6 +2893,111 @@ impl Tool for GetScreenSizeTool {
}
}
+/// Read the true X11 root-window size in pixels: (width, height).
+/// Shared by `get_screen_size` and `get_desktop_state`.
+fn x11_screen_size() -> anyhow::Result<(u32, u32)> {
+ use x11rb::connection::Connection;
+ use x11rb::rust_connection::RustConnection;
+ let (conn, screen_num) = RustConnection::connect(None)
+ .map_err(|e| anyhow::anyhow!("{e}{}", crate::no_display_hint()))?;
+ let setup = conn.setup();
+ let screen = &setup.roots[screen_num];
+ let w = screen.width_in_pixels as u32;
+ let h = screen.height_in_pixels as u32;
+ // WSLg / headless XWayland quirk: the X server connects but the
+ // root screen advertises a 0-px geometry until a real output is
+ // attached. Returning {width:0,height:0} here would propagate a
+ // success with zero dimensions to the client, which then either
+ // divides by zero when scaling or feeds the value into `int(...)`
+ // after the missing key collapses to None. Fail loudly with an
+ // actionable, typed error instead (never emit a 0/null where the
+ // client expects a usable int). See issue #2005.
+ if w == 0 || h == 0 {
+ anyhow::bail!(
+ "X11 connected but reports a 0x0 root screen — no usable \
+ display geometry.{}",
+ crate::no_display_hint()
+ );
+ }
+ Ok((w, h))
+}
+
+// ── get_desktop_state ─────────────────────────────────────────────────────────
+
+pub struct GetDesktopStateTool;
+static GDS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+
+#[async_trait]
+impl Tool for GetDesktopStateTool {
+ fn def(&self) -> &ToolDef {
+ GDS_DEF.get_or_init(|| ToolDef {
+ name: "get_desktop_state".into(),
+ description: "Full-display vision screenshot in true screen pixels (no downscale), \
+ for capture_scope=\"desktop\" GUI loops. Captures the entire display (root \
+ window) as native-size PNG so screen-absolute pixel coordinates land exactly. \
+ No AT-SPI walk.".into(),
+ input_schema: json!({"type":"object","properties":{
+ "session":{"type":"string","description":"Optional session id."},
+ "screenshot_out_file":{"type":"string","description":"Write PNG here instead of base64."}
+ },"additionalProperties":false}),
+ read_only: true, destructive: false, idempotent: false, open_world: false,
+ })
+ }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ let out_file = args.opt_str("screenshot_out_file");
+
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
+ // Vision-only: capture the FULL DISPLAY at native size. No downscale
+ // so screen-absolute pixels land exactly.
+ let png = crate::capture::screenshot_display_bytes()?;
+ let (shot_w, shot_h) = crate::capture::png_dimensions_pub(&png)?;
+ // True screen size from the X11 root window.
+ let (screen_w, screen_h) = x11_screen_size()?;
+ // Optional: write PNG to disk instead of returning base64.
+ let written = if let Some(path) = out_file.as_deref() {
+ std::fs::write(path, &png)?;
+ Some(path.to_string())
+ } else {
+ None
+ };
+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
+ let b64 = if written.is_some() { None } else { Some(B64.encode(&png)) };
+ Ok((b64, shot_w, shot_h, screen_w, screen_h, written))
+ }).await;
+
+ match result {
+ Ok(Ok((b64_opt, shot_w, shot_h, screen_w, screen_h, written))) => {
+ let mut content = Vec::new();
+ let mut structured = json!({
+ "platform": "linux",
+ "screenshot_width": shot_w,
+ "screenshot_height": shot_h,
+ "screen_width": screen_w,
+ "screen_height": screen_h,
+ "screenshot_mime_type": "image/png",
+ });
+ if let Some(b64) = b64_opt {
+ content.push(cua_driver_core::protocol::Content::image_png(b64));
+ }
+ if let Some(path) = written {
+ structured["screenshot_file_path"] = json!(path);
+ content.push(cua_driver_core::protocol::Content::text(format!(
+ "✅ Desktop screenshot {shot_w}x{shot_h} written to {path} (screen {screen_w}x{screen_h})"
+ )));
+ } else {
+ content.push(cua_driver_core::protocol::Content::text(format!(
+ "✅ Desktop screenshot {shot_w}x{shot_h} (screen {screen_w}x{screen_h})"
+ )));
+ }
+ ToolResult { content, is_error: None, structured_content: Some(structured) }
+ }
+ Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ }
+ }
+}
+
// ── get_cursor_position ───────────────────────────────────────────────────────
pub struct GetCursorPositionTool;
@@ -3373,6 +3460,7 @@ impl Tool for GetConfigTool {
"version": env!("CARGO_PKG_VERSION"),
"platform": "linux",
"capture_mode": cfg.capture_mode,
+ "capture_scope": cfg.capture_scope,
"max_image_dimension": cfg.max_image_dimension,
"experimental_pip": pip_enabled,
"experimental_pip_geometry": pip_geometry
@@ -3405,6 +3493,7 @@ impl Tool for SetConfigTool {
"key":{"type":"string","description":"Name of a single config field to write ({key, value} shape). Pair with `value`."},
"value":{"description":"New value for `key`. JSON type depends on the key."},
"capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Legacy per-field shape. Default capture mode for get_window_state."},
+ "capture_scope":{"type":"string","enum":["window","desktop"],"description":"Capture scope: single window or whole desktop. Default window."},
"max_image_dimension":{"type":"integer","description":"Legacy per-field shape. Max dimension for screenshot resizing (0 = no limit)."},
"experimental_pip":{"type":"boolean","description":"Enable the experimental PiP preview window (applies next restart; Linux backend stubbed)."},
"experimental_pip_geometry":{"type":"string","description":"PiP window size + optional position in `WxH` or `WxH+X+Y` form."}
@@ -3435,6 +3524,17 @@ impl Tool for SetConfigTool {
}
None => return ToolResult::error(format!("`capture_mode` must be a string, got {val}.")),
},
+ "capture_scope" => match val.as_str() {
+ Some(s @ ("window" | "desktop")) => {
+ cfg.capture_scope = s.to_owned();
+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(s.to_owned())) {
+ tracing::warn!("set_config: failed to persist capture_scope: {e}");
+ }
+ parts.push(format!("capture_scope={s}"));
+ }
+ Some(other) => return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{other}\".")),
+ None => return ToolResult::error(format!("`capture_scope` must be a string, got {val}.")),
+ },
"max_image_dimension" => match val.as_u64() {
Some(n) => {
cfg.max_image_dimension = n as u32;
@@ -3469,7 +3569,7 @@ impl Tool for SetConfigTool {
None => return ToolResult::error(format!("`experimental_pip_geometry` must be a string, got {val}.")),
},
other => return ToolResult::error(format!(
- "Unknown config key `{other}`. Known: capture_mode, max_image_dimension, experimental_pip, experimental_pip_geometry."
+ "Unknown config key `{other}`. Known: capture_mode, capture_scope, max_image_dimension, experimental_pip, experimental_pip_geometry."
)),
}
}
@@ -3481,6 +3581,16 @@ impl Tool for SetConfigTool {
parts.push(format!("capture_mode={mode}"));
cfg.capture_mode = mode;
}
+ if let Some(scope) = args.opt_str("capture_scope") {
+ if scope != "window" && scope != "desktop" {
+ return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."));
+ }
+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.clone())) {
+ tracing::warn!("set_config: failed to persist capture_scope: {e}");
+ }
+ parts.push(format!("capture_scope={scope}"));
+ cfg.capture_scope = scope;
+ }
if let Some(dim) = args.opt_u64("max_image_dimension") {
cfg.max_image_dimension = dim as u32;
if let Err(e) = pip_preview::write_config_key("max_image_dimension", Value::from(dim)) {
@@ -3514,6 +3624,7 @@ impl Tool for SetConfigTool {
ToolResult::text(msg)
.with_structured(json!({
"capture_mode": cfg.capture_mode,
+ "capture_scope": cfg.capture_scope,
"max_image_dimension": cfg.max_image_dimension,
"experimental_pip": pip_enabled,
"experimental_pip_geometry": pip_geometry
@@ -3867,6 +3978,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
// screenshot path is `get_window_state` with `capture_mode:"vision"`.
let _ = compat;
r.register(Box::new(GetScreenSizeTool));
+ r.register(Box::new(GetDesktopStateTool));
r.register(Box::new(GetCursorPositionTool));
r.register(Box::new(MoveCursorTool { state: state.clone() }));
r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));
@@ -3930,3 +4042,13 @@ mod click_button_schema_tests {
assert!(lc.contains("wayland"), "description should call out wayland fallback");
}
}
+
+#[cfg(test)]
+mod driver_config_tests {
+ use super::DriverConfig;
+
+ #[test]
+ fn capture_scope_defaults_to_window() {
+ assert_eq!(DriverConfig::default().capture_scope, "window");
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs
index f238a36d9f..0decd59c73 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs
@@ -37,9 +37,11 @@ impl Tool for GetConfigTool {
// sees its own override layered over the global; the anonymous session
// (absent `_session_id`) sees the raw global — today's behavior.
let session_id = args.opt_str("_session_id");
- let (capture_mode, max_image_dimension) = {
+ let (capture_mode, max_image_dimension, capture_scope) = {
let cfg = self.state.config.read().unwrap();
- self.state.session_config.effective(session_id.as_deref(), &cfg)
+ let (mode, dim) = self.state.session_config.effective(session_id.as_deref(), &cfg);
+ let scope = self.state.session_config.effective_scope(session_id.as_deref(), &cfg);
+ (mode, dim, scope)
};
// Report the CALLING session's own cursor enabled-state, not a
// nondeterministic HashMap.first(). Resolve the same key the click /
@@ -61,6 +63,7 @@ impl Tool for GetConfigTool {
"version": env!("CARGO_PKG_VERSION"),
"platform": "macos",
"capture_mode": capture_mode,
+ "capture_scope": capture_scope,
"max_image_dimension": max_image_dimension,
"agent_cursor": {
"enabled": cursor_enabled,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
new file mode 100644
index 0000000000..9413983471
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs
@@ -0,0 +1,152 @@
+//! `get_desktop_state` — full-display vision screenshot (macOS).
+//!
+//! Vision-only desktop capture: grabs the ENTIRE main display at native
+//! pixel size (no downscale) so screen-absolute pixel picks land exactly,
+//! then reports the true screen size + backing scale. No AX walk, no
+//! pid/window_id — this is the capture surface for `capture_scope="desktop"`
+//! GUI loops where the agent drives `click(x,y)` / `scroll(x,y)` against
+//! screen-absolute coordinates.
+//!
+//! Mirrors `get_window_state.rs`'s vision ToolResult shape: an `image_png`
+//! content part (or a written-out file path), a text summary line, and a
+//! `structuredContent` object.
+
+use async_trait::async_trait;
+use cua_driver_core::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}};
+use serde_json::Value;
+
+use super::get_screen_size::main_screen_size;
+
+pub struct GetDesktopStateTool;
+
+static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+
+fn def() -> &'static ToolDef {
+ DEF.get_or_init(|| ToolDef {
+ name: "get_desktop_state".into(),
+ description: "Capture a full-display vision screenshot in true screen pixels \
+ (no downscale), for capture_scope=\"desktop\" GUI loops where the agent then \
+ drives click(x,y)/scroll(x,y) with no pid/window_id. Returns the PNG at native \
+ display resolution plus the true screen size and backing scale factor so \
+ screen-absolute pixel picks land exactly. Vision-only: no AX tree walk."
+ .into(),
+ input_schema: serde_json::json!({
+ "type": "object",
+ "properties": {
+ "session": { "type": "string", "description": "Optional session id." },
+ "screenshot_out_file": { "type": "string", "description": "Write PNG here instead of base64." }
+ },
+ "additionalProperties": false
+ }),
+ read_only: true,
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ })
+}
+
+#[async_trait]
+impl Tool for GetDesktopStateTool {
+ fn def(&self) -> &ToolDef { def() }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ use cua_driver_core::tool_args::ArgsExt;
+
+ let screenshot_out_file = args.opt_str("screenshot_out_file").map(|s| {
+ // Expand ~ prefix (mirrors get_window_state).
+ if s.starts_with("~/") {
+ let home = std::env::var("HOME").unwrap_or_default();
+ format!("{home}/{}", &s[2..])
+ } else {
+ s
+ }
+ });
+
+ // True screen geometry (points + backing scale). Safe off the main thread.
+ let (screen_width, screen_height, scale_factor) = match main_screen_size() {
+ Some(t) => t,
+ None => return ToolResult::error("No main display detected."),
+ };
+
+ // Capture the FULL display at native size — no resize. Run the
+ // blocking screencapture subprocess off the async runtime.
+ let out_file = screenshot_out_file.clone();
+ let res = tokio::task::spawn_blocking(
+ move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32)> {
+ use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
+ let png = crate::capture::screenshot_display_bytes()?;
+ let (w, h) = crate::capture::png_dimensions(&png)?;
+ if let Some(ref path) = out_file {
+ std::fs::write(path, &png)?;
+ Ok((None, Some(path.clone()), w, h))
+ } else {
+ Ok((Some(BASE64.encode(&png)), None, w, h))
+ }
+ },
+ )
+ .await;
+
+ let (b64_opt, file_path, screenshot_width, screenshot_height) = match res {
+ Ok(Ok(v)) => v,
+ Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),
+ Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),
+ };
+
+ let mut content: Vec<Content> = Vec::new();
+ if let Some(b64) = b64_opt {
+ content.push(Content::image_png(b64));
+ }
+ let summary = format!(
+ "desktop screenshot {screenshot_width}x{screenshot_height} px \
+ (screen {screen_width}x{screen_height} pts @ {scale_factor}x)"
+ );
+ content.push(Content::text(summary));
+
+ let mut structured = serde_json::json!({
+ "platform": "macos",
+ "screenshot_width": screenshot_width,
+ "screenshot_height": screenshot_height,
+ "screen_width": screen_width,
+ "screen_height": screen_height,
+ "scale_factor": scale_factor,
+ "screenshot_mime_type": "image/png",
+ });
+ if let Some(ref fp) = file_path {
+ structured["screenshot_file_path"] = serde_json::json!(fp);
+ }
+
+ ToolResult { content, is_error: None, structured_content: Some(structured) }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn schema_has_no_pid_or_window_id_and_is_read_only() {
+ let d = def();
+ assert!(d.read_only, "get_desktop_state must be read_only");
+ assert!(!d.destructive);
+ assert!(!d.idempotent);
+ assert!(!d.open_world);
+
+ let props = d.input_schema["properties"].as_object().unwrap();
+ assert!(!props.contains_key("pid"), "must not accept pid");
+ assert!(!props.contains_key("window_id"), "must not accept window_id");
+ assert!(!props.contains_key("capture_mode"), "must not accept capture_mode");
+ assert!(props.contains_key("session"));
+ assert!(props.contains_key("screenshot_out_file"));
+ assert_eq!(d.input_schema["additionalProperties"], serde_json::json!(false));
+ }
+
+ #[test]
+ fn description_mentions_full_and_screen_or_display() {
+ let desc = def().description.to_lowercase();
+ assert!(desc.contains("full"), "description must mention 'full'");
+ assert!(
+ desc.contains("screen") || desc.contains("display"),
+ "description must mention 'screen' or 'display'"
+ );
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs
index 5fa196c120..dc39dee39a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs
@@ -45,7 +45,7 @@ impl Tool for GetScreenSizeTool {
/// The previous NSScreen-based implementation required `MainThreadMarker::new()`
/// which always returns `None` on async tokio threads, causing the tool to
/// return an error even when a display is attached.
-fn main_screen_size() -> Option<(i64, i64, f64)> {
+pub(crate) fn main_screen_size() -> Option<(i64, i64, f64)> {
use core_graphics::display::{CGMainDisplayID, CGDisplayBounds};
// SAFETY: CGMainDisplayID / CGDisplayBounds are thread-safe CG APIs.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 7d4457669e..6abd2db36b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -21,6 +21,7 @@ mod scroll;
// etc.) live elsewhere under CuaDriverCore::Capture and are reached
// through GetWindowStateTool.
pub(crate) mod get_screen_size;
+mod get_desktop_state;
mod get_cursor_position;
mod move_cursor;
mod cursor_tools;
@@ -114,6 +115,9 @@ impl ResizeRegistry {
pub struct DriverConfig {
/// Default capture_mode for get_window_state when not specified per-call.
pub capture_mode: String,
+ /// Capture scope for get_window_state: "window" (default) crops to the
+ /// target window; "desktop" captures the full display.
+ pub capture_scope: String,
/// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom.
/// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` —
/// the long edge is downscaled to this before encoding.
@@ -124,6 +128,7 @@ impl Default for DriverConfig {
fn default() -> Self {
Self {
capture_mode: "som".to_owned(),
+ capture_scope: "window".to_string(),
max_image_dimension: 1568,
}
}
@@ -153,6 +158,9 @@ pub fn load_driver_config() -> DriverConfig {
if let Some(v) = json.get("capture_mode").and_then(|v| v.as_str()) {
cfg.capture_mode = v.to_owned();
}
+ if let Some(v) = json.get("capture_scope").and_then(|v| v.as_str()) {
+ cfg.capture_scope = v.to_owned();
+ }
if let Some(v) = json.get("max_image_dimension").and_then(|v| v.as_u64()) {
if let Ok(v32) = u32::try_from(v) {
cfg.max_image_dimension = v32;
@@ -194,6 +202,7 @@ pub fn write_driver_config_key(key: &str, value: &serde_json::Value) -> Result<(
#[derive(Clone, Default)]
pub struct ConfigOverrides {
pub capture_mode: Option<String>,
+ pub capture_scope: Option<String>,
pub max_image_dimension: Option<u32>,
}
@@ -223,6 +232,9 @@ impl SessionConfigRegistry {
if delta.capture_mode.is_some() {
entry.capture_mode = delta.capture_mode;
}
+ if delta.capture_scope.is_some() {
+ entry.capture_scope = delta.capture_scope;
+ }
if delta.max_image_dimension.is_some() {
entry.max_image_dimension = delta.max_image_dimension;
}
@@ -242,6 +254,18 @@ impl SessionConfigRegistry {
}
}
+ /// Resolve the effective `capture_scope` for `session`, layering its
+ /// override over the global `DriverConfig`. Kept separate from `effective()`
+ /// so existing `(String, u32)` call sites stay unchanged. `session = None`
+ /// (anonymous) returns the global scope verbatim.
+ pub fn effective_scope(&self, session_id: Option<&str>, global: &DriverConfig) -> String {
+ let ov = session_id.and_then(|s| self.inner.lock().unwrap().get(s).cloned());
+ match ov {
+ Some(ov) => ov.capture_scope.unwrap_or_else(|| global.capture_scope.clone()),
+ None => global.capture_scope.clone(),
+ }
+ }
+
/// Drop `session`'s overrides. No-op for an unknown id (so `session_end`
/// for an anonymous / never-set session is harmless).
pub fn clear(&self, session: &str) {
@@ -330,6 +354,7 @@ pub fn register_all(registry: &mut ToolRegistry, compat: bool) {
// screenshot path is `get_window_state` with `capture_mode:"vision"`.
let _ = compat;
registry.register(Box::new(get_screen_size::GetScreenSizeTool));
+ registry.register(Box::new(get_desktop_state::GetDesktopStateTool));
registry.register(Box::new(get_cursor_position::GetCursorPositionTool));
registry.register(Box::new(move_cursor::MoveCursorTool::new(state.clone())));
registry.register(Box::new(cursor_tools::SetAgentCursorEnabledTool::new(state.clone())));
@@ -374,7 +399,35 @@ mod session_config_guard_tests {
use cua_driver_core::session::fire_session_end;
fn overrides(mode: &str) -> ConfigOverrides {
- ConfigOverrides { capture_mode: Some(mode.to_owned()), max_image_dimension: None }
+ ConfigOverrides { capture_mode: Some(mode.to_owned()), capture_scope: None, max_image_dimension: None }
+ }
+
+ #[test]
+ fn default_capture_scope_is_window() {
+ assert_eq!(DriverConfig::default().capture_scope, "window");
+ }
+
+ #[test]
+ fn effective_scope_uses_session_override_then_global() {
+ let reg = SessionConfigRegistry::new();
+ let global = DriverConfig::default();
+ let sid = "wb-scope-live-C1D2E3";
+ assert!(!cua_driver_core::session::is_session_ended(sid));
+
+ // No override yet → falls back to global ("window").
+ assert_eq!(reg.effective_scope(Some(sid), &global), "window");
+ // Anonymous → global.
+ assert_eq!(reg.effective_scope(None, &global), "window");
+
+ // Live session override applies.
+ reg.set(sid, ConfigOverrides {
+ capture_mode: None,
+ capture_scope: Some("desktop".to_owned()),
+ max_image_dimension: None,
+ });
+ assert_eq!(reg.effective_scope(Some(sid), &global), "desktop");
+ // Other sessions / anonymous still see global.
+ assert_eq!(reg.effective_scope(None, &global), "window");
}
#[test]
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs
index 345c881dd0..d5c82420c5 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs
@@ -31,6 +31,12 @@ fn def() -> &'static ToolDef {
"enum": ["som", "vision", "ax"],
"description": "Default capture mode for get_window_state."
},
+ "capture_scope": {
+ "type": "string",
+ "enum": ["window", "desktop"],
+ "description": "Capture scope for get_window_state: 'window' crops to the \
+ target window; 'desktop' captures the full display."
+ },
"max_image_dimension": {
"type": "integer",
"description": "Max dimension for screenshot resizing (0 = no limit)."
@@ -80,10 +86,22 @@ impl Tool for SetConfigTool {
};
let capture_mode = args.opt_str("capture_mode");
+ // Validate capture_scope up front so both branches share the check and
+ // we never half-apply an invalid value.
+ let capture_scope = args.opt_str("capture_scope");
+ if let Some(scope) = capture_scope.as_deref() {
+ if scope != "window" && scope != "desktop" {
+ return ToolResult::error(format!(
+ "capture_scope `{scope}` is invalid; expected `window` or `desktop`"
+ ));
+ }
+ }
+
let (effective_mode, effective_dim) = if let Some(sid) = session_id.as_deref() {
// Session-scoped override: in-memory only, no global write, no disk.
self.state.session_config.set(sid, ConfigOverrides {
capture_mode: capture_mode.clone(),
+ capture_scope: capture_scope.clone(),
max_image_dimension: max_dim,
});
self.state.session_config.effective(Some(sid), &self.state.config.read().unwrap())
@@ -97,6 +115,12 @@ impl Tool for SetConfigTool {
tracing::warn!("set_config: failed to persist capture_mode: {e}");
}
}
+ if let Some(scope) = capture_scope.clone() {
+ cfg.capture_scope = scope.clone();
+ if let Err(e) = write_driver_config_key("capture_scope", &Value::String(scope)) {
+ tracing::warn!("set_config: failed to persist capture_scope: {e}");
+ }
+ }
if let Some(dim32) = max_dim {
cfg.max_image_dimension = dim32;
if let Err(e) = write_driver_config_key("max_image_dimension", &Value::Number(u64::from(dim32).into())) {
@@ -105,6 +129,13 @@ impl Tool for SetConfigTool {
}
(cfg.capture_mode.clone(), cfg.max_image_dimension)
};
+ // Resolve the effective capture_scope for the echo via the same
+ // session/global precedence (kept separate from `effective()` so its
+ // `(String, u32)` signature stays unchanged for other call sites).
+ let effective_scope = self
+ .state
+ .session_config
+ .effective_scope(session_id.as_deref(), &self.state.config.read().unwrap());
// PiP keys persist to the same config.json but take effect only on
// next daemon restart — the backend is initialised once at startup.
let mut pip_note = String::new();
@@ -134,8 +165,8 @@ impl Tool for SetConfigTool {
""
};
ToolResult::text(format!(
- "Config updated: capture_mode={}, max_image_dimension={}{}{}",
- effective_mode, effective_dim, scope_note, pip_note
+ "Config updated: capture_mode={}, capture_scope={}, max_image_dimension={}{}{}",
+ effective_mode, effective_scope, effective_dim, scope_note, pip_note
))
}
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
index d1bb662f88..3209681ca4 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
@@ -15,7 +15,7 @@ pub mod dispatch;
pub mod inject;
pub use inject::{inject_click_screen, inject_key_cloaked, inject_text_cloaked, NoActivateGuard};
-pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized};
+pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized, send_wheel_synthesized};
pub use keyboard::{
is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,
send_key_synthesized,
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
index 7f8100c7ea..2d1e10c5fb 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
@@ -10,9 +10,10 @@ use std::time::Duration;
use windows::Win32::Foundation::{HWND, LPARAM, POINT, WPARAM};
use windows::Win32::Graphics::Gdi::{ClientToScreen, ScreenToClient};
use windows::Win32::UI::Input::KeyboardAndMouse::{
- INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,
- MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN,
- MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEINPUT, SendInput,
+ INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL, MOUSEEVENTF_LEFTDOWN,
+ MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE,
+ MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEEVENTF_WHEEL,
+ MOUSEINPUT, SendInput,
};
use windows::Win32::UI::WindowsAndMessaging::{
ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT,
@@ -594,3 +595,103 @@ pub fn send_drag_synthesized(
Ok(())
}
+
+/// Standard wheel notch delta. A `mouseData` value of `±WHEEL_DELTA` is one
+/// detent of the physical mouse wheel.
+const WHEEL_DELTA: i32 = 120;
+
+/// Compute the `MOUSEINPUT::mouseData` value for a wheel event of `ticks`
+/// detents. Positive ticks = wheel forward/up (vertical) or right (horizontal);
+/// negative = down / left. `mouseData` is a `u32` field carrying a signed
+/// 32-bit delta, so we compute as `i32` then bit-cast to `u32` (this is what
+/// the Win32 docs mean by "the value is a multiple of WHEEL_DELTA").
+///
+/// Factored out of [`send_wheel_synthesized`] so the sign/magnitude encoding is
+/// unit-testable without a live display / `SendInput`.
+fn wheel_mouse_data(ticks: i32) -> u32 {
+ (WHEEL_DELTA * ticks) as u32
+}
+
+/// Synthesize a single mouse-wheel event at screen coordinates `(sx, sy)` via
+/// `SendInput`.
+///
+/// The OS routes wheel input to the window **under the cursor**, not the
+/// foreground window, so we `SetCursorPos(sx, sy)` first to place the wheel
+/// over the intended target. `ticks` encodes both magnitude and direction:
+/// positive scrolls up (vertical) / right (horizontal), negative scrolls down /
+/// left — matching the `MOUSEEVENTF_WHEEL` / `MOUSEEVENTF_HWHEEL` convention
+/// where `mouseData = WHEEL_DELTA * ticks`.
+///
+/// Unlike [`send_click_synthesized`] this does NOT do a foreground swap: wheel
+/// delivery follows the cursor, so positioning the cursor is sufficient. The
+/// cursor is restored to its previous position afterward.
+pub fn send_wheel_synthesized(sx: i32, sy: i32, ticks: i32, horizontal: bool) -> Result<()> {
+ let flag = if horizontal { MOUSEEVENTF_HWHEEL } else { MOUSEEVENTF_WHEEL };
+ let mouse_data = wheel_mouse_data(ticks);
+
+ let wheel_input = INPUT {
+ r#type: INPUT_MOUSE,
+ Anonymous: INPUT_0 {
+ mi: MOUSEINPUT {
+ dx: 0,
+ dy: 0,
+ mouseData: mouse_data,
+ dwFlags: flag,
+ time: 0,
+ dwExtraInfo: 0,
+ },
+ },
+ };
+
+ unsafe {
+ let mut prev_cursor = POINT::default();
+ let _ = GetCursorPos(&mut prev_cursor);
+
+ // Wheel routes to the window under the cursor — place it on the target.
+ let _ = SetCursorPos(sx, sy);
+
+ let events = [wheel_input];
+ let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);
+ if sent as usize != events.len() {
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ bail!("SendInput inserted {sent}/{} wheel events", events.len());
+ }
+
+ // Brief settle, then restore the cursor.
+ sleep(Duration::from_millis(20));
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ }
+
+ Ok(())
+}
+
+#[cfg(test)]
+mod wheel_tests {
+ use super::{wheel_mouse_data, WHEEL_DELTA};
+
+ #[test]
+ fn wheel_data_up_is_positive_one_notch() {
+ // +1 tick (up / right) → +WHEEL_DELTA, bit-cast to u32.
+ assert_eq!(wheel_mouse_data(1), WHEEL_DELTA as u32);
+ assert_eq!(wheel_mouse_data(1), 120u32);
+ }
+
+ #[test]
+ fn wheel_data_down_is_negative_one_notch() {
+ // -1 tick (down / left) → -WHEEL_DELTA, bit-cast: 0xFFFFFF88.
+ assert_eq!(wheel_mouse_data(-1), (-WHEEL_DELTA) as u32);
+ assert_eq!(wheel_mouse_data(-1), 0xFFFF_FF88);
+ }
+
+ #[test]
+ fn wheel_data_scales_with_ticks() {
+ assert_eq!(wheel_mouse_data(3), (3 * WHEEL_DELTA) as u32);
+ assert_eq!(wheel_mouse_data(3), 360u32);
+ assert_eq!(wheel_mouse_data(-3) as i32, -360);
+ }
+
+ #[test]
+ fn wheel_data_zero_is_zero() {
+ assert_eq!(wheel_mouse_data(0), 0);
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 1e4249c6db..c930ad460e 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -154,16 +154,41 @@ pub(crate) fn resolve_cursor_key(args: &Value) -> String {
NO_CURSOR.to_owned()
}
+/// Returns `true` when a click/scroll invocation should take the **window-less
+/// screen-absolute** branch: the caller gave numeric `x` AND `y`, gave NO `pid`
+/// and NO `window_id`, and the effective `capture_scope` is `"desktop"`.
+///
+/// Pure + arg-shape agnostic (works for both click and scroll args) so it's
+/// unit-testable without Win32. When this returns `false` but `x,y` are present
+/// with no pid/window_id, the caller returns a `desktop_scope_disabled`
+/// structured error telling the agent to `set_config capture_scope=desktop`.
+fn is_windowless_desktop_action(args: &serde_json::Value, scope: &str) -> bool {
+ if scope != "desktop" {
+ return false;
+ }
+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);
+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);
+ if has_pid || has_window_id {
+ return false;
+ }
+ let has_num = |k: &str| args.get(k).map(|v| v.is_number()).unwrap_or(false);
+ has_num("x") && has_num("y")
+}
+
// ── DriverConfig + ResizeRegistry + ZoomRegistry ─────────────────────────────
#[derive(Clone)]
pub struct DriverConfig {
pub capture_mode: String,
+ /// Capture scope for vision loops: `"window"` (per-window, the default) or
+ /// `"desktop"` (full-display). Gates the window-less screen-absolute
+ /// click/scroll branches — those activate only under `"desktop"`.
+ pub capture_scope: String,
pub max_image_dimension: u32,
}
impl Default for DriverConfig {
- fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }
+ fn default() -> Self { Self { capture_mode: "som".into(), capture_scope: "window".into(), max_image_dimension: 1568 } }
}
/// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to
@@ -176,6 +201,9 @@ pub fn load_driver_config() -> DriverConfig {
if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) {
cfg.capture_mode = v;
}
+ if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) {
+ cfg.capture_scope = v;
+ }
if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) {
if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; }
}
@@ -1947,8 +1975,8 @@ impl Tool for ClickTool {
tool). The Swift-only `action` / `modifier` / `debug_image_out` schema \
fields aren't supported yet.".into(),
input_schema: json!({
- "type":"object","required":["pid"],"properties":{
- "pid":{"type":"integer","description":"Target process ID."},
+ "type":"object","properties":{
+ "pid":{"type":"integer","description":"Target process ID. Required UNLESS using window-less desktop-scope clicks: with capture_scope=\"desktop\" and no pid/window_id, x/y are treated as TRUE SCREEN pixels (call get_desktop_state first)."},
"window_id":{"type":"integer","description":"HWND for the window whose get_window_state produced the element_index. Required when element_index is used. Optional when element_token is supplied (the token carries it)."},
"element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."},
"element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
@@ -1969,6 +1997,78 @@ impl Tool for ClickTool {
use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error};
use crate::uia::cache::SnapshotKind;
let cursor_key = resolve_cursor_key(&args);
+
+ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────
+ // When the caller gives x,y with NO pid/window_id, treat x,y as TRUE
+ // SCREEN pixels. Gate on effective capture_scope: only "desktop" enables
+ // this; under "window" we return a structured `desktop_scope_disabled`
+ // error pointing the caller at set_config.
+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);
+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);
+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)
+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);
+ if !has_pid && !has_window_id && has_xy {
+ let scope = self.state.config.read().unwrap().capture_scope.clone();
+ if !is_windowless_desktop_action(&args, &scope) {
+ return ToolResult::error(
+ "click: x,y given with no pid/window_id, but capture_scope is \
+ \"window\". Screen-absolute clicks require desktop scope. Call \
+ set_config with capture_scope=desktop (and use get_desktop_state \
+ to pick coordinates), or pass a pid/window_id."
+ )
+ .with_structured(json!({
+ "code": "desktop_scope_disabled",
+ "capture_scope": scope,
+ "suggestion": "set_config capture_scope=desktop",
+ }));
+ }
+ // Resolve button (reuse the same validation as the pid path).
+ let button_raw = args.str_or("button", "left").to_lowercase();
+ if !matches!(button_raw.as_str(), "" | "left" | "right" | "middle") {
+ return ToolResult::error(format!(
+ "click: unknown button \"{button_raw}\" — expected one of left, right, middle."
+ ));
+ }
+ let button = if button_raw.is_empty() { "left".to_string() } else { button_raw };
+ let count = args.u64_or("count", 1) as usize;
+ let sx = args.f64_or("x", 0.0) as i32;
+ let sy = args.f64_or("y", 0.0) as i32;
+
+ // Animate the agent cursor to the screen point, then click.
+ overlay_glide_to(&cursor_key, sx as f64, sy as f64).await;
+ crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx as f64, y: sy as f64,
+ });
+
+ // Resolve the HWND that owns this screen pixel and click it via
+ // send_click_synthesized — it does the foreground-swap + UIPI checks
+ // on whatever owns the pixel, which is what lands Chromium-content
+ // clicks. WindowFromPoint walks to the leaf window at the point.
+ // (send_click_synthesized restores the previous foreground + cursor
+ // itself ~40ms after the click, so no extra restore guard here.)
+ let send_result = tokio::task::spawn_blocking(move || -> anyhow::Result<u64> {
+ use windows::Win32::Foundation::POINT;
+ use windows::Win32::UI::WindowsAndMessaging::WindowFromPoint;
+ let target = unsafe { WindowFromPoint(POINT { x: sx, y: sy }) };
+ if target.0.is_null() {
+ anyhow::bail!("No window under screen point ({sx},{sy}).");
+ }
+ let hwnd_u = target.0 as u64;
+ crate::input::send_click_synthesized(hwnd_u, sx, sy, count, &button)?;
+ Ok(hwnd_u)
+ }).await;
+ return match send_result {
+ Ok(Ok(hwnd_u)) => {
+ let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" };
+ ToolResult::text(format!(
+ "✅ Sent {click_word} via SendInput at screen ({sx},{sy}) on HWND 0x{hwnd_u:x} (desktop scope)."
+ ))
+ }
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
// Surface 6: element_token / element_index precedence resolution.
// Windows uses u64 HWND but the token registry stores u32; truncate
@@ -3318,7 +3418,9 @@ impl Tool for SetValueTool {
// ── scroll ────────────────────────────────────────────────────────────────────
-pub struct ScrollTool;
+pub struct ScrollTool {
+ state: Arc<ToolState>,
+}
static SCROLL_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
#[async_trait]
@@ -3339,12 +3441,14 @@ impl Tool for ScrollTool {
Note: `element_index` is accepted for cross-platform parity but currently \
no-op on Windows (UIA SetFocus not wired up yet — same caveat as `press_key`).".into(),
input_schema: json!({
- "type":"object","required":["pid","direction"],"properties":{
- "pid":{"type":"integer","description":"Target process ID."},
+ "type":"object","required":["direction"],"properties":{
+ "pid":{"type":"integer","description":"Target process ID. Required UNLESS using window-less desktop-scope scroll: with capture_scope=\"desktop\" and no pid/window_id, x/y are TRUE SCREEN pixels and the wheel routes to the window under that point."},
"direction":{"type":"string","enum":["up","down","left","right"]},
"by":{"type":"string","enum":["line","page"],"description":"Scroll granularity. Default: line."},
"amount":{"type":"integer","minimum":1,"maximum":50,
"description":"Number of scroll ticks. Default 3."},
+ "x":{"type":"number","description":"Screen-absolute X (desktop scope only) — wheel routes to the window under (x,y). Must be paired with y and no pid/window_id."},
+ "y":{"type":"number","description":"Screen-absolute Y (desktop scope only). Must be paired with x and no pid/window_id."},
"window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."},
"element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."},
"element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."},
@@ -3361,6 +3465,65 @@ impl Tool for ScrollTool {
// scroll-specific structured error (see commit 9e30d2cb). The
// dispatch enums are still in use just below.
use crate::input::dispatch::{DispatchMode, EventKind};
+ use cua_driver_core::tool_args::ArgsExt;
+ // `direction` is required in both the pid path and the window-less
+ // desktop path, so resolve it before the pid check.
+ let direction = match args.get("direction").and_then(|v| v.as_str()) {
+ Some(d) => d.to_owned(),
+ None => return ToolResult::error("Missing required string field direction."),
+ };
+ let amount = args.u64_or("amount", 3).clamp(1, 50) as u32;
+
+ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────
+ // No pid/window_id + numeric x,y + desktop scope → synthesize a wheel
+ // event at the screen point via SendInput. The wheel routes to whatever
+ // window is under (x,y). up/down map to a vertical wheel (sign), and
+ // left/right to a horizontal wheel; `amount` is the tick count.
+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);
+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);
+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)
+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);
+ if !has_pid && !has_window_id && has_xy {
+ let scope = self.state.config.read().unwrap().capture_scope.clone();
+ if !is_windowless_desktop_action(&args, &scope) {
+ return ToolResult::error(
+ "scroll: x,y given with no pid/window_id, but capture_scope is \
+ \"window\". Screen-absolute scroll requires desktop scope. Call \
+ set_config with capture_scope=desktop (and use get_desktop_state \
+ to pick coordinates), or pass a pid/window_id."
+ )
+ .with_structured(serde_json::json!({
+ "code": "desktop_scope_disabled",
+ "capture_scope": scope,
+ "suggestion": "set_config capture_scope=desktop",
+ }));
+ }
+ let sx = args.f64_or("x", 0.0) as i32;
+ let sy = args.f64_or("y", 0.0) as i32;
+ // Direction → (horizontal?, sign). Positive ticks = up / right.
+ let (horizontal, sign) = match direction.as_str() {
+ "up" => (false, 1),
+ "down" => (false, -1),
+ "right" => (true, 1),
+ "left" => (true, -1),
+ other => return ToolResult::error(format!(
+ "scroll: unknown direction \"{other}\" — expected up, down, left, right."
+ )),
+ };
+ let ticks = sign * amount as i32;
+ let dir_disp = direction.clone();
+ let result = tokio::task::spawn_blocking(move || {
+ crate::input::send_wheel_synthesized(sx, sy, ticks, horizontal)
+ }).await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Scrolled {dir_disp} via SendInput wheel ({amount} tick(s)) at screen ({sx},{sy}) (desktop scope)."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
// Swift error wording 1:1.
let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) => p,
@@ -3368,15 +3531,9 @@ impl Tool for ScrollTool {
};
let pid = raw_pid as u32;
let dispatch = DispatchMode::from_args(&args);
- let direction = match args.get("direction").and_then(|v| v.as_str()) {
- Some(d) => d.to_owned(),
- None => return ToolResult::error("Missing required string field direction."),
- };
- use cua_driver_core::tool_args::ArgsExt;
let by = args.str_or("by", "line");
let direction_display = direction.clone();
let by_display = by.clone();
- let amount = args.u64_or("amount", 3).clamp(1, 50) as u32;
// Surface 6: element_token / element_index precedence resolution.
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -4286,6 +4443,17 @@ impl Tool for DragTool {
// ── get_screen_size ───────────────────────────────────────────────────────────
+/// Read the primary display size in PHYSICAL pixels.
+///
+/// With permonitorv2 DPI awareness (set in cua-driver.manifest),
+/// `SM_CXSCREEN` / `SM_CYSCREEN` already return physical pixels — the same
+/// coordinate space screenshots and pixel clicks use on Windows. Shared by
+/// `get_screen_size` and `get_desktop_state`.
+fn physical_screen_size() -> (i32, i32) {
+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};
+ unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) }
+}
+
pub struct GetScreenSizeTool;
static GSS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
@@ -4302,13 +4470,12 @@ impl Tool for GetScreenSizeTool {
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
- use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};
use windows::Win32::UI::HiDpi::GetDpiForSystem;
// With permonitorv2 DPI awareness (set in cua-driver.manifest),
// SM_CXSCREEN/SM_CYSCREEN return PHYSICAL pixels — the same
// coordinate space screenshots and pixel clicks use on Windows.
// Report these as-is, along with the scale factor for reference.
- let (w, h) = unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) };
+ let (w, h) = physical_screen_size();
let dpi = unsafe { GetDpiForSystem() };
let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 };
ToolResult::text(format!("✅ Main display: {w}x{h} pixels @ {scale}x"))
@@ -4316,6 +4483,104 @@ impl Tool for GetScreenSizeTool {
}
}
+// ── get_desktop_state ─────────────────────────────────────────────────────────
+
+/// `get_desktop_state` — full-display vision screenshot (Windows).
+///
+/// Vision-only desktop capture: grabs the ENTIRE primary display at native
+/// physical-pixel size (no downscale) so screen-absolute pixel picks land
+/// exactly, then reports the true screen size. No UIA walk, no pid/window_id —
+/// this is the capture surface for `capture_scope="desktop"` GUI loops where
+/// the agent drives `click(x,y)` / `scroll(x,y)` against screen-absolute
+/// coordinates.
+///
+/// Mirrors the `get_window_state` vision branch's ToolResult shape: an
+/// `image_png` content part (or a written-out file path), a text summary line,
+/// and a `structuredContent` object.
+pub struct GetDesktopStateTool;
+static GDS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+
+#[async_trait]
+impl Tool for GetDesktopStateTool {
+ fn def(&self) -> &ToolDef {
+ GDS_DEF.get_or_init(|| ToolDef {
+ name: "get_desktop_state".into(),
+ description: "Capture a full-display vision screenshot in true screen pixels \
+ (no downscale), for capture_scope=\"desktop\" GUI loops where the agent then \
+ drives click(x,y)/scroll(x,y) with no pid/window_id. Returns the PNG at native \
+ display resolution plus the true screen size so screen-absolute pixel picks \
+ land exactly. Vision-only: no UIA tree walk.".into(),
+ input_schema: json!({
+ "type": "object",
+ "properties": {
+ "session": { "type": "string", "description": "Optional session id." },
+ "screenshot_out_file": { "type": "string", "description": "Write PNG here instead of base64." }
+ },
+ "additionalProperties": false
+ }),
+ read_only: true, destructive: false, idempotent: false, open_world: false,
+ })
+ }
+
+ async fn invoke(&self, args: Value) -> ToolResult {
+ use cua_driver_core::tool_args::ArgsExt;
+ use cua_driver_core::protocol::Content;
+
+ let screenshot_out_file = args.opt_str("screenshot_out_file");
+
+ // True screen geometry in physical pixels (same space as the capture).
+ let (screen_width, screen_height) = physical_screen_size();
+
+ // Capture the FULL display at native size — no resize. Run the
+ // blocking GDI capture off the async runtime.
+ let out_file = screenshot_out_file.clone();
+ let res = tokio::task::spawn_blocking(
+ move || -> anyhow::Result<(Option<String>, Option<String>, u32, u32)> {
+ use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
+ let png = crate::capture::screenshot_display_bytes()?;
+ let (w, h) = crate::capture::png_dimensions_pub(&png)?;
+ if let Some(ref path) = out_file {
+ std::fs::write(path, &png)?;
+ Ok((None, Some(path.clone()), w, h))
+ } else {
+ Ok((Some(BASE64.encode(&png)), None, w, h))
+ }
+ },
+ )
+ .await;
+
+ let (b64_opt, file_path, screenshot_width, screenshot_height) = match res {
+ Ok(Ok(v)) => v,
+ Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")),
+ Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")),
+ };
+
+ let mut content: Vec<Content> = Vec::new();
+ if let Some(b64) = b64_opt {
+ content.push(Content::image_png(b64));
+ }
+ let summary = format!(
+ "desktop screenshot {screenshot_width}x{screenshot_height} px \
+ (screen {screen_width}x{screen_height} px)"
+ );
+ content.push(Content::text(summary));
+
+ let mut structured = json!({
+ "platform": "windows",
+ "screenshot_width": screenshot_width,
+ "screenshot_height": screenshot_height,
+ "screen_width": screen_width,
+ "screen_height": screen_height,
+ "screenshot_mime_type": "image/png",
+ });
+ if let Some(ref fp) = file_path {
+ structured["screenshot_file_path"] = json!(fp);
+ }
+
+ ToolResult { content, is_error: None, structured_content: Some(structured) }
+ }
+}
+
// ── get_cursor_position ───────────────────────────────────────────────────────
pub struct GetCursorPositionTool;
@@ -4970,6 +5235,7 @@ impl Tool for GetConfigTool {
"version": env!("CARGO_PKG_VERSION"),
"platform": "windows",
"capture_mode": cfg.capture_mode,
+ "capture_scope": cfg.capture_scope,
"max_image_dimension": cfg.max_image_dimension,
"agent_cursor": { "enabled": cursor_enabled },
"experimental_pip": pip_enabled,
@@ -5013,6 +5279,7 @@ impl Tool for SetConfigTool {
"key":{"type":"string","description":"Dotted snake_case path to a leaf config field (Swift-compatible shape). Pair with `value`."},
"value":{"description":"New value for `key`. JSON type depends on the key."},
"capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Legacy per-field shape."},
+ "capture_scope":{"type":"string","enum":["window","desktop"],"description":"Capture scope: single window (default) or whole desktop. Desktop scope enables window-less screen-absolute click/scroll (no pid/window_id). Accepted in both the {key,value} and legacy per-field shapes."},
"max_image_dimension":{"type":"integer","description":"Legacy per-field shape."},
"experimental_pip":{"type":"boolean","description":"Legacy per-field shape. Enables PiP preview (applies next restart)."},
"experimental_pip_geometry":{"type":"string","description":"Legacy per-field shape. PiP window size + optional position."}
@@ -5039,6 +5306,17 @@ impl Tool for SetConfigTool {
}
None => return ToolResult::error(format!("`capture_mode` must be a string, got {val}.")),
},
+ "capture_scope" => match val.as_str() {
+ Some(s @ ("window" | "desktop")) => {
+ cfg.capture_scope = s.to_owned();
+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(s.to_owned())) {
+ tracing::warn!("set_config: failed to persist capture_scope: {e}");
+ }
+ applied = true;
+ }
+ Some(other) => return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{other}\".")),
+ None => return ToolResult::error(format!("`capture_scope` must be a string, got {val}.")),
+ },
"max_image_dimension" => match val.as_u64() {
Some(n) => {
cfg.max_image_dimension = n as u32;
@@ -5073,7 +5351,7 @@ impl Tool for SetConfigTool {
None => return ToolResult::error(format!("`experimental_pip_geometry` must be a string, got {val}.")),
},
other => return ToolResult::error(format!(
- "Unknown config key `{other}`. Known: capture_mode, max_image_dimension, experimental_pip, experimental_pip_geometry."
+ "Unknown config key `{other}`. Known: capture_mode, capture_scope, max_image_dimension, experimental_pip, experimental_pip_geometry."
)),
}
}
@@ -5085,6 +5363,16 @@ impl Tool for SetConfigTool {
}
applied = true;
}
+ if let Some(scope) = args.get("capture_scope").and_then(|v| v.as_str()) {
+ if !matches!(scope, "window" | "desktop") {
+ return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\"."));
+ }
+ cfg.capture_scope = scope.to_owned();
+ if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.to_owned())) {
+ tracing::warn!("set_config: failed to persist capture_scope: {e}");
+ }
+ applied = true;
+ }
if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) {
cfg.max_image_dimension = dim as u32;
if let Err(e) = pip_preview::write_config_key("max_image_dimension", Value::from(dim)) {
@@ -5124,6 +5412,7 @@ impl Tool for SetConfigTool {
"version": env!("CARGO_PKG_VERSION"),
"platform": "windows",
"capture_mode": cfg.capture_mode,
+ "capture_scope": cfg.capture_scope,
"max_image_dimension": cfg.max_image_dimension,
"agent_cursor": { "enabled": cursor_enabled },
"experimental_pip": pip_enabled,
@@ -5897,7 +6186,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
r.register(Box::new(PressKeyTool));
r.register(Box::new(HotkeyTool));
r.register(Box::new(SetValueTool { state: state.clone() }));
- r.register(Box::new(ScrollTool));
+ r.register(Box::new(ScrollTool { state: state.clone() }));
// `screenshot` / `ScreenshotCompatTool` removed from the tool surface
// — `get_window_state` with `capture_mode:"vision"` is the single
// canonical path for getting a window screenshot. Reasons:
@@ -5917,6 +6206,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
// depend on them via `Tool` trait reflection.
let _ = compat; // formerly drove the ScreenshotCompatTool branch
r.register(Box::new(GetScreenSizeTool));
+ r.register(Box::new(GetDesktopStateTool));
r.register(Box::new(GetCursorPositionTool));
r.register(Box::new(MoveCursorTool { state: state.clone() }));
r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() }));
@@ -6210,3 +6500,75 @@ mod click_button_schema_tests {
}
}
}
+
+#[cfg(test)]
+mod desktop_scope_tests {
+ use super::{is_windowless_desktop_action, DriverConfig, GetDesktopStateTool};
+ use cua_driver_core::tool::Tool;
+ use serde_json::json;
+
+ // ── is_windowless_desktop_action ──────────────────────────────────────────
+
+ #[test]
+ fn windowless_true_for_xy_under_desktop_scope_click_shape() {
+ // Click arg shape: {x, y}.
+ assert!(is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "desktop"));
+ }
+
+ #[test]
+ fn windowless_true_for_xy_under_desktop_scope_scroll_shape() {
+ // Scroll arg shape: {direction, x, y}.
+ assert!(is_windowless_desktop_action(
+ &json!({"direction": "down", "x": 10, "y": 20}),
+ "desktop"
+ ));
+ }
+
+ #[test]
+ fn windowless_false_when_pid_present() {
+ assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20, "pid": 5}), "desktop"));
+ }
+
+ #[test]
+ fn windowless_false_when_window_id_present() {
+ assert!(!is_windowless_desktop_action(
+ &json!({"x": 10, "y": 20, "window_id": 99}),
+ "desktop"
+ ));
+ }
+
+ #[test]
+ fn windowless_false_under_window_scope() {
+ assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "window"));
+ }
+
+ #[test]
+ fn windowless_false_when_xy_missing() {
+ assert!(!is_windowless_desktop_action(&json!({"x": 10}), "desktop"));
+ assert!(!is_windowless_desktop_action(&json!({"y": 20}), "desktop"));
+ assert!(!is_windowless_desktop_action(&json!({}), "desktop"));
+ // Non-numeric x/y must not qualify.
+ assert!(!is_windowless_desktop_action(&json!({"x": "10", "y": "20"}), "desktop"));
+ }
+
+ // ── DriverConfig default ──────────────────────────────────────────────────
+
+ #[test]
+ fn default_capture_scope_is_window() {
+ assert_eq!(DriverConfig::default().capture_scope, "window");
+ }
+
+ // ── get_desktop_state schema ──────────────────────────────────────────────
+
+ #[test]
+ fn get_desktop_state_schema_shape() {
+ let d = GetDesktopStateTool.def();
+ assert!(d.read_only, "get_desktop_state must be read_only");
+ let props = d.input_schema["properties"].as_object().unwrap();
+ assert!(!props.contains_key("pid"), "must not accept pid");
+ assert!(!props.contains_key("window_id"), "must not accept window_id");
+ assert!(props.contains_key("session"));
+ assert!(props.contains_key("screenshot_out_file"));
+ assert_eq!(d.input_schema["additionalProperties"], json!(false));
+ }
+}