Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): extend first Linux snapshot budget (#4375)

Iflaq khurshid920a42f1043a
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/window-state.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/window-state.mdx
index 4a85e27086..10d3172128 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/window-state.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/window-state.mdx
@@ -72,7 +72,7 @@ The mirror image: pass `include_accessibility_tree:false` to SKIP the AT-SPI wal
Optional `max_elements` / `max_depth` bound the AT-SPI walk to mitigate context-window blow-up on Electron / large web apps that produce 10k+ element trees. When applied, BOTH the markdown and the structured elements are truncated identically. Omit both for current default behaviour.
-TIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI walk. Large apps (LibreOffice, GIMP, file managers) can exceed it; the call then returns the PARTIAL tree with `truncated: true`, `truncation_reason`, `nodes_visited`/`nodes_pending` and `elements_complete: false`. Every element listed is real and clickable; elements after the cut are simply missing. Retry with a larger `timeout_ms` (e.g. 5000) or narrow with `query`/`max_depth` when the element you need is absent.
+TIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI walk. When omitted, a window's first snapshot gets a 2000 ms cold-start budget so Chromium/Electron can finish publishing its initial tree. Explicit values are always honored. Large apps (LibreOffice, GIMP, file managers) can exceed the budget; the call then returns the PARTIAL tree with `truncated: true`, `truncation_reason`, `nodes_visited`/`nodes_pending` and `elements_complete: false`. Every element listed is real and clickable; elements after the cut are simply missing. Retry with a larger `timeout_ms` (e.g. 5000) or narrow with `query`/`max_depth` when the element you need is absent.
SCREENSHOT SCALE: the screenshot is delivered at or below 1.15 megapixels (long edge <= max_image_dimension, 1568 by default), because larger images are downsized before a model reads them and its pixel coordinates would then be uniformly short. Element `screenshot_frame`s and x/y for the pointer tools are pixels of the delivered screenshot (`frame` stays in screen coordinates, as on every platform); `frame_scale` < 1 reports the downsizing. An explicit per-call `max_image_dimension` (0 = native) replaces this cap.
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs
index 85f9af055e..b30a5573d7 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs
@@ -27,6 +27,7 @@ struct Snapshot<S> {
screenshot_owner: Option<String>,
screenshot_scale: Option<f64>,
zoom: Option<ZoomContext>,
+ semantic: bool,
payload: S,
}
@@ -150,6 +151,31 @@ impl<S: SnapshotPayload> SnapshotStore<S> {
payload: S,
session: Option<&str>,
screenshot_scale: Option<f64>,
+ ) -> Option<(u32, Vec<u32>)> {
+ self.publish_snapshot(pid, window_id, payload, session, screenshot_scale, true)
+ }
+
+ /// Publish screenshot/capture state without claiming that an accessibility
+ /// walk has completed for this window.
+ pub fn publish_capture_for_session(
+ &self,
+ pid: i32,
+ window_id: u64,
+ payload: S,
+ session: Option<&str>,
+ screenshot_scale: Option<f64>,
+ ) -> Option<(u32, Vec<u32>)> {
+ self.publish_snapshot(pid, window_id, payload, session, screenshot_scale, false)
+ }
+
+ fn publish_snapshot(
+ &self,
+ pid: i32,
+ window_id: u64,
+ payload: S,
+ session: Option<&str>,
+ screenshot_scale: Option<f64>,
+ semantic: bool,
) -> Option<(u32, Vec<u32>)> {
let (id, retired) = {
let mut inner = self.inner.lock().unwrap();
@@ -171,6 +197,7 @@ impl<S: SnapshotPayload> SnapshotStore<S> {
screenshot_owner: session.map(str::to_owned),
screenshot_scale,
zoom: None,
+ semantic,
payload,
});
(id, retired)
@@ -323,6 +350,24 @@ impl<S: SnapshotPayload> SnapshotStore<S> {
.map(|entry| entry.window_id)
}
+ /// Whether this runtime has already published any snapshot for the window.
+ pub fn contains_window(&self, pid: i32, window_id: u64) -> bool {
+ self.inner
+ .lock()
+ .unwrap()
+ .get(&pid)
+ .is_some_and(|lane| lane.iter().any(|entry| entry.window_id == window_id))
+ }
+
+ /// Whether an accessibility/semantic snapshot has been published for the
+ /// window. Screenshot-only previews deliberately do not satisfy this.
+ pub fn contains_semantic_window(&self, pid: i32, window_id: u64) -> bool {
+ self.inner.lock().unwrap().get(&pid).is_some_and(|lane| {
+ lane.iter()
+ .any(|entry| entry.window_id == window_id && entry.semantic)
+ })
+ }
+
pub fn resolve(
&self,
pid: i32,
@@ -525,6 +570,28 @@ mod tests {
.is_err());
}
+ #[test]
+ fn semantic_membership_ignores_capture_only_publication() {
+ let cache = SnapshotStore::new();
+ assert!(!cache.contains_window(9, 99));
+ assert!(!cache.contains_semantic_window(9, 99));
+
+ cache
+ .publish_capture_for_session(9, 99, Payload(vec![]), None, Some(1.0))
+ .unwrap();
+ assert!(cache.contains_window(9, 99));
+ assert!(!cache.contains_semantic_window(9, 99));
+
+ cache.publish(9, 99, Payload(vec![]));
+ assert!(cache.contains_semantic_window(9, 99));
+ assert!(!cache.contains_semantic_window(9, 100));
+ assert!(!cache.contains_semantic_window(10, 99));
+
+ cache.remove(9, 99);
+ assert!(!cache.contains_window(9, 99));
+ assert!(!cache.contains_semantic_window(9, 99));
+ }
+
fn token_refusal(cache: &SnapshotStore<Payload>, pid: i32, token: &str) -> serde_json::Value {
cache
.resolve(pid, &serde_json::json!({ "element_token": token }))
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs
index b43a2b5aba..e9420949ec 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs
@@ -153,6 +153,21 @@ pub fn resolve_timeout_ms(value: Option<&Value>) -> u64 {
.unwrap_or(TIMEOUT_MS_DEFAULT)
}
+/// Resolve a walk timeout while allowing a platform to grant an omitted
+/// timeout a larger budget until that window has produced its first snapshot.
+/// Explicit caller values always retain the shared clamp semantics.
+pub fn resolve_timeout_ms_with_first_snapshot_grace(
+ value: Option<&Value>,
+ has_prior_snapshot: bool,
+ first_snapshot_default: u64,
+) -> u64 {
+ if value.is_none() && !has_prior_snapshot {
+ first_snapshot_default.clamp(TIMEOUT_MS_MIN, TIMEOUT_MS_MAX)
+ } else {
+ resolve_timeout_ms(value)
+ }
+}
+
// ── The gate ─────────────────────────────────────────────────────────────────
/// The canonical *shape* of each shared param (description stripped). `None` for
@@ -313,6 +328,22 @@ mod tests {
.is_empty());
}
+ #[test]
+ fn first_snapshot_grace_never_overrides_an_explicit_timeout() {
+ assert_eq!(
+ resolve_timeout_ms_with_first_snapshot_grace(None, false, 2_000),
+ 2_000
+ );
+ assert_eq!(
+ resolve_timeout_ms_with_first_snapshot_grace(None, true, 2_000),
+ TIMEOUT_MS_DEFAULT
+ );
+ assert_eq!(
+ resolve_timeout_ms_with_first_snapshot_grace(Some(&json!(750)), false, 2_000),
+ 750
+ );
+ }
+
#[test]
fn structural_strips_description_keeps_type_and_enum() {
let with_prose = json!({ "type": "string", "enum": ["a", "b"], "description": "x" });
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 45db4e46de..d5d3855a8e 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -995,6 +995,16 @@ pub struct GetWindowStateTool {
state: Arc<ToolState>,
}
+const COLD_START_WALK_TIMEOUT_MS: u64 = 2_000;
+
+fn linux_snapshot_timeout_ms(timeout: Option<&Value>, has_prior_snapshot: bool) -> u64 {
+ cua_driver_core::tool_schema::resolve_timeout_ms_with_first_snapshot_grace(
+ timeout,
+ has_prior_snapshot,
+ COLD_START_WALK_TIMEOUT_MS,
+ )
+}
+
static GWS_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
#[async_trait]
@@ -1043,7 +1053,10 @@ impl Tool for GetWindowStateTool {
the markdown and the structured elements are truncated \
identically. Omit both for current default behaviour.\n\n\
TIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI \
- walk. Large apps (LibreOffice, GIMP, file managers) can exceed it; \
+ walk. When omitted, a window's first snapshot gets a 2000 ms \
+ cold-start budget so Chromium/Electron can finish publishing its \
+ initial tree. Explicit values are always honored. Large apps \
+ (LibreOffice, GIMP, file managers) can exceed the budget; \
the call then returns the PARTIAL tree with `truncated: true`, \
`truncation_reason`, `nodes_visited`/`nodes_pending` and \
`elements_complete: false`. Every element listed is real and \
@@ -1197,7 +1210,12 @@ impl Tool for GetWindowStateTool {
.get("max_depth")
.and_then(|v| v.as_u64())
.map(|v| v.max(1) as usize);
- let timeout_ms = cua_driver_core::tool_schema::resolve_timeout_ms(args.get("timeout_ms"));
+ let timeout_ms = linux_snapshot_timeout_ms(
+ args.get("timeout_ms"),
+ self.state
+ .snapshots
+ .contains_semantic_window(pid as i32, xid),
+ );
let walk_timeout = std::time::Duration::from_millis(timeout_ms);
let process_is_live = crate::proc_fs::is_process_live(pid);
@@ -1606,7 +1624,7 @@ impl Tool for GetWindowStateTool {
}
if !observation_only && !published_snapshot && screenshot_scale.is_some() {
- if let Some((_, replaced)) = state.snapshots.publish_for_session(
+ if let Some((_, replaced)) = state.snapshots.publish_capture_for_session(
pid as i32,
xid,
crate::atspi::snapshot::AtspiSnapshot::from_nodes(&[]),
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
index d9b74f5575..de6251f684 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
@@ -43,7 +43,7 @@
"read_only": true
},
{
- "description": "Walk a running app's AT-SPI tree and return BOTH a structured `elements` array (preferred) AND a Markdown rendering of the same tree (back-compat). Every actionable element is tagged with [element_index N] in the markdown and as `element_index` in the structured array; pass each element's `element_token` to `click`, `type_text`, `set_value`, etc.\n\nPREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value`, `enabled`, `selected`, `actions` (names of AT-SPI actions exposed by the element, omitted when empty), `frame: {x,y,w,h}` when AT-SPI reports usable bounds, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side. Set `query` to project BOTH representations to matching rows plus their ancestor chain while preserving original indices. `total_element_count` reports the complete snapshot and `returned_element_count` reports the projection.\n\nAlways returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces). Choose the modality at ACTION time: an element ax action (element_token → accessibility rung) or an element px action (x,y → pixel rung off this screenshot). capture_mode is deprecated and ignored. On Wayland, where output capture cannot prove the requested surface's identity, the truthful tree is returned without a screenshot and `screenshot_error.code` is `surface_identity_unproven`.\n\nThe mirror image: pass `include_accessibility_tree:false` to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Setting BOTH `include_accessibility_tree:false` and `include_screenshot:false` is an error. Optional `max_dimension` caps the returned screenshot's long edge in pixels for a cheap thumbnail.\n\nOptional `max_elements` / `max_depth` bound the AT-SPI walk to mitigate context-window blow-up on Electron / large web apps that produce 10k+ element trees. When applied, BOTH the markdown and the structured elements are truncated identically. Omit both for current default behaviour.\n\nTIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI walk. Large apps (LibreOffice, GIMP, file managers) can exceed it; the call then returns the PARTIAL tree with `truncated: true`, `truncation_reason`, `nodes_visited`/`nodes_pending` and `elements_complete: false`. Every element listed is real and clickable; elements after the cut are simply missing. Retry with a larger `timeout_ms` (e.g. 5000) or narrow with `query`/`max_depth` when the element you need is absent.\n\nSCREENSHOT SCALE: the screenshot is delivered at or below 1.15 megapixels (long edge <= max_image_dimension, 1568 by default), because larger images are downsized before a model reads them and its pixel coordinates would then be uniformly short. Element `screenshot_frame`s and x/y for the pointer tools are pixels of the delivered screenshot (`frame` stays in screen coordinates, as on every platform); `frame_scale` < 1 reports the downsizing. An explicit per-call `max_image_dimension` (0 = native) replaces this cap.\n\nPOPUP MENUS: a context menu / popover / combo list is an override-redirect window that list_windows never shows. A click or right_click that opened one names it in its result (`popup: {window_id, bounds, title}`); pass that window_id here to walk the popup's own AT-SPI toplevel so its menu items get element indices (then click them by element_token). Omitting window_id while a popup of this pid is open walks that popup.",
+ "description": "Walk a running app's AT-SPI tree and return BOTH a structured `elements` array (preferred) AND a Markdown rendering of the same tree (back-compat). Every actionable element is tagged with [element_index N] in the markdown and as `element_index` in the structured array; pass each element's `element_token` to `click`, `type_text`, `set_value`, etc.\n\nPREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value`, `enabled`, `selected`, `actions` (names of AT-SPI actions exposed by the element, omitted when empty), `frame: {x,y,w,h}` when AT-SPI reports usable bounds, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side. Set `query` to project BOTH representations to matching rows plus their ancestor chain while preserving original indices. `total_element_count` reports the complete snapshot and `returned_element_count` reports the projection.\n\nAlways returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces). Choose the modality at ACTION time: an element ax action (element_token → accessibility rung) or an element px action (x,y → pixel rung off this screenshot). capture_mode is deprecated and ignored. On Wayland, where output capture cannot prove the requested surface's identity, the truthful tree is returned without a screenshot and `screenshot_error.code` is `surface_identity_unproven`.\n\nThe mirror image: pass `include_accessibility_tree:false` to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Setting BOTH `include_accessibility_tree:false` and `include_screenshot:false` is an error. Optional `max_dimension` caps the returned screenshot's long edge in pixels for a cheap thumbnail.\n\nOptional `max_elements` / `max_depth` bound the AT-SPI walk to mitigate context-window blow-up on Electron / large web apps that produce 10k+ element trees. When applied, BOTH the markdown and the structured elements are truncated identically. Omit both for current default behaviour.\n\nTIME BUDGET: `timeout_ms` (default 1000) bounds the whole AT-SPI walk. When omitted, a window's first snapshot gets a 2000 ms cold-start budget so Chromium/Electron can finish publishing its initial tree. Explicit values are always honored. Large apps (LibreOffice, GIMP, file managers) can exceed the budget; the call then returns the PARTIAL tree with `truncated: true`, `truncation_reason`, `nodes_visited`/`nodes_pending` and `elements_complete: false`. Every element listed is real and clickable; elements after the cut are simply missing. Retry with a larger `timeout_ms` (e.g. 5000) or narrow with `query`/`max_depth` when the element you need is absent.\n\nSCREENSHOT SCALE: the screenshot is delivered at or below 1.15 megapixels (long edge <= max_image_dimension, 1568 by default), because larger images are downsized before a model reads them and its pixel coordinates would then be uniformly short. Element `screenshot_frame`s and x/y for the pointer tools are pixels of the delivered screenshot (`frame` stays in screen coordinates, as on every platform); `frame_scale` < 1 reports the downsizing. An explicit per-call `max_image_dimension` (0 = native) replaces this cap.\n\nPOPUP MENUS: a context menu / popover / combo list is an override-redirect window that list_windows never shows. A click or right_click that opened one names it in its result (`popup: {window_id, bounds, title}`); pass that window_id here to walk the popup's own AT-SPI toplevel so its menu items get element indices (then click them by element_token). Omitting window_id while a popup of this pid is open walks that popup.",
"destructive": false,
"idempotent": false,
"input_schema": {