Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): ask get_window_state for elements in in-repo callers (#4913)

Francesco Bonaccifbd031c0cd8c
diff --git a/apps/cua-spaces/scripts/golden/permission-approver.py b/apps/cua-spaces/scripts/golden/permission-approver.py
index 7dad3775b2..6d6e5a0b82 100644
--- a/apps/cua-spaces/scripts/golden/permission-approver.py
+++ b/apps/cua-spaces/scripts/golden/permission-approver.py
@@ -674,6 +674,10 @@ def main() -> int:
# cheap on Electron/Unity windows that slip the size
# gate.
"max_elements": 200,
+ # The buttons come from `elements` and the summary
+ # from `tree_markdown`; since cua-driver 0.35 a read
+ # returns only the markdown unless asked for both.
+ "tree_format": "both",
})
except OSError:
# window_id_not_found, or a window owned by a process we
diff --git a/docs/content/docs/cua-driver/guides/operate.mdx b/docs/content/docs/cua-driver/guides/operate.mdx
index d201c01135..494d9d7546 100644
--- a/docs/content/docs/cua-driver/guides/operate.mdx
+++ b/docs/content/docs/cua-driver/guides/operate.mdx
@@ -185,6 +185,35 @@ If pacman owns the executable, update with `sudo pacman -Syu`; Driver's own
update and channel commands refuse with pacman guidance (builds with
[#3636](https://github.com/trycua/cua/pull/3636)).
+### Before updating to 0.35
+
+0.35 changes three things that scripts and wrappers may depend on. The typed
+Rust, Python and TypeScript SDKs keep the full `get_window_state` response, so
+they are only affected by item 3.
+
+1. **`get_window_state` is lean by default**
+ ([#4743](https://github.com/trycua/cua/pull/4743)). A read returns only the
+ compact `tree_markdown`, walks at most 250 nodes, and leaves out `_note` and
+ `background_input`. If you read `structuredContent.elements`, pass
+ `tree_format: "elements"` (or `"both"` if you also read `tree_markdown`).
+ Pass `verbose: true` for `_note` and `background_input`, and `max_elements`
+ for a larger walk. `full_output: true` returns the 0.34 response unchanged.
+ A row `[N]` of a read whose `snapshot_id` is `s0000002a` has the
+ `element_token` `s0000002a:N`.
+2. **macOS reads collapse closed menus**
+ ([#4892](https://github.com/trycua/cua/pull/4892)). A default read lists
+ only the menu bar's titles, in `tree_markdown` and in `elements`. Run a menu
+ item with `invoke_menu` and its path, or find it with `query`;
+ `full_output: true` still walks every menu. The walk diagnostics
+ `nodes_visited`, `nodes_pending`, `walk_elapsed_ms` and
+ `screenshot_refreshed_snapshot` need `verbose: true`.
+3. **macOS `type_text` sends `\n`, `\r` and `\t` as Return and Tab**
+ ([#4821](https://github.com/trycua/cua/pull/4821)) when it types keys
+ (Chromium and Electron fields, `delivery_mode: "foreground"`, pixel
+ targets). Text with a line break now submits a form or an address bar, and
+ a tab moves focus. Remove them from text that should not submit, or write
+ the value with `set_value` where the field supports it.
+
### Before updating to token-only element targeting
Releases that include [#3873](https://github.com/trycua/cua/pull/3873) address
@@ -193,7 +222,8 @@ wrappers that act on elements before you update:
1. Pass `element_token` instead of `element_index` or `snapshot_id`. Read it
from `structuredContent.elements[].element_token` in the latest
- `get_window_state` result and send it with the window's `pid`. The token
+ `get_window_state` result (from 0.35, pass `tree_format: "elements"` to
+ get that array) and send it with the window's `pid`. The token
names its window, so `window_id` is optional; a different `window_id` is
refused with `conflicting_element_target`. `get_window_state` still returns
`element_index` and `snapshot_id` for display and correlation.
diff --git a/docs/scripts/verify-driver-first-run-linux.py b/docs/scripts/verify-driver-first-run-linux.py
index d1439c9b5b..94b4057400 100644
--- a/docs/scripts/verify-driver-first-run-linux.py
+++ b/docs/scripts/verify-driver-first-run-linux.py
@@ -72,6 +72,7 @@ class MCP:
def __init__(self, binary, endpoint, env, evidence):
self.evidence = evidence
self.sequence = 0
+ self.read_arguments = {}
self.lines = queue.Queue()
self.log = (evidence / "mcp-stderr.log").open("w")
self.process = subprocess.Popen(
@@ -176,7 +177,7 @@ def button(snapshot, labels):
def snapshot(mcp, target, label):
- state, images = mcp.call("get_window_state", target, label)
+ state, images = mcp.call("get_window_state", {**target, **mcp.read_arguments}, label)
require(images, f"{label} did not produce an actual snapshot image")
require(not state.get("degraded") and state.get("elements"),
f"{label} lacks usable AT-SPI controls")
@@ -323,6 +324,13 @@ def main():
save(evidence, "mcp-tools.json", listing)
require({"launch_app", "get_window_state", "click", "get_desktop_state"}.issubset(
{tool["name"] for tool in listing["tools"]}), "MCP discovery lacks required tools")
+ # The buttons and the display come from structured `elements`. From
+ # 0.35 a read returns them only with tree_format "elements"; earlier
+ # releases always return them and refuse the unknown argument.
+ read_schema = next(tool for tool in listing["tools"] if tool["name"] == "get_window_state")
+ mcp.read_arguments = ({"tree_format": "elements"}
+ if "tree_format" in read_schema.get("inputSchema", {}).get("properties", {})
+ else {})
mcp.call("get_desktop_state", {}, "before-launch-desktop")
before_launch = observe(evidence, "before-launch")
launch, _ = mcp.call("launch_app", {"name": "galculator"}, "launch-calculator")
diff --git a/libs/cua-driver/examples/jev-use/python/capture_window_state.py b/libs/cua-driver/examples/jev-use/python/capture_window_state.py
index e8ab82ee6e..80297c0028 100644
--- a/libs/cua-driver/examples/jev-use/python/capture_window_state.py
+++ b/libs/cua-driver/examples/jev-use/python/capture_window_state.py
@@ -59,6 +59,8 @@ async def capture(pid: int, title: str, source: str, max_depth: int | None = Non
"window_id": int(window["window_id"]),
"include_accessibility_tree": True,
"include_screenshot": True,
+ # Fixtures record the full response (elements + markdown).
+ "full_output": True,
**({"max_depth": max_depth} if max_depth is not None else {}),
},
)
diff --git a/libs/cua-driver/examples/jev-use/python/run_native.py b/libs/cua-driver/examples/jev-use/python/run_native.py
index 55c29737b6..4a47a3355d 100644
--- a/libs/cua-driver/examples/jev-use/python/run_native.py
+++ b/libs/cua-driver/examples/jev-use/python/run_native.py
@@ -97,11 +97,23 @@ async def observe(
"window_id": window_id,
"include_accessibility_tree": True,
"include_screenshot": True,
+ # The observation needs structured elements, the markdown tree and the
+ # completeness flags of the walk; cua-driver 0.35 returns the lean
+ # markdown-only read unless asked for the full response.
+ "full_output": True,
**task.scope.window_state_arguments(),
}
if timeout_ms is not None:
arguments["timeout_ms"] = timeout_ms
- payload = await driver.call("get_window_state", arguments)
+ try:
+ payload = await driver.call("get_window_state", arguments)
+ except DriverToolError as error:
+ # Drivers before 0.35 always return the full response and refuse the
+ # unknown argument.
+ if "unknown argument full_output" not in str(error):
+ raise
+ arguments.pop("full_output")
+ payload = await driver.call("get_window_state", arguments)
return NativeObservation.from_window_state(
payload, expected_pid=pid, expected_window_id=window_id
)
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_run_native_observe.py b/libs/cua-driver/examples/jev-use/python/tests/test_run_native_observe.py
new file mode 100644
index 0000000000..f1cceefaa5
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_run_native_observe.py
@@ -0,0 +1,56 @@
+"""observe() asks for the full get_window_state response, on any Driver."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from run import DriverToolError
+from run_native import observe
+
+FIXTURE = Path(__file__).resolve().parents[2] / "fixtures/native/appkit-window-state-initial-v1.json"
+
+
+class FakeDriver:
+ def __init__(self, refuse_full_output: bool) -> None:
+ self.refuse_full_output = refuse_full_output
+ self.calls: list[dict] = []
+
+ async def call(self, name, arguments):
+ self.calls.append(dict(arguments))
+ if self.refuse_full_output and "full_output" in arguments:
+ raise DriverToolError(
+ "get_window_state failed: [TextContent(type='text', "
+ "text='get_window_state: unknown argument full_output')]"
+ )
+ return json.loads(FIXTURE.read_text())
+
+
+TASK = SimpleNamespace(scope=SimpleNamespace(window_state_arguments=lambda: {}))
+
+
+class ObserveTests(unittest.TestCase):
+ def setUp(self) -> None:
+ payload = json.loads(FIXTURE.read_text())
+ self.pid, self.window_id = payload["pid"], payload["window_id"]
+
+ def test_asks_for_the_full_response(self) -> None:
+ driver = FakeDriver(refuse_full_output=False)
+ asyncio.run(observe(driver, TASK, self.pid, self.window_id))
+ self.assertEqual([call.get("full_output") for call in driver.calls], [True])
+
+ def test_retries_without_the_flag_on_drivers_before_0_35(self) -> None:
+ driver = FakeDriver(refuse_full_output=True)
+ observation = asyncio.run(observe(driver, TASK, self.pid, self.window_id))
+ self.assertTrue(observation.elements)
+ self.assertEqual(["full_output" in call for call in driver.calls], [True, False])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libs/cua-driver/examples/jev-use/typescript/run_native.ts b/libs/cua-driver/examples/jev-use/typescript/run_native.ts
index 644b3381e1..17b3b4f7eb 100644
--- a/libs/cua-driver/examples/jev-use/typescript/run_native.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/run_native.ts
@@ -119,14 +119,28 @@ async function observe(
windowId: number,
timeoutMs?: number
): Promise<NativeObservation> {
- const payload = await driver.call('get_window_state', {
+ const read = {
pid,
window_id: windowId,
include_accessibility_tree: true,
include_screenshot: true,
...windowStateArguments(task.scope),
...(timeoutMs !== undefined ? { timeout_ms: timeoutMs } : {}),
- });
+ };
+ let payload;
+ try {
+ // The observation needs structured elements, the markdown tree and the
+ // completeness flags; cua-driver 0.35 returns the lean markdown-only
+ // read unless asked for the full response.
+ payload = await driver.call('get_window_state', { ...read, full_output: true });
+ } catch (error) {
+ // Drivers before 0.35 always return the full response and refuse the
+ // unknown argument.
+ if (!(error instanceof DriverToolError && error.message.includes('unknown argument full_output'))) {
+ throw error;
+ }
+ payload = await driver.call('get_window_state', read);
+ }
return parseWindowState(payload, pid, windowId);
}
diff --git a/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.py b/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.py
index 632f43376b..8c30c68130 100644
--- a/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.py
+++ b/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.py
@@ -520,7 +520,10 @@ def observe(mcp, target, filename):
"""
attempts = []
for attempt in range(1, OBSERVATION_ATTEMPTS + 1):
- result = content(mcp.tool('get_window_state', {**target, 'timeout_ms': OBSERVATION_TIMEOUT_MS}))
+ # full_output: grounding needs structured elements from a complete
+ # walk, not the lean 250-node markdown read (cua-driver 0.35).
+ result = content(mcp.tool('get_window_state', {**target, 'timeout_ms': OBSERVATION_TIMEOUT_MS,
+ 'full_output': True}))
assert filename in result.get('window_title', ''), 'snapshot is not the synthetic document'
retryable = identity_unproven_only(result)
if not retryable:
diff --git a/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.py b/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.py
index 50180ccaca..82e4fcaa1f 100644
--- a/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.py
+++ b/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.py
@@ -669,7 +669,8 @@ def run(args):
assert len(matches) == 1 and matches[0].get('window_id') == target['window_id'], \
'reviewed PID/window identity is stale or ambiguous'
result = mcp.tool('get_window_state', {**target,
- **({'timeout_ms': OBSERVATION_TIMEOUT_MS} if full else {'max_elements': 100, 'max_depth': 6}),
+ **({'timeout_ms': OBSERVATION_TIMEOUT_MS, 'full_output': True} if full
+ else {'max_elements': 100, 'max_depth': 6}),
**({'session': session} if session else {})})
assert not result.get('isError'), result
content = result['structuredContent']
diff --git a/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.py b/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.py
index 49c687d9e8..cf2578706f 100644
--- a/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.py
+++ b/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.py
@@ -373,7 +373,8 @@ class InputTests(unittest.TestCase):
['get_window_state', 'list_windows', 'type_text', 'get_window_state', 'list_windows'])
self.assertEqual(calls[2].args[1], {**TARGET, 'text': 'abc', 'delivery_mode': 'background'})
for call in (calls[0], calls[3]):
- self.assertEqual(call.args[1], {**TARGET, 'timeout_ms': OBSERVATION_TIMEOUT_MS})
+ self.assertEqual(call.args[1], {**TARGET, 'timeout_ms': OBSERVATION_TIMEOUT_MS,
+ 'full_output': True})
self.assertEqual(OBSERVATION_TIMEOUT_MS, 15000)
def test_truncated_or_unproven_tree_sends_no_input(self):
@@ -582,7 +583,7 @@ class ObjectsSetupTests(unittest.TestCase):
mcp.tool.side_effect = observed(INKSCAPE)
observe(mcp, TARGET, 'cua-smoke-inkscape.svg')
self.assertEqual(mcp.tool.call_args_list[0].args,
- ('get_window_state', {**TARGET, 'timeout_ms': 15000}))
+ ('get_window_state', {**TARGET, 'timeout_ms': 15000, 'full_output': True}))
UNPROVEN = {'degraded': True,
diff --git a/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swift b/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swift
index 3fa85382f2..4bd4118cd2 100644
--- a/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swift
+++ b/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swift
@@ -158,7 +158,9 @@ let windows = launched["windows"] as? [[String: Any]] ?? []
let windowId = windows.first?["window_id"] as? Int ?? 0
log("launch_app(Finder): pid: \(pid), windows: \(windows.count)")
-let state = call("get_window_state", ["pid": pid, "window_id": windowId])
+// `tree_format: "elements"` asks for the structured element array; the
+// default read returns only the compact markdown tree.
+let state = call("get_window_state", ["pid": pid, "window_id": windowId, "tree_format": "elements"])
let images = (state["content"] as? [[String: Any]] ?? [])
.filter { $0["type"] as? String == "image" }
let hasTree = (state["structuredContent"] as? [String: Any])?["elements"] != nil
diff --git a/libs/cua-s1/python/src/cua_s1/driver.py b/libs/cua-s1/python/src/cua_s1/driver.py
index 5031cef185..dcf399639b 100644
--- a/libs/cua-s1/python/src/cua_s1/driver.py
+++ b/libs/cua-s1/python/src/cua_s1/driver.py
@@ -215,14 +215,24 @@ class BaseDriver:
return windows[0]
def window_state(self, target: WindowTarget) -> WindowSnapshot:
- state = self.call(
- "get_window_state",
- pid=target.pid,
- window_id=target.window_id,
- session=self.session,
- include_accessibility_tree=True,
- include_screenshot=False,
- )
+ read = {
+ "pid": target.pid,
+ "window_id": target.window_id,
+ "session": self.session,
+ "include_accessibility_tree": True,
+ "include_screenshot": False,
+ }
+ try:
+ # Since cua-driver 0.35 the default read is lean (markdown only,
+ # 250 nodes). Actions need the structured elements of a complete
+ # walk, so ask for the full response.
+ state = self.call("get_window_state", **read, full_output=True)
+ except DriverError as error:
+ # Drivers before 0.35 always return the full response and refuse
+ # the unknown argument.
+ if "unknown argument full_output" not in repr(error.details):
+ raise
+ state = self.call("get_window_state", **read)
raw_elements = state.get("elements")
if not isinstance(raw_elements, list):
raise DriverError(
diff --git a/libs/cua-s1/python/tests/test_driver.py b/libs/cua-s1/python/tests/test_driver.py
index 930b8e833b..1809751a64 100644
--- a/libs/cua-s1/python/tests/test_driver.py
+++ b/libs/cua-s1/python/tests/test_driver.py
@@ -122,3 +122,46 @@ def test_click_payload_and_effect_expectations_conform_to_portable_manifest():
"refused",
}
assert all(tool["name"] != "set_value" for tool in manifest["tools"])
+
+
+def test_window_state_asks_for_the_full_response():
+ """cua-driver 0.35 returns lean markdown by default; s1 needs elements."""
+
+ calls = []
+
+ class RecordingDriver(BaseDriver):
+ def call(self, tool, **args):
+ calls.append((tool, args))
+ return {
+ "snapshot_id": "s1",
+ "elements_complete": True,
+ "elements": [],
+ "tree_markdown": "",
+ }
+
+ snapshot = RecordingDriver(session="test").window_state(WindowTarget(pid=10, window_id=20))
+ assert snapshot.elements_complete is True
+ assert calls[0][0] == "get_window_state"
+ assert calls[0][1]["full_output"] is True
+
+
+def test_window_state_falls_back_for_drivers_before_0_35():
+ calls = []
+
+ class OldDriver(BaseDriver):
+ def call(self, tool, **args):
+ calls.append(args)
+ if "full_output" in args:
+ raise DriverError(
+ "driver_tool_refused",
+ "refused",
+ tool=tool,
+ details={
+ "response": {"error": "get_window_state: unknown argument full_output"}
+ },
+ )
+ return {"snapshot_id": "s1", "elements_complete": True, "elements": []}
+
+ snapshot = OldDriver(session="test").window_state(WindowTarget(pid=10, window_id=20))
+ assert snapshot.snapshot_id == "s1"
+ assert [("full_output" in args) for args in calls] == [True, False]
diff --git a/libs/images/macos/tools/ax_probe.py b/libs/images/macos/tools/ax_probe.py
index 1566ea387d..8b62760aee 100644
--- a/libs/images/macos/tools/ax_probe.py
+++ b/libs/images/macos/tools/ax_probe.py
@@ -77,6 +77,13 @@ def normalize(text: str) -> str:
return "".join(ch for ch in text if ch.isalnum() or ch in ".-")
+def read(base: dict[str, Any]) -> dict[str, Any]:
+ """get_window_state arguments for `base`. Keys come from `elements` and the
+ display from `tree_markdown`; since cua-driver 0.35 a read returns only
+ the markdown unless asked for both."""
+ return {**base, "tree_format": "both"}
+
+
def display_text(state: dict[str, Any]) -> str:
"""The window's static texts (Calculator's display), from the tree
rendering: `elements` lists only actionable nodes."""
@@ -121,7 +128,7 @@ async def run(url: str, token: str, log: list[dict[str, Any]]) -> bool:
for w in candidates:
base = {"pid": pid, "window_id": w["window_id"]}
origin = (w.get("bounds", {}).get("x", 0), w.get("bounds", {}).get("y", 0))
- state = await call("get_window_state", {**base, "include_screenshot": False})
+ state = await call("get_window_state", {**read(base), "include_screenshot": False})
elements = state.get("elements", [])
log[-1]["elements"] = len(elements)
if elements:
@@ -141,16 +148,16 @@ async def run(url: str, token: str, log: list[dict[str, Any]]) -> bool:
return {"x": f["x"] + f["w"] / 2 - origin[0], "y": f["y"] + f["h"] / 2 - origin[1]}
clear = find_key(elements, ("All Clear", "Clear", "AC", "C"))
- state = await call("get_window_state", base)
+ state = await call("get_window_state", read(base))
if clear:
await call("click", {**base, **target(clear)})
- state = await call("get_window_state", base)
+ state = await call("get_window_state", read(base))
for key in ("7", "+", "2", "="):
element = find_key(state.get("elements", []), KEYS[key])
if element is None:
raise RuntimeError(f"no {key!r} key in the AX tree")
await call("click", {**base, **target(element)})
- state = await call("get_window_state", base)
+ state = await call("get_window_state", read(base))
shown = display_text(state)
log.append({"display": shown})
calculated = any(normalize(t) == "9" for t in shown.split(" | "))
diff --git a/samples/driver/multi-cursor/orchestrator/src/main.rs b/samples/driver/multi-cursor/orchestrator/src/main.rs
index 81c46b5684..00a764357b 100644
--- a/samples/driver/multi-cursor/orchestrator/src/main.rs
+++ b/samples/driver/multi-cursor/orchestrator/src/main.rs
@@ -247,12 +247,15 @@ fn main() {
thread::sleep(Duration::from_millis(800)); // let resized layouts settle
// Discover UIA elements per corner (field + SUBMIT). GDI has none -> pixel.
+ // Reads pass tree_format "both": rows are found in the markdown and their
+ // element_token comes from `elements` (cua-driver 0.35 returns only the
+ // markdown by default).
for c in corners.iter_mut() {
let tree = run_call_out(
&cua,
"get_window_state",
&format!(
- r#"{{"pid":{},"window_id":{},"capture_mode":"ax","session":"{}"}}"#,
+ r#"{{"pid":{},"window_id":{},"capture_mode":"ax","tree_format":"both","session":"{}"}}"#,
c.pid, c.hwnd.0 as isize, c.session
),
);
@@ -293,7 +296,7 @@ fn main() {
let title = c.title.to_string();
handles.push(thread::spawn(move || {
let hwnd = HWND(hwnd_addr as *mut _);
- let ax = format!(r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"ax","session":"{session}"}}"#);
+ let ax = format!(r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"ax","tree_format":"both","session":"{session}"}}"#);
let mut seen_records: i64 = -1;
for act in rx {
// Re-read the window before each action: element tokens are