Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): ask get_window_state for elements in in-repo callers (#4913)
Francesco Bonaccifbd031c0cd8c
diff --git a/apps/cua-spaces/scripts/golden/permission-approver.py b/apps/cua-spaces/scripts/golden/permission-approver.pyindex 7dad3775b2..6d6e5a0b82 100644--- a/apps/cua-spaces/scripts/golden/permission-approver.py+++ b/apps/cua-spaces/scripts/golden/permission-approver.py@@ -674,6 +674,10 @@ def main() -> int:# cheap on Electron/Unity windows that slip the size# gate."max_elements": 200,+ # The buttons come from `elements` and the summary+ # from `tree_markdown`; since cua-driver 0.35 a read+ # returns only the markdown unless asked for both.+ "tree_format": "both",})except OSError:# window_id_not_found, or a window owned by a process wediff --git a/docs/content/docs/cua-driver/guides/operate.mdx b/docs/content/docs/cua-driver/guides/operate.mdxindex d201c01135..494d9d7546 100644--- a/docs/content/docs/cua-driver/guides/operate.mdx+++ b/docs/content/docs/cua-driver/guides/operate.mdx@@ -185,6 +185,35 @@ If pacman owns the executable, update with `sudo pacman -Syu`; Driver's ownupdate and channel commands refuse with pacman guidance (builds with[#3636](https://github.com/trycua/cua/pull/3636)).+### Before updating to 0.35++0.35 changes three things that scripts and wrappers may depend on. The typed+Rust, Python and TypeScript SDKs keep the full `get_window_state` response, so+they are only affected by item 3.++1. **`get_window_state` is lean by default**+ ([#4743](https://github.com/trycua/cua/pull/4743)). A read returns only the+ compact `tree_markdown`, walks at most 250 nodes, and leaves out `_note` and+ `background_input`. If you read `structuredContent.elements`, pass+ `tree_format: "elements"` (or `"both"` if you also read `tree_markdown`).+ Pass `verbose: true` for `_note` and `background_input`, and `max_elements`+ for a larger walk. `full_output: true` returns the 0.34 response unchanged.+ A row `[N]` of a read whose `snapshot_id` is `s0000002a` has the+ `element_token` `s0000002a:N`.+2. **macOS reads collapse closed menus**+ ([#4892](https://github.com/trycua/cua/pull/4892)). A default read lists+ only the menu bar's titles, in `tree_markdown` and in `elements`. Run a menu+ item with `invoke_menu` and its path, or find it with `query`;+ `full_output: true` still walks every menu. The walk diagnostics+ `nodes_visited`, `nodes_pending`, `walk_elapsed_ms` and+ `screenshot_refreshed_snapshot` need `verbose: true`.+3. **macOS `type_text` sends `\n`, `\r` and `\t` as Return and Tab**+ ([#4821](https://github.com/trycua/cua/pull/4821)) when it types keys+ (Chromium and Electron fields, `delivery_mode: "foreground"`, pixel+ targets). Text with a line break now submits a form or an address bar, and+ a tab moves focus. Remove them from text that should not submit, or write+ the value with `set_value` where the field supports it.+### Before updating to token-only element targetingReleases that include [#3873](https://github.com/trycua/cua/pull/3873) address@@ -193,7 +222,8 @@ wrappers that act on elements before you update:1. Pass `element_token` instead of `element_index` or `snapshot_id`. Read itfrom `structuredContent.elements[].element_token` in the latest- `get_window_state` result and send it with the window's `pid`. The token+ `get_window_state` result (from 0.35, pass `tree_format: "elements"` to+ get that array) and send it with the window's `pid`. The tokennames its window, so `window_id` is optional; a different `window_id` isrefused with `conflicting_element_target`. `get_window_state` still returns`element_index` and `snapshot_id` for display and correlation.diff --git a/docs/scripts/verify-driver-first-run-linux.py b/docs/scripts/verify-driver-first-run-linux.pyindex d1439c9b5b..94b4057400 100644--- a/docs/scripts/verify-driver-first-run-linux.py+++ b/docs/scripts/verify-driver-first-run-linux.py@@ -72,6 +72,7 @@ class MCP:def __init__(self, binary, endpoint, env, evidence):self.evidence = evidenceself.sequence = 0+ self.read_arguments = {}self.lines = queue.Queue()self.log = (evidence / "mcp-stderr.log").open("w")self.process = subprocess.Popen(@@ -176,7 +177,7 @@ def button(snapshot, labels):def snapshot(mcp, target, label):- state, images = mcp.call("get_window_state", target, label)+ state, images = mcp.call("get_window_state", {**target, **mcp.read_arguments}, label)require(images, f"{label} did not produce an actual snapshot image")require(not state.get("degraded") and state.get("elements"),f"{label} lacks usable AT-SPI controls")@@ -323,6 +324,13 @@ def main():save(evidence, "mcp-tools.json", listing)require({"launch_app", "get_window_state", "click", "get_desktop_state"}.issubset({tool["name"] for tool in listing["tools"]}), "MCP discovery lacks required tools")+ # The buttons and the display come from structured `elements`. From+ # 0.35 a read returns them only with tree_format "elements"; earlier+ # releases always return them and refuse the unknown argument.+ read_schema = next(tool for tool in listing["tools"] if tool["name"] == "get_window_state")+ mcp.read_arguments = ({"tree_format": "elements"}+ if "tree_format" in read_schema.get("inputSchema", {}).get("properties", {})+ else {})mcp.call("get_desktop_state", {}, "before-launch-desktop")before_launch = observe(evidence, "before-launch")launch, _ = mcp.call("launch_app", {"name": "galculator"}, "launch-calculator")diff --git a/libs/cua-driver/examples/jev-use/python/capture_window_state.py b/libs/cua-driver/examples/jev-use/python/capture_window_state.pyindex e8ab82ee6e..80297c0028 100644--- a/libs/cua-driver/examples/jev-use/python/capture_window_state.py+++ b/libs/cua-driver/examples/jev-use/python/capture_window_state.py@@ -59,6 +59,8 @@ async def capture(pid: int, title: str, source: str, max_depth: int | None = Non"window_id": int(window["window_id"]),"include_accessibility_tree": True,"include_screenshot": True,+ # Fixtures record the full response (elements + markdown).+ "full_output": True,**({"max_depth": max_depth} if max_depth is not None else {}),},)diff --git a/libs/cua-driver/examples/jev-use/python/run_native.py b/libs/cua-driver/examples/jev-use/python/run_native.pyindex 55c29737b6..4a47a3355d 100644--- a/libs/cua-driver/examples/jev-use/python/run_native.py+++ b/libs/cua-driver/examples/jev-use/python/run_native.py@@ -97,11 +97,23 @@ async def observe("window_id": window_id,"include_accessibility_tree": True,"include_screenshot": True,+ # The observation needs structured elements, the markdown tree and the+ # completeness flags of the walk; cua-driver 0.35 returns the lean+ # markdown-only read unless asked for the full response.+ "full_output": True,**task.scope.window_state_arguments(),}if timeout_ms is not None:arguments["timeout_ms"] = timeout_ms- payload = await driver.call("get_window_state", arguments)+ try:+ payload = await driver.call("get_window_state", arguments)+ except DriverToolError as error:+ # Drivers before 0.35 always return the full response and refuse the+ # unknown argument.+ if "unknown argument full_output" not in str(error):+ raise+ arguments.pop("full_output")+ payload = await driver.call("get_window_state", arguments)return NativeObservation.from_window_state(payload, expected_pid=pid, expected_window_id=window_id)diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_run_native_observe.py b/libs/cua-driver/examples/jev-use/python/tests/test_run_native_observe.pynew file mode 100644index 0000000000..f1cceefaa5--- /dev/null+++ b/libs/cua-driver/examples/jev-use/python/tests/test_run_native_observe.py@@ -0,0 +1,56 @@+"""observe() asks for the full get_window_state response, on any Driver."""++from __future__ import annotations++import asyncio+import json+import sys+import unittest+from pathlib import Path+from types import SimpleNamespace++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from run import DriverToolError+from run_native import observe++FIXTURE = Path(__file__).resolve().parents[2] / "fixtures/native/appkit-window-state-initial-v1.json"+++class FakeDriver:+ def __init__(self, refuse_full_output: bool) -> None:+ self.refuse_full_output = refuse_full_output+ self.calls: list[dict] = []++ async def call(self, name, arguments):+ self.calls.append(dict(arguments))+ if self.refuse_full_output and "full_output" in arguments:+ raise DriverToolError(+ "get_window_state failed: [TextContent(type='text', "+ "text='get_window_state: unknown argument full_output')]"+ )+ return json.loads(FIXTURE.read_text())+++TASK = SimpleNamespace(scope=SimpleNamespace(window_state_arguments=lambda: {}))+++class ObserveTests(unittest.TestCase):+ def setUp(self) -> None:+ payload = json.loads(FIXTURE.read_text())+ self.pid, self.window_id = payload["pid"], payload["window_id"]++ def test_asks_for_the_full_response(self) -> None:+ driver = FakeDriver(refuse_full_output=False)+ asyncio.run(observe(driver, TASK, self.pid, self.window_id))+ self.assertEqual([call.get("full_output") for call in driver.calls], [True])++ def test_retries_without_the_flag_on_drivers_before_0_35(self) -> None:+ driver = FakeDriver(refuse_full_output=True)+ observation = asyncio.run(observe(driver, TASK, self.pid, self.window_id))+ self.assertTrue(observation.elements)+ self.assertEqual(["full_output" in call for call in driver.calls], [True, False])+++if __name__ == "__main__":+ unittest.main()diff --git a/libs/cua-driver/examples/jev-use/typescript/run_native.ts b/libs/cua-driver/examples/jev-use/typescript/run_native.tsindex 644b3381e1..17b3b4f7eb 100644--- a/libs/cua-driver/examples/jev-use/typescript/run_native.ts+++ b/libs/cua-driver/examples/jev-use/typescript/run_native.ts@@ -119,14 +119,28 @@ async function observe(windowId: number,timeoutMs?: number): Promise<NativeObservation> {- const payload = await driver.call('get_window_state', {+ const read = {pid,window_id: windowId,include_accessibility_tree: true,include_screenshot: true,...windowStateArguments(task.scope),...(timeoutMs !== undefined ? { timeout_ms: timeoutMs } : {}),- });+ };+ let payload;+ try {+ // The observation needs structured elements, the markdown tree and the+ // completeness flags; cua-driver 0.35 returns the lean markdown-only+ // read unless asked for the full response.+ payload = await driver.call('get_window_state', { ...read, full_output: true });+ } catch (error) {+ // Drivers before 0.35 always return the full response and refuse the+ // unknown argument.+ if (!(error instanceof DriverToolError && error.message.includes('unknown argument full_output'))) {+ throw error;+ }+ payload = await driver.call('get_window_state', read);+ }return parseWindowState(payload, pid, windowId);}diff --git a/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.py b/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.pyindex 632f43376b..8c30c68130 100644--- a/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.py+++ b/libs/cua-driver/hyprland-plugin/proofs/production_app_smoke.py@@ -520,7 +520,10 @@ def observe(mcp, target, filename):"""attempts = []for attempt in range(1, OBSERVATION_ATTEMPTS + 1):- result = content(mcp.tool('get_window_state', {**target, 'timeout_ms': OBSERVATION_TIMEOUT_MS}))+ # full_output: grounding needs structured elements from a complete+ # walk, not the lean 250-node markdown read (cua-driver 0.35).+ result = content(mcp.tool('get_window_state', {**target, 'timeout_ms': OBSERVATION_TIMEOUT_MS,+ 'full_output': True}))assert filename in result.get('window_title', ''), 'snapshot is not the synthetic document'retryable = identity_unproven_only(result)if not retryable:diff --git a/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.py b/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.pyindex 50180ccaca..82e4fcaa1f 100644--- a/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.py+++ b/libs/cua-driver/hyprland-plugin/proofs/production_realapp_proof.py@@ -669,7 +669,8 @@ def run(args):assert len(matches) == 1 and matches[0].get('window_id') == target['window_id'], \'reviewed PID/window identity is stale or ambiguous'result = mcp.tool('get_window_state', {**target,- **({'timeout_ms': OBSERVATION_TIMEOUT_MS} if full else {'max_elements': 100, 'max_depth': 6}),+ **({'timeout_ms': OBSERVATION_TIMEOUT_MS, 'full_output': True} if full+ else {'max_elements': 100, 'max_depth': 6}),**({'session': session} if session else {})})assert not result.get('isError'), resultcontent = result['structuredContent']diff --git a/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.py b/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.pyindex 49c687d9e8..cf2578706f 100644--- a/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.py+++ b/libs/cua-driver/hyprland-plugin/tests/production_app_smoke_test.py@@ -373,7 +373,8 @@ class InputTests(unittest.TestCase):['get_window_state', 'list_windows', 'type_text', 'get_window_state', 'list_windows'])self.assertEqual(calls[2].args[1], {**TARGET, 'text': 'abc', 'delivery_mode': 'background'})for call in (calls[0], calls[3]):- self.assertEqual(call.args[1], {**TARGET, 'timeout_ms': OBSERVATION_TIMEOUT_MS})+ self.assertEqual(call.args[1], {**TARGET, 'timeout_ms': OBSERVATION_TIMEOUT_MS,+ 'full_output': True})self.assertEqual(OBSERVATION_TIMEOUT_MS, 15000)def test_truncated_or_unproven_tree_sends_no_input(self):@@ -582,7 +583,7 @@ class ObjectsSetupTests(unittest.TestCase):mcp.tool.side_effect = observed(INKSCAPE)observe(mcp, TARGET, 'cua-smoke-inkscape.svg')self.assertEqual(mcp.tool.call_args_list[0].args,- ('get_window_state', {**TARGET, 'timeout_ms': 15000}))+ ('get_window_state', {**TARGET, 'timeout_ms': 15000, 'full_output': True}))UNPROVEN = {'degraded': True,diff --git a/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swift b/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swiftindex 3fa85382f2..4bd4118cd2 100644--- a/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swift+++ b/libs/cua-driver/rust/examples/embedded-host-macos/ExampleAgentHarness.swift@@ -158,7 +158,9 @@ let windows = launched["windows"] as? [[String: Any]] ?? []let windowId = windows.first?["window_id"] as? Int ?? 0log("launch_app(Finder): pid: \(pid), windows: \(windows.count)")-let state = call("get_window_state", ["pid": pid, "window_id": windowId])+// `tree_format: "elements"` asks for the structured element array; the+// default read returns only the compact markdown tree.+let state = call("get_window_state", ["pid": pid, "window_id": windowId, "tree_format": "elements"])let images = (state["content"] as? [[String: Any]] ?? []).filter { $0["type"] as? String == "image" }let hasTree = (state["structuredContent"] as? [String: Any])?["elements"] != nildiff --git a/libs/cua-s1/python/src/cua_s1/driver.py b/libs/cua-s1/python/src/cua_s1/driver.pyindex 5031cef185..dcf399639b 100644--- a/libs/cua-s1/python/src/cua_s1/driver.py+++ b/libs/cua-s1/python/src/cua_s1/driver.py@@ -215,14 +215,24 @@ class BaseDriver:return windows[0]def window_state(self, target: WindowTarget) -> WindowSnapshot:- state = self.call(- "get_window_state",- pid=target.pid,- window_id=target.window_id,- session=self.session,- include_accessibility_tree=True,- include_screenshot=False,- )+ read = {+ "pid": target.pid,+ "window_id": target.window_id,+ "session": self.session,+ "include_accessibility_tree": True,+ "include_screenshot": False,+ }+ try:+ # Since cua-driver 0.35 the default read is lean (markdown only,+ # 250 nodes). Actions need the structured elements of a complete+ # walk, so ask for the full response.+ state = self.call("get_window_state", **read, full_output=True)+ except DriverError as error:+ # Drivers before 0.35 always return the full response and refuse+ # the unknown argument.+ if "unknown argument full_output" not in repr(error.details):+ raise+ state = self.call("get_window_state", **read)raw_elements = state.get("elements")if not isinstance(raw_elements, list):raise DriverError(diff --git a/libs/cua-s1/python/tests/test_driver.py b/libs/cua-s1/python/tests/test_driver.pyindex 930b8e833b..1809751a64 100644--- a/libs/cua-s1/python/tests/test_driver.py+++ b/libs/cua-s1/python/tests/test_driver.py@@ -122,3 +122,46 @@ def test_click_payload_and_effect_expectations_conform_to_portable_manifest():"refused",}assert all(tool["name"] != "set_value" for tool in manifest["tools"])+++def test_window_state_asks_for_the_full_response():+ """cua-driver 0.35 returns lean markdown by default; s1 needs elements."""++ calls = []++ class RecordingDriver(BaseDriver):+ def call(self, tool, **args):+ calls.append((tool, args))+ return {+ "snapshot_id": "s1",+ "elements_complete": True,+ "elements": [],+ "tree_markdown": "",+ }++ snapshot = RecordingDriver(session="test").window_state(WindowTarget(pid=10, window_id=20))+ assert snapshot.elements_complete is True+ assert calls[0][0] == "get_window_state"+ assert calls[0][1]["full_output"] is True+++def test_window_state_falls_back_for_drivers_before_0_35():+ calls = []++ class OldDriver(BaseDriver):+ def call(self, tool, **args):+ calls.append(args)+ if "full_output" in args:+ raise DriverError(+ "driver_tool_refused",+ "refused",+ tool=tool,+ details={+ "response": {"error": "get_window_state: unknown argument full_output"}+ },+ )+ return {"snapshot_id": "s1", "elements_complete": True, "elements": []}++ snapshot = OldDriver(session="test").window_state(WindowTarget(pid=10, window_id=20))+ assert snapshot.snapshot_id == "s1"+ assert [("full_output" in args) for args in calls] == [True, False]diff --git a/libs/images/macos/tools/ax_probe.py b/libs/images/macos/tools/ax_probe.pyindex 1566ea387d..8b62760aee 100644--- a/libs/images/macos/tools/ax_probe.py+++ b/libs/images/macos/tools/ax_probe.py@@ -77,6 +77,13 @@ def normalize(text: str) -> str:return "".join(ch for ch in text if ch.isalnum() or ch in ".-")+def read(base: dict[str, Any]) -> dict[str, Any]:+ """get_window_state arguments for `base`. Keys come from `elements` and the+ display from `tree_markdown`; since cua-driver 0.35 a read returns only+ the markdown unless asked for both."""+ return {**base, "tree_format": "both"}++def display_text(state: dict[str, Any]) -> str:"""The window's static texts (Calculator's display), from the treerendering: `elements` lists only actionable nodes."""@@ -121,7 +128,7 @@ async def run(url: str, token: str, log: list[dict[str, Any]]) -> bool:for w in candidates:base = {"pid": pid, "window_id": w["window_id"]}origin = (w.get("bounds", {}).get("x", 0), w.get("bounds", {}).get("y", 0))- state = await call("get_window_state", {**base, "include_screenshot": False})+ state = await call("get_window_state", {**read(base), "include_screenshot": False})elements = state.get("elements", [])log[-1]["elements"] = len(elements)if elements:@@ -141,16 +148,16 @@ async def run(url: str, token: str, log: list[dict[str, Any]]) -> bool:return {"x": f["x"] + f["w"] / 2 - origin[0], "y": f["y"] + f["h"] / 2 - origin[1]}clear = find_key(elements, ("All Clear", "Clear", "AC", "C"))- state = await call("get_window_state", base)+ state = await call("get_window_state", read(base))if clear:await call("click", {**base, **target(clear)})- state = await call("get_window_state", base)+ state = await call("get_window_state", read(base))for key in ("7", "+", "2", "="):element = find_key(state.get("elements", []), KEYS[key])if element is None:raise RuntimeError(f"no {key!r} key in the AX tree")await call("click", {**base, **target(element)})- state = await call("get_window_state", base)+ state = await call("get_window_state", read(base))shown = display_text(state)log.append({"display": shown})calculated = any(normalize(t) == "9" for t in shown.split(" | "))diff --git a/samples/driver/multi-cursor/orchestrator/src/main.rs b/samples/driver/multi-cursor/orchestrator/src/main.rsindex 81c46b5684..00a764357b 100644--- a/samples/driver/multi-cursor/orchestrator/src/main.rs+++ b/samples/driver/multi-cursor/orchestrator/src/main.rs@@ -247,12 +247,15 @@ fn main() {thread::sleep(Duration::from_millis(800)); // let resized layouts settle// Discover UIA elements per corner (field + SUBMIT). GDI has none -> pixel.+ // Reads pass tree_format "both": rows are found in the markdown and their+ // element_token comes from `elements` (cua-driver 0.35 returns only the+ // markdown by default).for c in corners.iter_mut() {let tree = run_call_out(&cua,"get_window_state",&format!(- r#"{{"pid":{},"window_id":{},"capture_mode":"ax","session":"{}"}}"#,+ r#"{{"pid":{},"window_id":{},"capture_mode":"ax","tree_format":"both","session":"{}"}}"#,c.pid, c.hwnd.0 as isize, c.session),);@@ -293,7 +296,7 @@ fn main() {let title = c.title.to_string();handles.push(thread::spawn(move || {let hwnd = HWND(hwnd_addr as *mut _);- let ax = format!(r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"ax","session":"{session}"}}"#);+ let ax = format!(r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"ax","tree_format":"both","session":"{session}"}}"#);let mut seen_records: i64 = -1;for act in rx {// Re-read the window before each action: element tokens are